From noreply at buildbot.pypy.org  Sun Apr  1 08:32:59 2012
From: noreply at buildbot.pypy.org (taavi_burns)
Date: Sun,  1 Apr 2012 08:32:59 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-ufuncs3: Expose and test left_shift and
	right_shift
Message-ID: <20120401063259.5699A820C7@wyvern.cs.uni-duesseldorf.de>

Author: Taavi Burns <taavi.burns at gmail.com>
Branch: numpy-ufuncs3
Changeset: r54114:24b6c8a2f4ce
Date: 2012-04-01 02:32 -0400
http://bitbucket.org/pypy/pypy/changeset/24b6c8a2f4ce/

Log:	Expose and test left_shift and right_shift

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -131,6 +131,8 @@
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
         ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -597,6 +597,13 @@
         assert (bitwise_not(a) == ~a).all()
         assert (invert(a) == ~a).all()
 
+    def test_shift(self):
+        from _numpypy import left_shift, right_shift
+        import sys
+
+        assert (left_shift([5, 1], [2, 31]) == [20, 2**31]).all()
+        assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal

From noreply at buildbot.pypy.org  Sun Apr  1 12:34:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 12:34:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: Extension to cpyext proposed by Stefan
 Behnel: PyErr_{Get, Set}ExcInfo().
Message-ID: <20120401103434.C1140820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54115:623bcea85df3
Date: 2012-04-01 12:30 +0200
http://bitbucket.org/pypy/pypy/changeset/623bcea85df3/

Log:	Extension to cpyext proposed by Stefan Behnel:
	PyErr_{Get,Set}ExcInfo().

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -167,6 +167,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -315,3 +315,63 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        if w_traceback is None or space.is_w(w_traceback, space.w_None):
+            tb = None
+        else:
+            tb = w_traceback
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info

From notifications-noreply at bitbucket.org  Sun Apr  1 14:08:13 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 01 Apr 2012 12:08:13 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120401120813.11994.41784@bitbucket12.managed.contegix.com>

You have received a notification from unbit.

Hi, I forked pypy. My fork is at https://bitbucket.org/unbit/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Sun Apr  1 15:53:02 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  1 Apr 2012 15:53:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix translation (?)
Message-ID: <20120401135302.1BD08820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54116:ba8760573e6d
Date: 2012-04-01 15:52 +0200
http://bitbucket.org/pypy/pypy/changeset/ba8760573e6d/

Log:	fix translation (?)

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -57,7 +57,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+        ptraceback[0] = make_ref(space, operror.get_traceback())
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)

From noreply at buildbot.pypy.org  Sun Apr  1 16:25:47 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 16:25:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: Backed out changeset ba8760573e6d
Message-ID: <20120401142547.47613820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54117:24d65f2b0735
Date: 2012-04-01 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/24d65f2b0735/

Log:	Backed out changeset ba8760573e6d

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -57,7 +57,7 @@
     if operror:
         ptype[0] = make_ref(space, operror.w_type)
         pvalue[0] = make_ref(space, operror.get_w_value(space))
-        ptraceback[0] = make_ref(space, operror.get_traceback())
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
     else:
         ptype[0] = lltype.nullptr(PyObject.TO)
         pvalue[0] = lltype.nullptr(PyObject.TO)

From noreply at buildbot.pypy.org  Sun Apr  1 16:25:48 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 16:25:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix.
Message-ID: <20120401142548.88570820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54118:ee1a3b5094b8
Date: 2012-04-01 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/ee1a3b5094b8/

Log:	Fix.

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -364,10 +365,12 @@
     if w_value is None or space.is_w(w_value, space.w_None):
         operror = None
     else:
-        if w_traceback is None or space.is_w(w_traceback, space.w_None):
-            tb = None
-        else:
-            tb = w_traceback
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
         operror = OperationError(w_type, w_value, tb)
     #
     ec = space.getexecutioncontext()

From noreply at buildbot.pypy.org  Sun Apr  1 16:34:01 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 16:34:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: Cannot do this testing when running
	with "py.test -A".
Message-ID: <20120401143401.A2E22820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54119:7acf5b725d8f
Date: 2012-04-01 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/7acf5b725d8f/

Log:	Cannot do this testing when running with "py.test -A".

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -600,30 +600,33 @@
 class AppTestSysExcInfoDirect:
 
     def setup_method(self, meth):
-        self.seen = []
-        from pypy.module.sys import vm
-        def exc_info_with_tb(*args):
-            self.seen.append("n")     # not optimized
-            return self.old[0](*args)
-        def exc_info_without_tb(*args):
-            self.seen.append("y")     # optimized
-            return self.old[1](*args)
-        self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
-        vm.exc_info_with_tb = exc_info_with_tb
-        vm.exc_info_without_tb = exc_info_without_tb
-        #
-        from pypy.rlib import jit
-        self.old2 = [jit.we_are_jitted]
-        jit.we_are_jitted = lambda: True
+        self.checking = not option.runappdirect
+        if self.checking:
+            self.seen = []
+            from pypy.module.sys import vm
+            def exc_info_with_tb(*args):
+                self.seen.append("n")     # not optimized
+                return self.old[0](*args)
+            def exc_info_without_tb(*args):
+                self.seen.append("y")     # optimized
+                return self.old[1](*args)
+            self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
+            vm.exc_info_with_tb = exc_info_with_tb
+            vm.exc_info_without_tb = exc_info_without_tb
+            #
+            from pypy.rlib import jit
+            self.old2 = [jit.we_are_jitted]
+            jit.we_are_jitted = lambda: True
 
     def teardown_method(self, meth):
-        from pypy.module.sys import vm
-        from pypy.rlib import jit
-        vm.exc_info_with_tb = self.old[0]
-        vm.exc_info_without_tb = self.old[1]
-        jit.we_are_jitted = self.old2[0]
-        #
-        assert ''.join(self.seen) == meth.expected
+        if self.checking:
+            from pypy.module.sys import vm
+            from pypy.rlib import jit
+            vm.exc_info_with_tb = self.old[0]
+            vm.exc_info_without_tb = self.old[1]
+            jit.we_are_jitted = self.old2[0]
+            #
+            assert ''.join(self.seen) == meth.expected
 
     def test_returns_none(self):
         import sys

From noreply at buildbot.pypy.org  Sun Apr  1 16:48:33 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 16:48:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the test: the 'len' attribute is
 now promoted to the common base
Message-ID: <20120401144833.4EF74820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54120:27de13d1a9e7
Date: 2012-04-01 16:48 +0200
http://bitbucket.org/pypy/pypy/changeset/27de13d1a9e7/

Log:	Fix the test: the 'len' attribute is now promoted to the common base
	class W_ArrayBase, which is ok.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -212,7 +212,7 @@
             i19 = int_add(i12, 1)
             setfield_gc(p9, i19, descr=<FieldS .*W_AbstractSeqIterObject.inst_index .*>)
             guard_nonnull_class(p17, 146982464, descr=...)
-            i21 = getfield_gc(p17, descr=<FieldS .*W_ArrayTypei.inst_len .*>)
+            i21 = getfield_gc(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
             i24 = getfield_gc(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)

From noreply at buildbot.pypy.org  Sun Apr  1 17:57:48 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  1 Apr 2012 17:57:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: two missing hop.exception_cannot_occur()
Message-ID: <20120401155748.D8534820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54121:db8d9bfdd9aa
Date: 2012-04-01 17:57 +0200
http://bitbucket.org/pypy/pypy/changeset/db8d9bfdd9aa/

Log:	two missing hop.exception_cannot_occur()

diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -100,6 +100,7 @@
         def specialize_call(self, hop):
             bk = hop.rtyper.annotator.bookkeeper
             s_obj = identity.get_input_annotation(bk)
+            hop.exception_cannot_occur()
             return hop.r_result.rtype_erase(hop, s_obj)
 
     class Entry(ExtRegistryEntry):
@@ -113,6 +114,7 @@
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
+            hop.exception_cannot_occur()
             return hop.args_r[0].rtype_unerase(hop, v)
 
     return erase, unerase

From noreply at buildbot.pypy.org  Sun Apr  1 18:15:45 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 18:15:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add two new passing tests about calls
	to string methods.
Message-ID: <20120401161545.39209820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54122:0e905871fc78
Date: 2012-04-01 16:58 +0200
http://bitbucket.org/pypy/pypy/changeset/0e905871fc78/

Log:	Add two new passing tests about calls to string methods.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -198,3 +198,37 @@
             i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>)
             guard_value(i49, 1, descr=...)
             ''')
+
+    def test_remove_duplicate_method_calls(self):
+        def main(n):
+            lst = []
+            for i in range(n):
+                s = 'Hello %d' % i
+                t = s.lower()   # ID: callone
+                u = s.lower()   # ID: calltwo
+                lst.append(t)
+                lst.append(u)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '''
+            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr 4 r EF=3>)
+            guard_no_exception(descr=...)
+            ''')
+        loop.match_by_id('calltwo', '')    # nothing
+
+    def test_move_method_call_out_of_loop(self):
+        def main(n):
+            lst = []
+            s = 'Hello %d' % n
+            for i in range(n):
+                t = s.lower()   # ID: callone
+                lst.append(t)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '')    # nothing

From noreply at buildbot.pypy.org  Sun Apr  1 18:15:46 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 18:15:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120401161546.C5B34820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54123:3baa10110134
Date: 2012-04-01 18:12 +0200
http://bitbucket.org/pypy/pypy/changeset/3baa10110134/

Log:	merge heads

diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -100,6 +100,7 @@
         def specialize_call(self, hop):
             bk = hop.rtyper.annotator.bookkeeper
             s_obj = identity.get_input_annotation(bk)
+            hop.exception_cannot_occur()
             return hop.r_result.rtype_erase(hop, s_obj)
 
     class Entry(ExtRegistryEntry):
@@ -113,6 +114,7 @@
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
+            hop.exception_cannot_occur()
             return hop.args_r[0].rtype_unerase(hop, v)
 
     return erase, unerase

From noreply at buildbot.pypy.org  Sun Apr  1 20:08:49 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:08:49 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: A branch in which to
 make exception_cannot_occur() or
Message-ID: <20120401180849.D8226820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54124:0d14d5e18743
Date: 2012-04-01 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/0d14d5e18743/

Log:	A branch in which to make exception_cannot_occur() or
	exception_is_here() mandatory in specialize_call()s.


From noreply at buildbot.pypy.org  Sun Apr  1 20:08:51 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:08:51 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: Add the logic to detect
	missing cases.
Message-ID: <20120401180851.2BBC6820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54125:6ec59dc45235
Date: 2012-04-01 19:03 +0200
http://bitbucket.org/pypy/pypy/changeset/6ec59dc45235/

Log:	Add the logic to detect missing cases.

diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -111,25 +111,32 @@
         raise TyperError("don't know about built-in function %r" % (
             self.builtinfunc,))
 
+    def _call(self, hop2, **kwds_i):
+        bltintyper = self.findbltintyper(hop2.rtyper)
+        hop2.llops._called_exception_is_here_or_cannot_occur = False
+        v_result = bltintyper(hop2, **kwds_i)
+        if not hop2.llops._called_exception_is_here_or_cannot_occur:
+            raise TyperError("missing hop.exception_cannot_occur() or "
+                             "hop.exception_is_here() in %s" % bltintyper)
+        return v_result
+
     def rtype_simple_call(self, hop):
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
-        return bltintyper(hop2)
+        return self._call(hop2)
 
     def rtype_call_args(self, hop):
         # calling a built-in function with keyword arguments:
         # mostly for rpython.objectmodel.hint()
         hop, kwds_i = call_args_expand(hop)
 
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
         hop2.r_s_popfirstarg()
         # the RPython-level keyword args are passed with an 'i_' prefix and
         # the corresponding value is an *index* in the hop2 arguments,
         # to be used with hop.inputarg(arg=..)
-        return bltintyper(hop2, **kwds_i)
+        return self._call(hop2, **kwds_i)
 
 
 class BuiltinMethodRepr(Repr):
@@ -264,6 +271,7 @@
     pass
 
 def rtype_OSError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("OSError() should not be called with "
                          "a single argument")
@@ -274,6 +282,7 @@
         r_self.setfield(v_self, 'errno', v_errno, hop.llops)
 
 def rtype_WindowsError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("WindowsError() should not be called with "
                          "a single argument")
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -846,6 +846,7 @@
         return result
 
     def exception_is_here(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
@@ -861,6 +862,7 @@
         self.llops.llop_raising_exceptions = len(self.llops)
 
     def exception_cannot_occur(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:

From noreply at buildbot.pypy.org  Sun Apr  1 20:08:52 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:08:52 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: Add some
 exception_cannot_occur/exception_is_here.
Message-ID: <20120401180852.6A55E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54126:7adda6345851
Date: 2012-04-01 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/7adda6345851/

Log:	Add some exception_cannot_occur/exception_is_here.

diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -111,10 +111,10 @@
             return identity.leave_tunnel(self.bookkeeper)
 
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
-            hop.exception_cannot_occur()
             return hop.args_r[0].rtype_unerase(hop, v)
 
     return erase, unerase
@@ -155,6 +155,7 @@
     def specialize_call(self, hop):
         [v] = hop.inputargs(hop.args_r[0])
         assert isinstance(hop.s_result, annmodel.SomeInteger)
+        hop.exception_cannot_occur()
         return hop.args_r[0].rtype_unerase_int(hop, v)
 
 def ll_unerase_int(gcref):
diff --git a/pypy/rpython/lltypesystem/rbuiltin.py b/pypy/rpython/lltypesystem/rbuiltin.py
--- a/pypy/rpython/lltypesystem/rbuiltin.py
+++ b/pypy/rpython/lltypesystem/rbuiltin.py
@@ -9,6 +9,7 @@
 from pypy.rpython.rbool import bool_repr
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr or hop.args_r[1] == pyobj_repr:
diff --git a/pypy/rpython/ootypesystem/rbuiltin.py b/pypy/rpython/ootypesystem/rbuiltin.py
--- a/pypy/rpython/ootypesystem/rbuiltin.py
+++ b/pypy/rpython/ootypesystem/rbuiltin.py
@@ -7,12 +7,14 @@
 from pypy.rpython.error import TyperError
 
 def rtype_new(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputargs(ootype.Void)
     return hop.genop('new', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_oonewarray(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputarg(ootype.Void, arg=0)
     vlength = hop.inputarg(ootype.Signed, arg=1)
@@ -20,23 +22,27 @@
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_null(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     TYPE = hop.args_s[0].const
     nullvalue = ootype.null(TYPE)
     return hop.inputconst(TYPE, nullvalue)
 
 def rtype_classof(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
     vlist = hop.inputargs(hop.args_r[0])
     return hop.genop('classof', vlist,
                      resulttype = ootype.Class)
 
 def rtype_subclassof(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr, rootype.ooclass_repr)
     return hop.genop('subclassof', vlist,
                      resulttype = ootype.Bool)
 
 def rtype_instanceof(hop):
+    hop.exception_cannot_occur()
     INSTANCE = hop.args_v[1].value
     v_inst = hop.inputarg(hop.args_r[0], arg=0)
     c_cls = hop.inputconst(ootype.Void, INSTANCE)
@@ -44,23 +50,27 @@
                      resulttype=ootype.Bool)
 
 def rtype_runtimenew(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr)
     return hop.genop('runtimenew', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_ooupcast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_oodowncast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_to_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOStaticMeth) or \
            isinstance(hop.args_s[0], annmodel.SomeOOClass) or \
            isinstance(hop.args_s[0].ootype, ootype.OOType)
@@ -68,12 +78,14 @@
     return hop.genop('cast_to_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_from_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.OOType)
     assert isinstance(hop.args_s[1], annmodel.SomeOOObject)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('cast_from_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(ootype.Bool, hop.s_result.const)
 
@@ -99,6 +111,7 @@
     return ootype.subclassof(c1, class_)
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     if hop.args_s[0].is_constant():
 ##        INSTANCE = hop.s_result.rtyper_makerepr(hop.rtyper).lowleveltype
 ##        v_instance = hop.inputconst(ootype.Void, INSTANCE)
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -605,6 +605,7 @@
 
 def rtype_offsetof(hop):
     TYPE, field = hop.inputargs(lltype.Void, lltype.Void)
+    hop.exception_cannot_occur()
     return hop.inputconst(lltype.Signed,
                           llmemory.offsetof(TYPE.value, field.value))
 
@@ -626,6 +627,7 @@
 # keepalive_until_here
 
 def rtype_keepalive_until_here(hop):
+    hop.exception_cannot_occur()
     for v in hop.args_v:
         hop.genop('keepalive', [v], resulttype=lltype.Void)
     return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -107,8 +107,10 @@
     if isinstance(hop.r_result, AbstractRangeRepr):
         if hop.r_result.step != 0:
             c_rng = hop.inputconst(Void, hop.r_result.RANGE)
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrange, c_rng, vstart, vstop)
         else:
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrangest, vstart, vstop, vstep)
     else:
         # cannot build a RANGE object, needs a real list
@@ -117,6 +119,7 @@
         if isinstance(ITEMTYPE, Ptr):
             ITEMTYPE = ITEMTYPE.TO
         cLIST = hop.inputconst(Void, ITEMTYPE)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_range2list, cLIST, vstart, vstop, vstep)
 
 rtype_builtin_xrange = rtype_builtin_range
diff --git a/pypy/rpython/test/test_extregistry.py b/pypy/rpython/test/test_extregistry.py
--- a/pypy/rpython/test/test_extregistry.py
+++ b/pypy/rpython/test/test_extregistry.py
@@ -114,6 +114,7 @@
         _about_ = dummy_func
         s_result_annotation = annmodel.SomeInteger()
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             return hop.inputconst(lltype.Signed, 42)
 
     def func():

From noreply at buildbot.pypy.org  Sun Apr  1 20:08:53 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:08:53 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: A bunch more.
Message-ID: <20120401180853.A3614820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54127:83e14fbf623c
Date: 2012-04-01 19:31 +0200
http://bitbucket.org/pypy/pypy/changeset/83e14fbf623c/

Log:	A bunch more.

diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -419,6 +419,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_identity_hash() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_identityhash', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -441,6 +442,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_id', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -452,6 +454,7 @@
 
     def specialize_call(self, hop):
         vobj, = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         if hop.rtyper.type_system.name == 'lltypesystem':
             from pypy.rpython.lltypesystem import lltype
             if isinstance(vobj.concretetype, lltype.Ptr):
diff --git a/pypy/rpython/lltypesystem/rbuiltin.py b/pypy/rpython/lltypesystem/rbuiltin.py
--- a/pypy/rpython/lltypesystem/rbuiltin.py
+++ b/pypy/rpython/lltypesystem/rbuiltin.py
@@ -34,6 +34,7 @@
     return my_instantiate()
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     s_class = hop.args_s[0]
     assert isinstance(s_class, annmodel.SomePBC)
     if len(s_class.descriptions) != 1:
@@ -47,6 +48,7 @@
     return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops)
 
 def rtype_builtin_hasattr(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr:
@@ -57,6 +59,7 @@
     raise TyperError("hasattr is only suported on a constant or on PyObject")
 
 def rtype_builtin___import__(hop):
+    xxx    # should not be used any more
     args_v = hop.inputargs(*[pyobj_repr for ign in hop.args_r])
     c = hop.inputconst(pyobj_repr, __import__)
     return hop.genop('simple_call', [c] + args_v, resulttype = pyobj_repr)
diff --git a/pypy/rpython/ootypesystem/ooregistry.py b/pypy/rpython/ootypesystem/ooregistry.py
--- a/pypy/rpython/ootypesystem/ooregistry.py
+++ b/pypy/rpython/ootypesystem/ooregistry.py
@@ -22,6 +22,7 @@
                                          annmodel.SomeOOInstance,
                                          annmodel.SomeString))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oostring', vlist, resulttype = ootype.String)
 
 class Entry_oounicode(ExtRegistryEntry):
@@ -38,6 +39,7 @@
         assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint,
                                           annmodel.SomeOOInstance))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oounicode', vlist, resulttype = ootype.Unicode)
     
 
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -205,6 +205,7 @@
 # ____________________________________________________________
 
 def rtype_builtin_bool(hop):
+    xxx   # not called any more?
     assert hop.nb_args == 1
     return hop.args_r[0].rtype_is_true(hop)
 
@@ -451,6 +452,7 @@
     assert hop.args_s[0].is_constant()
     TGT = hop.args_s[0].const
     v_type, v_value = hop.inputargs(lltype.Void, hop.args_r[1])
+    hop.exception_cannot_occur()
     return gen_cast(hop.llops, TGT, v_value)
 
 _cast_to_Signed = {
@@ -615,6 +617,7 @@
 # non-gc objects
 
 def rtype_free_non_gc_object(hop):
+    hop.exception_cannot_occur()
     vinst, = hop.inputargs(hop.args_r[0])
     flavor = hop.args_r[0].gcflavor
     assert flavor != 'gc'
diff --git a/pypy/rpython/rint.py b/pypy/rpython/rint.py
--- a/pypy/rpython/rint.py
+++ b/pypy/rpython/rint.py
@@ -310,6 +310,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_chr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_char', vlist, resulttype=Char)
 
     def rtype_unichr(_, hop):
@@ -317,6 +319,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_unichr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_unichar', vlist, resulttype=UniChar)
 
     def rtype_is_true(self, hop):

From noreply at buildbot.pypy.org  Sun Apr  1 20:08:54 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:08:54 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: More.
Message-ID: <20120401180854.E117A820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54128:2335956a2d4d
Date: 2012-04-01 19:37 +0200
http://bitbucket.org/pypy/pypy/changeset/2335956a2d4d/

Log:	More.

diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -55,6 +55,7 @@
         vtup = hop.inputarg(self, 0)
         LIST = hop.r_result.lowleveltype.TO
         cno = inputconst(Signed, nitems)
+        hop.exception_is_here()
         vlist = hop.gendirectcall(LIST.ll_newlist, cno)
         v_func = hop.inputconst(Void, rlist.dum_nocheck)
         for index in range(nitems):
diff --git a/pypy/rpython/ootypesystem/rtuple.py b/pypy/rpython/ootypesystem/rtuple.py
--- a/pypy/rpython/ootypesystem/rtuple.py
+++ b/pypy/rpython/ootypesystem/rtuple.py
@@ -39,6 +39,7 @@
         RESULT = hop.r_result.lowleveltype
         c_resulttype = inputconst(ootype.Void, RESULT)
         c_length = inputconst(ootype.Signed, len(self.items_r))
+        hop.exception_is_here()
         if isinstance(RESULT, ootype.Array):
             v_list = hop.genop('oonewarray', [c_resulttype, c_length], resulttype=RESULT)
         else:
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -249,6 +249,7 @@
 
 def rtype_builtin_min(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_min, v1, v2)
 
 def ll_min(i1, i2):
@@ -258,6 +259,7 @@
 
 def rtype_builtin_max(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_max, v1, v2)
 
 def ll_max(i1, i2):
@@ -569,6 +571,7 @@
 
 def rtype_raw_malloc(hop):
     v_size, = hop.inputargs(lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_malloc', [v_size], resulttype=llmemory.Address)
 
 def rtype_raw_malloc_usage(hop):
@@ -597,6 +600,7 @@
     if s_addr.is_null_address():
         raise TyperError("raw_memclear(x, n) where x is the constant NULL")
     v_list = hop.inputargs(llmemory.Address, lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_memclear', v_list)
 
 BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -115,6 +115,7 @@
     def rtype_bltn_list(self, hop):
         v_lst = hop.inputarg(self, 0)
         cRESLIST = hop.inputconst(Void, hop.r_result.LIST)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_copy, cRESLIST, v_lst)
 
     def rtype_len(self, hop):

From noreply at buildbot.pypy.org  Sun Apr  1 20:08:56 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:08:56 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: More.
Message-ID: <20120401180856.3D18B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54129:9a1134601e5b
Date: 2012-04-01 19:46 +0200
http://bitbucket.org/pypy/pypy/changeset/9a1134601e5b/

Log:	More.

diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -215,6 +215,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
@@ -397,6 +398,7 @@
         r_obj, = hop.args_r
         v_obj, = hop.inputargs(r_obj)
         ll_fn = r_obj.get_ll_hash_function()
+        hop.exception_is_here()
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rpython/rbool.py b/pypy/rpython/rbool.py
--- a/pypy/rpython/rbool.py
+++ b/pypy/rpython/rbool.py
@@ -34,6 +34,7 @@
 
     def rtype_float(_, hop):
         vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
         return vlist[0]
 
 #
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -205,7 +205,7 @@
 # ____________________________________________________________
 
 def rtype_builtin_bool(hop):
-    xxx   # not called any more?
+    # not called any more?
     assert hop.nb_args == 1
     return hop.args_r[0].rtype_is_true(hop)
 
@@ -536,11 +536,13 @@
 
 def rtype_identity_hash(hop):
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('gc_identityhash', vlist, resulttype=lltype.Signed)
 
 def rtype_runtime_type_info(hop):
     assert isinstance(hop.args_r[0], rptr.PtrRepr)
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('runtime_type_info', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
diff --git a/pypy/rpython/rfloat.py b/pypy/rpython/rfloat.py
--- a/pypy/rpython/rfloat.py
+++ b/pypy/rpython/rfloat.py
@@ -136,7 +136,10 @@
         hop.exception_cannot_occur()
         return hop.genop('cast_float_to_int', vlist, resulttype=Signed)
 
-    rtype_float = rtype_pos
+    def rtype_float(_, hop):
+        vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
+        return vlist[0]
 
     # version picked by specialisation based on which
     # type system rtyping is using, from <type_system>.ll_str module
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -215,4 +215,5 @@
                                      [v_index, v_item])
 
 def rtype_builtin_enumerate(hop):
+    hop.exception_cannot_occur()
     return hop.r_result.r_baseiter.newiter(hop)
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1085,6 +1085,7 @@
                 return annmodel.SomeInteger()
             def specialize_call(self, hop):
                 [v_instance] = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.gendirectcall(ll_my_gethash, v_instance)
 
         def f(n):

From noreply at buildbot.pypy.org  Sun Apr  1 20:08:57 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:08:57 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: More.
Message-ID: <20120401180857.9300E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54130:b39ccd2db7fa
Date: 2012-04-01 19:59 +0200
http://bitbucket.org/pypy/pypy/changeset/b39ccd2db7fa/

Log:	More.

diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -382,7 +382,7 @@
         pass
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 vref_None = non_virtual_ref(None)
 
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -85,7 +85,7 @@
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 
 class rbigint(object):
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -130,6 +130,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 def enum_ops_without_sideeffects(raising_is_ok=False):
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -746,4 +746,5 @@
         assert isinstance(TYPE, GcStruct)
         assert lltype._castdepth(TYPE, OBJECT) > 0
         hop.rtyper.set_type_for_typeptr(vtable, TYPE)
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/module/r_os_stat.py b/pypy/rpython/module/r_os_stat.py
--- a/pypy/rpython/module/r_os_stat.py
+++ b/pypy/rpython/module/r_os_stat.py
@@ -65,4 +65,5 @@
     r_StatResult = hop.rtyper.getrepr(ll_os_stat.s_StatResult)
     [v_result] = hop.inputargs(r_StatResult.r_tuple)
     # no-op conversion from r_StatResult.r_tuple to r_StatResult
+    hop.exception_cannot_occur()
     return v_result
diff --git a/pypy/rpython/rstr.py b/pypy/rpython/rstr.py
--- a/pypy/rpython/rstr.py
+++ b/pypy/rpython/rstr.py
@@ -288,6 +288,8 @@
 
     def rtype_unicode(self, hop):
         if hop.args_s[0].is_constant():
+            # convertion errors occur during annotation, so cannot any more:
+            hop.exception_cannot_occur()
             return hop.inputconst(hop.r_result, hop.s_result.const)
         repr = hop.args_r[0].repr
         v_str = hop.inputarg(repr, 0)

From noreply at buildbot.pypy.org  Sun Apr  1 20:27:55 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:27:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: Bah. This test used to fail if the
 current "nice" level is 17, 18 or
Message-ID: <20120401182755.86263820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54131:76c5931f64cc
Date: 2012-04-01 20:27 +0200
http://bitbucket.org/pypy/pypy/changeset/76c5931f64cc/

Log:	Bah. This test used to fail if the current "nice" level is 17, 18
	or 19.

diff --git a/pypy/translator/c/test/test_extfunc.py b/pypy/translator/c/test/test_extfunc.py
--- a/pypy/translator/c/test/test_extfunc.py
+++ b/pypy/translator/c/test/test_extfunc.py
@@ -919,4 +919,5 @@
             t, cbuilder = self.compile(does_stuff)
             data = cbuilder.cmdexec('')
             res = os.nice(0) + 3
+            if res > 19: res = 19    # xxx Linux specific, probably
             assert data.startswith('os.nice returned %d\n' % res)

From noreply at buildbot.pypy.org  Sun Apr  1 20:34:38 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  1 Apr 2012 20:34:38 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: More.
Message-ID: <20120401183438.6F503820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54132:7d6635820406
Date: 2012-04-01 20:32 +0200
http://bitbucket.org/pypy/pypy/changeset/7d6635820406/

Log:	More.

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1797,6 +1797,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1813,6 +1814,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -86,6 +86,7 @@
         return s_None
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 # ____________________________________________________________
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -22,6 +22,7 @@
                 c_name = hop.inputconst(lltype.Void, 'access_helper')
                 args_v = [hop.inputarg(arg, arg=i)
                           for i, arg in enumerate(hop.args_r)]
+                hop.exception_cannot_occur()
                 return hop.genop('jit_marker', [c_name, c_func] + args_v,
                                  resulttype=hop.r_result)
         return helper
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -155,7 +155,6 @@
     def specialize_call(self, hop):
         [v] = hop.inputargs(hop.args_r[0])
         assert isinstance(hop.s_result, annmodel.SomeInteger)
-        hop.exception_cannot_occur()
         return hop.args_r[0].rtype_unerase_int(hop, v)
 
 def ll_unerase_int(gcref):
@@ -217,6 +216,7 @@
         return hop.genop('cast_opaque_ptr', [v], resulttype=hop.r_result)
 
     def rtype_unerase_int(self, hop, v):
+        hop.exception_cannot_occur()
         return hop.gendirectcall(ll_unerase_int, v)
 
     def rtype_erase_int(self, hop):
@@ -267,6 +267,7 @@
 
     def rtype_unerase_int(self, hop, v):
         c_one = hop.inputconst(lltype.Signed, 1)
+        hop.exception_cannot_occur()
         v2 = hop.genop('oounbox_int', [v], resulttype=hop.r_result)
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -382,6 +382,7 @@
     def compute_result_annotation(self):
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_roots', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
@@ -392,6 +393,7 @@
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_referents', vlist,
                          resulttype = hop.r_result)
 
@@ -402,6 +404,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_memory_usage', vlist,
                          resulttype = hop.r_result)
 
@@ -412,6 +415,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
@@ -430,6 +434,7 @@
         return annmodel.SomeBool()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_is_rpy_instance', vlist,
                          resulttype = hop.r_result)
 
@@ -449,6 +454,7 @@
         classrepr = getclassrepr(hop.rtyper, classdef)
         vtable = classrepr.getvtable()
         assert lltype.typeOf(vtable) == rclass.CLASSTYPE
+        hop.exception_cannot_occur()
         return Constant(vtable, concretetype=rclass.CLASSTYPE)
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py
--- a/pypy/rlib/rstring.py
+++ b/pypy/rlib/rstring.py
@@ -245,5 +245,5 @@
             raise ValueError("Value is not no_nul")
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
diff --git a/pypy/rpython/controllerentry.py b/pypy/rpython/controllerentry.py
--- a/pypy/rpython/controllerentry.py
+++ b/pypy/rpython/controllerentry.py
@@ -201,6 +201,7 @@
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
         assert hop.s_result.is_constant()
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
diff --git a/pypy/translator/cli/dotnet.py b/pypy/translator/cli/dotnet.py
--- a/pypy/translator/cli/dotnet.py
+++ b/pypy/translator/cli/dotnet.py
@@ -459,6 +459,7 @@
             
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         assert hop.args_s[1].is_constant()
         TYPE = hop.args_s[1].const
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
@@ -507,6 +508,7 @@
 
     def specialize_call(self, hop):
         v_obj, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('same_as', [v_obj], hop.r_result.lowleveltype)
 
 def new_array(type, length):
@@ -608,6 +610,7 @@
 
     def specialize_call(self, hop):
         v_type, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('cli_typeof', [v_type], hop.r_result.lowleveltype)
 
 
@@ -626,6 +629,7 @@
         v_obj, = hop.inputargs(*hop.args_r)
         methodname = hop.args_r[0].methodname
         c_methodname = hop.inputconst(ootype.Void, methodname)
+        hop.exception_cannot_occur()
         return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype)
 
 
@@ -647,6 +651,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -668,6 +673,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -701,6 +707,7 @@
 
     def specialize_call(self, hop):
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype)
 
 

From noreply at buildbot.pypy.org  Mon Apr  2 07:51:20 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 07:51:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix for issue #1104,
	allow null bytes in certs
Message-ID: <20120402055120.7EDF0820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54133:5bfd8e84f0f2
Date: 2012-04-02 07:50 +0200
http://bitbucket.org/pypy/pypy/changeset/5bfd8e84f0f2/

Log:	fix for issue #1104, allow null bytes in certs

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -432,7 +432,8 @@
                     raise _ssl_seterror(self.space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return self.space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return self.space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                              length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:

From noreply at buildbot.pypy.org  Mon Apr  2 08:45:22 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 08:45:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: add a debug repr and a passing test
Message-ID: <20120402064522.BADA0820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54134:80e8c18538f3
Date: 2012-04-02 08:44 +0200
http://bitbucket.org/pypy/pypy/changeset/80e8c18538f3/

Log:	add a debug repr and a passing test

diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -219,6 +219,9 @@
         assert isinstance(arr, VirtualSlice)
         return self.child.eval(frame, arr.child)
 
+    def debug_repr(self):
+        return 'VirtualSlice(%s)' % self.child.debug_repr()
+
 class Call1(Signature):
     _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1338,6 +1338,10 @@
         dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0)
         assert str(dims_disagree.value) == \
             "array dimensions must agree except for axis being concatenated"
+        a = array([1, 2, 3, 4, 5, 6])
+        a = (a + a)[::2]
+        b = concatenate((a[:3], a[-3:]))
+        assert (b == [2, 6, 10, 2, 6, 10]).all()
 
     def test_std(self):
         from _numpypy import array

From noreply at buildbot.pypy.org  Mon Apr  2 09:19:32 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 09:19:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: a failing test
Message-ID: <20120402071932.AFED346E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54135:e69f9eaa2352
Date: 2012-04-02 09:15 +0200
http://bitbucket.org/pypy/pypy/changeset/e69f9eaa2352/

Log:	a failing test

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -395,11 +395,21 @@
         assert a[3] == 0.
 
     def test_newaxis(self):
-        from _numpypy import array
+        import math
+        from _numpypy import array, cos, zeros
         from numpypy.core.numeric import newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
+        a = array(range(3))
+        b = array([1, 3])
+        expected = zeros((3, 2))
+        for x in range(3):
+            for y in range(2):
+                expected[x, y] = math.cos(a[x]) * math.cos(b[y])
+        print (cos(a)[:,newaxis] * cos(b).T)
+        print expected
+        assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):
         from _numpypy import array

From noreply at buildbot.pypy.org  Mon Apr  2 09:37:21 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 09:37:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: oops
Message-ID: <20120402073721.B535746E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54136:78c41b32d5f1
Date: 2012-04-02 09:22 +0200
http://bitbucket.org/pypy/pypy/changeset/78c41b32d5f1/

Log:	oops

diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -431,7 +431,7 @@
         frame.cur_value = self.binfunc(self.calc_dtype, frame.cur_value, rval)
 
     def debug_repr(self):
-        return 'ReduceSig(%s)' % (self.name, self.right.debug_repr())
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
 
 class SliceloopSignature(Call2):
     def eval(self, frame, arr):

From noreply at buildbot.pypy.org  Mon Apr  2 09:37:22 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 09:37:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: grumble,
	of course you apply stuff in this order
Message-ID: <20120402073722.EDADA46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54137:73cc3cc4b016
Date: 2012-04-02 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/73cc3cc4b016/

Log:	grumble, of course you apply stuff in this order

diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -211,7 +211,7 @@
     def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        transforms = transforms + [ViewTransform(arr.chunks)]
+        transforms = [ViewTransform(arr.chunks)] + transforms 
         self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
@@ -277,7 +277,7 @@
         from pypy.module.micronumpy.interp_numarray import Call1
 
         assert isinstance(arr, Call1)
-        vtransforms = transforms + [BroadcastTransform(arr.values.shape)]
+        vtransforms = [BroadcastTransform(arr.values.shape)] + transforms
         self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
         self.res._create_iter(iterlist, arraylist, arr.res, transforms)
 
@@ -355,7 +355,7 @@
         from pypy.module.micronumpy.interp_numarray import ResultArray
 
         assert isinstance(arr, ResultArray)
-        rtransforms = transforms + [BroadcastTransform(arr.left.shape)]
+        rtransforms = [BroadcastTransform(arr.left.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -382,7 +382,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
@@ -395,7 +395,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -408,8 +408,8 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -455,7 +455,7 @@
         from pypy.module.micronumpy.interp_numarray import SliceArray
 
         assert isinstance(arr, SliceArray)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -407,8 +407,6 @@
         for x in range(3):
             for y in range(2):
                 expected[x, y] = math.cos(a[x]) * math.cos(b[y])
-        print (cos(a)[:,newaxis] * cos(b).T)
-        print expected
         assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):

From noreply at buildbot.pypy.org  Mon Apr  2 09:45:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 09:45:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: 64-bit compat.
Message-ID: <20120402074500.21BF446E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54138:4b68bb8c88de
Date: 2012-04-02 09:44 +0200
http://bitbucket.org/pypy/pypy/changeset/4b68bb8c88de/

Log:	64-bit compat.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -214,7 +214,7 @@
         loops = log.loops_by_filename(self.filepath)
         loop, = loops
         loop.match_by_id('callone', '''
-            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr 4 r EF=3>)
+            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr . r EF=3>)
             guard_no_exception(descr=...)
             ''')
         loop.match_by_id('calltwo', '')    # nothing

From noreply at buildbot.pypy.org  Mon Apr  2 10:19:09 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 10:19:09 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: Disable a test that
 occasionally crashes because of ll2ctypes not being
Message-ID: <20120402081909.226E246E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54139:2b2762fea668
Date: 2012-04-02 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/2b2762fea668/

Log:	Disable a test that occasionally crashes because of ll2ctypes not
	being thread-safe at all.

diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py
--- a/pypy/module/thread/test/test_ll_thread.py
+++ b/pypy/module/thread/test/test_ll_thread.py
@@ -66,7 +66,6 @@
     def test_gc_locking(self):
         import time
         from pypy.rlib.objectmodel import invoke_around_extcall
-        from pypy.rlib.objectmodel import we_are_translated
         from pypy.rlib.debug import ll_assert
 
         class State:
@@ -129,8 +128,6 @@
             state.finished = 0
             # the next line installs before_extcall() and after_extcall()
             # to be called automatically around external function calls.
-            # When not translated it does not work around time.sleep(),
-            # so we have to call them manually for this test.
             invoke_around_extcall(before_extcall, after_extcall)
 
             g(10, 1)
@@ -142,13 +139,9 @@
                 willing_to_wait_more -= 1
                 done = len(state.answers) == expected
 
-                if not we_are_translated(): before_extcall()
                 time.sleep(0.01)
-                if not we_are_translated(): after_extcall()
 
-            if not we_are_translated(): before_extcall()
             time.sleep(0.1)
-            if not we_are_translated(): after_extcall()
 
             return len(state.answers)
 
@@ -160,12 +153,11 @@
         answers = fn()
         assert answers == expected
 
-class TestRunDirectly(AbstractThreadTests):
-    def getcompiled(self, f, argtypes):
-        return f
-
-    def test_start_new_thread(self):
-        py.test.skip("deadlocks occasionally -- why???")
+#class TestRunDirectly(AbstractThreadTests):
+#    def getcompiled(self, f, argtypes):
+#        return f
+# These are disabled because they crash occasionally for bad reasons
+# related to the fact that ll2ctypes is not at all thread-safe
 
 class TestUsingBoehm(AbstractThreadTests):
     gcpolicy = 'boehm'

From noreply at buildbot.pypy.org  Mon Apr  2 10:19:10 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 10:19:10 +0200 (CEST)
Subject: [pypy-commit] pypy exception-cannot-occur: Close branch ready for
	merge.
Message-ID: <20120402081910.525BD46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: exception-cannot-occur
Changeset: r54140:cbe26c9ba624
Date: 2012-04-02 10:14 +0200
http://bitbucket.org/pypy/pypy/changeset/cbe26c9ba624/

Log:	Close branch ready for merge.


From noreply at buildbot.pypy.org  Mon Apr  2 10:19:12 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 10:19:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge exception-cannot-occur: now we
	have to say either
Message-ID: <20120402081912.8D06F46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54141:5820e4a53018
Date: 2012-04-02 10:17 +0200
http://bitbucket.org/pypy/pypy/changeset/5820e4a53018/

Log:	Merge exception-cannot-occur: now we have to say either
	hop.exception_cannot_occur() or hop.exception_is_here() in all
	specialize_call() methods we write. Previously, it "kind of worked"
	for a while until someone tried to call the special function within
	a try: block.

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1797,6 +1797,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1813,6 +1814,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py
--- a/pypy/module/thread/test/test_ll_thread.py
+++ b/pypy/module/thread/test/test_ll_thread.py
@@ -66,7 +66,6 @@
     def test_gc_locking(self):
         import time
         from pypy.rlib.objectmodel import invoke_around_extcall
-        from pypy.rlib.objectmodel import we_are_translated
         from pypy.rlib.debug import ll_assert
 
         class State:
@@ -129,8 +128,6 @@
             state.finished = 0
             # the next line installs before_extcall() and after_extcall()
             # to be called automatically around external function calls.
-            # When not translated it does not work around time.sleep(),
-            # so we have to call them manually for this test.
             invoke_around_extcall(before_extcall, after_extcall)
 
             g(10, 1)
@@ -142,13 +139,9 @@
                 willing_to_wait_more -= 1
                 done = len(state.answers) == expected
 
-                if not we_are_translated(): before_extcall()
                 time.sleep(0.01)
-                if not we_are_translated(): after_extcall()
 
-            if not we_are_translated(): before_extcall()
             time.sleep(0.1)
-            if not we_are_translated(): after_extcall()
 
             return len(state.answers)
 
@@ -160,12 +153,11 @@
         answers = fn()
         assert answers == expected
 
-class TestRunDirectly(AbstractThreadTests):
-    def getcompiled(self, f, argtypes):
-        return f
-
-    def test_start_new_thread(self):
-        py.test.skip("deadlocks occasionally -- why???")
+#class TestRunDirectly(AbstractThreadTests):
+#    def getcompiled(self, f, argtypes):
+#        return f
+# These are disabled because they crash occasionally for bad reasons
+# related to the fact that ll2ctypes is not at all thread-safe
 
 class TestUsingBoehm(AbstractThreadTests):
     gcpolicy = 'boehm'
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -86,6 +86,7 @@
         return s_None
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 # ____________________________________________________________
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -382,7 +382,7 @@
         pass
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 vref_None = non_virtual_ref(None)
 
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -22,6 +22,7 @@
                 c_name = hop.inputconst(lltype.Void, 'access_helper')
                 args_v = [hop.inputarg(arg, arg=i)
                           for i, arg in enumerate(hop.args_r)]
+                hop.exception_cannot_occur()
                 return hop.genop('jit_marker', [c_name, c_func] + args_v,
                                  resulttype=hop.r_result)
         return helper
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -215,6 +215,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
@@ -397,6 +398,7 @@
         r_obj, = hop.args_r
         v_obj, = hop.inputargs(r_obj)
         ll_fn = r_obj.get_ll_hash_function()
+        hop.exception_is_here()
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
@@ -419,6 +421,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_identity_hash() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_identityhash', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -441,6 +444,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_id', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -452,6 +456,7 @@
 
     def specialize_call(self, hop):
         vobj, = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         if hop.rtyper.type_system.name == 'lltypesystem':
             from pypy.rpython.lltypesystem import lltype
             if isinstance(vobj.concretetype, lltype.Ptr):
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -85,7 +85,7 @@
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 
 class rbigint(object):
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -111,10 +111,10 @@
             return identity.leave_tunnel(self.bookkeeper)
 
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
-            hop.exception_cannot_occur()
             return hop.args_r[0].rtype_unerase(hop, v)
 
     return erase, unerase
@@ -216,6 +216,7 @@
         return hop.genop('cast_opaque_ptr', [v], resulttype=hop.r_result)
 
     def rtype_unerase_int(self, hop, v):
+        hop.exception_cannot_occur()
         return hop.gendirectcall(ll_unerase_int, v)
 
     def rtype_erase_int(self, hop):
@@ -266,6 +267,7 @@
 
     def rtype_unerase_int(self, hop, v):
         c_one = hop.inputconst(lltype.Signed, 1)
+        hop.exception_cannot_occur()
         v2 = hop.genop('oounbox_int', [v], resulttype=hop.r_result)
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -382,6 +382,7 @@
     def compute_result_annotation(self):
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_roots', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
@@ -392,6 +393,7 @@
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_referents', vlist,
                          resulttype = hop.r_result)
 
@@ -402,6 +404,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_memory_usage', vlist,
                          resulttype = hop.r_result)
 
@@ -412,6 +415,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
@@ -430,6 +434,7 @@
         return annmodel.SomeBool()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_is_rpy_instance', vlist,
                          resulttype = hop.r_result)
 
@@ -449,6 +454,7 @@
         classrepr = getclassrepr(hop.rtyper, classdef)
         vtable = classrepr.getvtable()
         assert lltype.typeOf(vtable) == rclass.CLASSTYPE
+        hop.exception_cannot_occur()
         return Constant(vtable, concretetype=rclass.CLASSTYPE)
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py
--- a/pypy/rlib/rstring.py
+++ b/pypy/rlib/rstring.py
@@ -245,5 +245,5 @@
             raise ValueError("Value is not no_nul")
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
diff --git a/pypy/rpython/controllerentry.py b/pypy/rpython/controllerentry.py
--- a/pypy/rpython/controllerentry.py
+++ b/pypy/rpython/controllerentry.py
@@ -201,6 +201,7 @@
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
         assert hop.s_result.is_constant()
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -130,6 +130,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 def enum_ops_without_sideeffects(raising_is_ok=False):
diff --git a/pypy/rpython/lltypesystem/rbuiltin.py b/pypy/rpython/lltypesystem/rbuiltin.py
--- a/pypy/rpython/lltypesystem/rbuiltin.py
+++ b/pypy/rpython/lltypesystem/rbuiltin.py
@@ -9,6 +9,7 @@
 from pypy.rpython.rbool import bool_repr
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr or hop.args_r[1] == pyobj_repr:
@@ -33,6 +34,7 @@
     return my_instantiate()
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     s_class = hop.args_s[0]
     assert isinstance(s_class, annmodel.SomePBC)
     if len(s_class.descriptions) != 1:
@@ -46,6 +48,7 @@
     return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops)
 
 def rtype_builtin_hasattr(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr:
@@ -56,6 +59,7 @@
     raise TyperError("hasattr is only suported on a constant or on PyObject")
 
 def rtype_builtin___import__(hop):
+    xxx    # should not be used any more
     args_v = hop.inputargs(*[pyobj_repr for ign in hop.args_r])
     c = hop.inputconst(pyobj_repr, __import__)
     return hop.genop('simple_call', [c] + args_v, resulttype = pyobj_repr)
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -746,4 +746,5 @@
         assert isinstance(TYPE, GcStruct)
         assert lltype._castdepth(TYPE, OBJECT) > 0
         hop.rtyper.set_type_for_typeptr(vtable, TYPE)
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -55,6 +55,7 @@
         vtup = hop.inputarg(self, 0)
         LIST = hop.r_result.lowleveltype.TO
         cno = inputconst(Signed, nitems)
+        hop.exception_is_here()
         vlist = hop.gendirectcall(LIST.ll_newlist, cno)
         v_func = hop.inputconst(Void, rlist.dum_nocheck)
         for index in range(nitems):
diff --git a/pypy/rpython/module/r_os_stat.py b/pypy/rpython/module/r_os_stat.py
--- a/pypy/rpython/module/r_os_stat.py
+++ b/pypy/rpython/module/r_os_stat.py
@@ -65,4 +65,5 @@
     r_StatResult = hop.rtyper.getrepr(ll_os_stat.s_StatResult)
     [v_result] = hop.inputargs(r_StatResult.r_tuple)
     # no-op conversion from r_StatResult.r_tuple to r_StatResult
+    hop.exception_cannot_occur()
     return v_result
diff --git a/pypy/rpython/ootypesystem/ooregistry.py b/pypy/rpython/ootypesystem/ooregistry.py
--- a/pypy/rpython/ootypesystem/ooregistry.py
+++ b/pypy/rpython/ootypesystem/ooregistry.py
@@ -22,6 +22,7 @@
                                          annmodel.SomeOOInstance,
                                          annmodel.SomeString))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oostring', vlist, resulttype = ootype.String)
 
 class Entry_oounicode(ExtRegistryEntry):
@@ -38,6 +39,7 @@
         assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint,
                                           annmodel.SomeOOInstance))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oounicode', vlist, resulttype = ootype.Unicode)
     
 
diff --git a/pypy/rpython/ootypesystem/rbuiltin.py b/pypy/rpython/ootypesystem/rbuiltin.py
--- a/pypy/rpython/ootypesystem/rbuiltin.py
+++ b/pypy/rpython/ootypesystem/rbuiltin.py
@@ -7,12 +7,14 @@
 from pypy.rpython.error import TyperError
 
 def rtype_new(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputargs(ootype.Void)
     return hop.genop('new', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_oonewarray(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputarg(ootype.Void, arg=0)
     vlength = hop.inputarg(ootype.Signed, arg=1)
@@ -20,23 +22,27 @@
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_null(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     TYPE = hop.args_s[0].const
     nullvalue = ootype.null(TYPE)
     return hop.inputconst(TYPE, nullvalue)
 
 def rtype_classof(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
     vlist = hop.inputargs(hop.args_r[0])
     return hop.genop('classof', vlist,
                      resulttype = ootype.Class)
 
 def rtype_subclassof(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr, rootype.ooclass_repr)
     return hop.genop('subclassof', vlist,
                      resulttype = ootype.Bool)
 
 def rtype_instanceof(hop):
+    hop.exception_cannot_occur()
     INSTANCE = hop.args_v[1].value
     v_inst = hop.inputarg(hop.args_r[0], arg=0)
     c_cls = hop.inputconst(ootype.Void, INSTANCE)
@@ -44,23 +50,27 @@
                      resulttype=ootype.Bool)
 
 def rtype_runtimenew(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr)
     return hop.genop('runtimenew', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_ooupcast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_oodowncast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_to_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOStaticMeth) or \
            isinstance(hop.args_s[0], annmodel.SomeOOClass) or \
            isinstance(hop.args_s[0].ootype, ootype.OOType)
@@ -68,12 +78,14 @@
     return hop.genop('cast_to_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_from_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.OOType)
     assert isinstance(hop.args_s[1], annmodel.SomeOOObject)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('cast_from_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(ootype.Bool, hop.s_result.const)
 
@@ -99,6 +111,7 @@
     return ootype.subclassof(c1, class_)
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     if hop.args_s[0].is_constant():
 ##        INSTANCE = hop.s_result.rtyper_makerepr(hop.rtyper).lowleveltype
 ##        v_instance = hop.inputconst(ootype.Void, INSTANCE)
diff --git a/pypy/rpython/ootypesystem/rtuple.py b/pypy/rpython/ootypesystem/rtuple.py
--- a/pypy/rpython/ootypesystem/rtuple.py
+++ b/pypy/rpython/ootypesystem/rtuple.py
@@ -39,6 +39,7 @@
         RESULT = hop.r_result.lowleveltype
         c_resulttype = inputconst(ootype.Void, RESULT)
         c_length = inputconst(ootype.Signed, len(self.items_r))
+        hop.exception_is_here()
         if isinstance(RESULT, ootype.Array):
             v_list = hop.genop('oonewarray', [c_resulttype, c_length], resulttype=RESULT)
         else:
diff --git a/pypy/rpython/rbool.py b/pypy/rpython/rbool.py
--- a/pypy/rpython/rbool.py
+++ b/pypy/rpython/rbool.py
@@ -34,6 +34,7 @@
 
     def rtype_float(_, hop):
         vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
         return vlist[0]
 
 #
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -111,25 +111,32 @@
         raise TyperError("don't know about built-in function %r" % (
             self.builtinfunc,))
 
+    def _call(self, hop2, **kwds_i):
+        bltintyper = self.findbltintyper(hop2.rtyper)
+        hop2.llops._called_exception_is_here_or_cannot_occur = False
+        v_result = bltintyper(hop2, **kwds_i)
+        if not hop2.llops._called_exception_is_here_or_cannot_occur:
+            raise TyperError("missing hop.exception_cannot_occur() or "
+                             "hop.exception_is_here() in %s" % bltintyper)
+        return v_result
+
     def rtype_simple_call(self, hop):
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
-        return bltintyper(hop2)
+        return self._call(hop2)
 
     def rtype_call_args(self, hop):
         # calling a built-in function with keyword arguments:
         # mostly for rpython.objectmodel.hint()
         hop, kwds_i = call_args_expand(hop)
 
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
         hop2.r_s_popfirstarg()
         # the RPython-level keyword args are passed with an 'i_' prefix and
         # the corresponding value is an *index* in the hop2 arguments,
         # to be used with hop.inputarg(arg=..)
-        return bltintyper(hop2, **kwds_i)
+        return self._call(hop2, **kwds_i)
 
 
 class BuiltinMethodRepr(Repr):
@@ -198,6 +205,7 @@
 # ____________________________________________________________
 
 def rtype_builtin_bool(hop):
+    # not called any more?
     assert hop.nb_args == 1
     return hop.args_r[0].rtype_is_true(hop)
 
@@ -241,6 +249,7 @@
 
 def rtype_builtin_min(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_min, v1, v2)
 
 def ll_min(i1, i2):
@@ -250,6 +259,7 @@
 
 def rtype_builtin_max(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_max, v1, v2)
 
 def ll_max(i1, i2):
@@ -264,6 +274,7 @@
     pass
 
 def rtype_OSError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("OSError() should not be called with "
                          "a single argument")
@@ -274,6 +285,7 @@
         r_self.setfield(v_self, 'errno', v_errno, hop.llops)
 
 def rtype_WindowsError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("WindowsError() should not be called with "
                          "a single argument")
@@ -442,6 +454,7 @@
     assert hop.args_s[0].is_constant()
     TGT = hop.args_s[0].const
     v_type, v_value = hop.inputargs(lltype.Void, hop.args_r[1])
+    hop.exception_cannot_occur()
     return gen_cast(hop.llops, TGT, v_value)
 
 _cast_to_Signed = {
@@ -523,11 +536,13 @@
 
 def rtype_identity_hash(hop):
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('gc_identityhash', vlist, resulttype=lltype.Signed)
 
 def rtype_runtime_type_info(hop):
     assert isinstance(hop.args_r[0], rptr.PtrRepr)
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('runtime_type_info', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
@@ -558,6 +573,7 @@
 
 def rtype_raw_malloc(hop):
     v_size, = hop.inputargs(lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_malloc', [v_size], resulttype=llmemory.Address)
 
 def rtype_raw_malloc_usage(hop):
@@ -586,6 +602,7 @@
     if s_addr.is_null_address():
         raise TyperError("raw_memclear(x, n) where x is the constant NULL")
     v_list = hop.inputargs(llmemory.Address, lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_memclear', v_list)
 
 BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc
@@ -596,6 +613,7 @@
 
 def rtype_offsetof(hop):
     TYPE, field = hop.inputargs(lltype.Void, lltype.Void)
+    hop.exception_cannot_occur()
     return hop.inputconst(lltype.Signed,
                           llmemory.offsetof(TYPE.value, field.value))
 
@@ -605,6 +623,7 @@
 # non-gc objects
 
 def rtype_free_non_gc_object(hop):
+    hop.exception_cannot_occur()
     vinst, = hop.inputargs(hop.args_r[0])
     flavor = hop.args_r[0].gcflavor
     assert flavor != 'gc'
@@ -617,6 +636,7 @@
 # keepalive_until_here
 
 def rtype_keepalive_until_here(hop):
+    hop.exception_cannot_occur()
     for v in hop.args_v:
         hop.genop('keepalive', [v], resulttype=lltype.Void)
     return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/rfloat.py b/pypy/rpython/rfloat.py
--- a/pypy/rpython/rfloat.py
+++ b/pypy/rpython/rfloat.py
@@ -136,7 +136,10 @@
         hop.exception_cannot_occur()
         return hop.genop('cast_float_to_int', vlist, resulttype=Signed)
 
-    rtype_float = rtype_pos
+    def rtype_float(_, hop):
+        vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
+        return vlist[0]
 
     # version picked by specialisation based on which
     # type system rtyping is using, from <type_system>.ll_str module
diff --git a/pypy/rpython/rint.py b/pypy/rpython/rint.py
--- a/pypy/rpython/rint.py
+++ b/pypy/rpython/rint.py
@@ -310,6 +310,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_chr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_char', vlist, resulttype=Char)
 
     def rtype_unichr(_, hop):
@@ -317,6 +319,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_unichr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_unichar', vlist, resulttype=UniChar)
 
     def rtype_is_true(self, hop):
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -115,6 +115,7 @@
     def rtype_bltn_list(self, hop):
         v_lst = hop.inputarg(self, 0)
         cRESLIST = hop.inputconst(Void, hop.r_result.LIST)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_copy, cRESLIST, v_lst)
 
     def rtype_len(self, hop):
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -107,8 +107,10 @@
     if isinstance(hop.r_result, AbstractRangeRepr):
         if hop.r_result.step != 0:
             c_rng = hop.inputconst(Void, hop.r_result.RANGE)
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrange, c_rng, vstart, vstop)
         else:
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrangest, vstart, vstop, vstep)
     else:
         # cannot build a RANGE object, needs a real list
@@ -117,6 +119,7 @@
         if isinstance(ITEMTYPE, Ptr):
             ITEMTYPE = ITEMTYPE.TO
         cLIST = hop.inputconst(Void, ITEMTYPE)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_range2list, cLIST, vstart, vstop, vstep)
 
 rtype_builtin_xrange = rtype_builtin_range
@@ -212,4 +215,5 @@
                                      [v_index, v_item])
 
 def rtype_builtin_enumerate(hop):
+    hop.exception_cannot_occur()
     return hop.r_result.r_baseiter.newiter(hop)
diff --git a/pypy/rpython/rstr.py b/pypy/rpython/rstr.py
--- a/pypy/rpython/rstr.py
+++ b/pypy/rpython/rstr.py
@@ -288,6 +288,8 @@
 
     def rtype_unicode(self, hop):
         if hop.args_s[0].is_constant():
+            # convertion errors occur during annotation, so cannot any more:
+            hop.exception_cannot_occur()
             return hop.inputconst(hop.r_result, hop.s_result.const)
         repr = hop.args_r[0].repr
         v_str = hop.inputarg(repr, 0)
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -846,6 +846,7 @@
         return result
 
     def exception_is_here(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
@@ -861,6 +862,7 @@
         self.llops.llop_raising_exceptions = len(self.llops)
 
     def exception_cannot_occur(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
diff --git a/pypy/rpython/test/test_extregistry.py b/pypy/rpython/test/test_extregistry.py
--- a/pypy/rpython/test/test_extregistry.py
+++ b/pypy/rpython/test/test_extregistry.py
@@ -114,6 +114,7 @@
         _about_ = dummy_func
         s_result_annotation = annmodel.SomeInteger()
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             return hop.inputconst(lltype.Signed, 42)
 
     def func():
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1085,6 +1085,7 @@
                 return annmodel.SomeInteger()
             def specialize_call(self, hop):
                 [v_instance] = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.gendirectcall(ll_my_gethash, v_instance)
 
         def f(n):
diff --git a/pypy/translator/cli/dotnet.py b/pypy/translator/cli/dotnet.py
--- a/pypy/translator/cli/dotnet.py
+++ b/pypy/translator/cli/dotnet.py
@@ -459,6 +459,7 @@
             
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         assert hop.args_s[1].is_constant()
         TYPE = hop.args_s[1].const
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
@@ -507,6 +508,7 @@
 
     def specialize_call(self, hop):
         v_obj, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('same_as', [v_obj], hop.r_result.lowleveltype)
 
 def new_array(type, length):
@@ -608,6 +610,7 @@
 
     def specialize_call(self, hop):
         v_type, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('cli_typeof', [v_type], hop.r_result.lowleveltype)
 
 
@@ -626,6 +629,7 @@
         v_obj, = hop.inputargs(*hop.args_r)
         methodname = hop.args_r[0].methodname
         c_methodname = hop.inputconst(ootype.Void, methodname)
+        hop.exception_cannot_occur()
         return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype)
 
 
@@ -647,6 +651,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -668,6 +673,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -701,6 +707,7 @@
 
     def specialize_call(self, hop):
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype)
 
 

From noreply at buildbot.pypy.org  Mon Apr  2 10:27:24 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 10:27:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Export tp_doc.
Message-ID: <20120402082724.4DAC846E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54142:171b149844d2
Date: 2012-04-02 10:26 +0200
http://bitbucket.org/pypy/pypy/changeset/171b149844d2/

Log:	Export tp_doc.

diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -307,6 +307,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -624,7 +626,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses

From noreply at buildbot.pypy.org  Mon Apr  2 10:34:10 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 10:34:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for issue1111: wrap_objobjargproc()
 returns None in CPython.
Message-ID: <20120402083410.C650946E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54143:7da597d3c86d
Date: 2012-04-02 10:33 +0200
http://bitbucket.org/pypy/pypy/changeset/7da597d3c86d/

Log:	Fix for issue1111: wrap_objobjargproc() returns None in CPython.

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -183,7 +183,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -415,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -426,6 +429,8 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [

From noreply at buildbot.pypy.org  Mon Apr  2 10:51:38 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 10:51:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix for issue1107: "x in y"
	must return a bool.
Message-ID: <20120402085138.CDB0A46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54144:849cc68d148d
Date: 2012-04-02 10:47 +0200
http://bitbucket.org/pypy/pypy/changeset/849cc68d148d/

Log:	Test and fix for issue1107: "x in y" must return a bool.

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,6 +167,7 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
@@ -174,7 +175,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -432,6 +432,34 @@
         res = obj.__setitem__('foo', None)
         assert res is None
 
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
+
     def test_tp_iter(self):
         module = self.import_extension('foo', [
            ("tp_iter", "METH_O",

From noreply at buildbot.pypy.org  Mon Apr  2 11:21:38 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 11:21:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: Re-add an optimization that was
 incorrectly implemented and
Message-ID: <20120402092138.8D12646E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54145:4767e4614bfa
Date: 2012-04-02 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/4767e4614bfa/

Log:	Re-add an optimization that was incorrectly implemented and so
	killed by the merge of list strategies.

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -43,6 +43,13 @@
     return w_eq
 type_eq._annspecialcase_ = 'specialize:memo'
 
+def list_iter(space):
+    "Utility that returns the app-level descriptor list.__iter__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_list,
+                                               '__iter__')
+    return w_iter
+list_iter._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     w_type = space.type(w_obj)
     typename = w_type.getname(space)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -439,6 +439,8 @@
             t = w_obj.getitems()
         elif isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
+        elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            t = w_obj.getitems()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
         if expected_length != -1 and len(t) != expected_length:
@@ -456,6 +458,8 @@
             return w_obj.listview_str()
         if isinstance(w_obj, W_StringObject):
             return w_obj.listview_str()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_str()
         return None
 
     def listview_int(self, w_obj):
@@ -465,8 +469,14 @@
             return w_obj.listview_int()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_int()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_int()
         return None
 
+    def _uses_list_iter(self, w_obj):
+        from pypy.objspace.descroperation import list_iter
+        return self.lookup(w_obj, '__iter__') is list_iter(self)
+
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1186,14 +1186,23 @@
         # of dicts, because the OrderedDict in the stdlib relies on this.
         # we extend the use case to lists and sets, i.e. all types that have
         # strategies, to avoid surprizes depending on the strategy.
-        for base, arg in [(list, []), (list, [5]), (list, ['x']),
-                          (set, []),  (set,  [5]), (set,  ['x']),
-                          (dict, []), (dict, [(5,6)]), (dict, [('x',7)])]:
+        class X: pass
+        for base, arg in [
+                (list, []), (list, [5]), (list, ['x']), (list, [X]),
+                (set, []),  (set,  [5]), (set,  ['x']), (set, [X]),
+                (dict, []), (dict, [(5,6)]), (dict, [('x',7)]), (dict, [(X,8)]),
+                ]:
             print base, arg
             class SubClass(base):
                 def __iter__(self):
                     return iter("foobar")
             assert list(SubClass(arg)) == ['f', 'o', 'o', 'b', 'a', 'r']
+            class Sub2(base):
+                pass
+            assert list(Sub2(arg)) == list(base(arg))
+            s = set()
+            s.update(Sub2(arg))
+            assert s == set(base(arg))
 
 class AppTestForRangeLists(AppTestW_ListObject):
 

From noreply at buildbot.pypy.org  Mon Apr  2 12:11:13 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 12:11:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: implement repeat. a tad inefficient,
 but I could not care less
Message-ID: <20120402101113.E9E1C46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54146:5da12de5d696
Date: 2012-04-02 10:41 +0200
http://bitbucket.org/pypy/pypy/changeset/5da12de5d696/

Log:	implement repeat. a tad inefficient, but I could not care less

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -1261,6 +1261,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1399,6 +1399,15 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):

From noreply at buildbot.pypy.org  Mon Apr  2 12:11:15 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 12:11:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: repeat as a method
Message-ID: <20120402101115.8374846E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54147:6e49d0a11647
Date: 2012-04-02 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/6e49d0a11647/

Log:	repeat as a method

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -673,6 +673,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -1411,6 +1415,7 @@
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1400,13 +1400,14 @@
         assert array(3.0).nbytes == 8
 
     def test_repeat(self):
-        from _numpypy import repeat
+        from _numpypy import repeat, array
         assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
                                                 3, 3, 3, 4, 4, 4]).all()
         assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
                                                         [3, 4]]).all()
         assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
                                                         4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
 class AppTestMultiDim(BaseNumpyAppTest):

From noreply at buildbot.pypy.org  Mon Apr  2 12:11:16 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 12:11:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120402101116.BE47546E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54148:5708f4c44e53
Date: 2012-04-02 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/5708f4c44e53/

Log:	merge

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -673,6 +673,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -1261,6 +1265,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1386,6 +1415,7 @@
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1399,6 +1399,16 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat, array
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):

From noreply at buildbot.pypy.org  Mon Apr  2 12:20:15 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 12:20:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: Translation fix.
Message-ID: <20120402102015.BCFC646E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54149:53c5958727e6
Date: 2012-04-02 12:18 +0200
http://bitbucket.org/pypy/pypy/changeset/53c5958727e6/

Log:	Translation fix.

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -173,7 +173,8 @@
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.wrap(bool(res))
 

From noreply at buildbot.pypy.org  Mon Apr  2 12:20:17 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 12:20:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120402102017.0387A46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54150:c519e300f325
Date: 2012-04-02 12:19 +0200
http://bitbucket.org/pypy/pypy/changeset/c519e300f325/

Log:	merge heads

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -673,6 +673,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -1261,6 +1265,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1386,6 +1415,7 @@
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1399,6 +1399,16 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat, array
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):

From noreply at buildbot.pypy.org  Mon Apr  2 14:04:21 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 14:04:21 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Sprint announcement, first draft
Message-ID: <20120402120421.B873D46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4169:6da880f2846e
Date: 2012-04-02 14:03 +0200
http://bitbucket.org/pypy/extradoc/changeset/6da880f2846e/

Log:	Sprint announcement, first draft

diff --git a/sprintinfo/leipzig2012/announce.txt b/sprintinfo/leipzig2012/announce.txt
new file mode 100644
--- /dev/null
+++ b/sprintinfo/leipzig2012/announce.txt
@@ -0,0 +1,47 @@
+Leipzig PyPy sprint June 22 - 27, 2012
+======================================
+
+The next PyPy sprint will be held in the Python Academy of Leipzig,
+Germany, from the 22nd to the 27th of June 2012.  This is a fully public
+sprint, everyone is welcome to join us.
+
+Topics and goals
+----------------
+
+Open. (to be defined more precisely)
+
+Location
+--------
+
+Python Academy Leipzig, Germany.
+http://www.python-academy.com/center/find.html
+Thanks to Mike M&#252;ller for inviting us.
+
+The room holds 8 - 12 people depending how much space each individual
+needs.  Drinks, snacks and a simple lunch, e.g. pizza or other fast food
+is provided.  Dinner would be extra.
+
+Some accommodations:
+http://www.python-academy.com/center/accommodation.html
+Pretty much all hotels and hostels in the city center
+are fine too. The tram needs only 16 minutes from the
+central station to the Academy location.
+
+Grants
+------
+
+For students, we have the possibility to support some costs via PyPy
+funds.  Additionally, we can support you applying for grants from the
+PSF and other sources.
+
+Registration
+------------
+
+If you'd like to come, please *sign up* either by announcing yourself on
+`pypy-dev`_, or by directly adding yourself to the `list of people`_.
+(We need to have a head count for the organization.)  If you are new to
+the project please drop a note about your interests and post any
+questions.
+
+.. _`pypy-dev`: http://mail.python.org/mailman/listinfo/pypy-dev
+.. _`list of poeple`: https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leipzig2012/people.txt
diff --git a/sprintinfo/leipzig2012/people.txt b/sprintinfo/leipzig2012/people.txt
new file mode 100644
--- /dev/null
+++ b/sprintinfo/leipzig2012/people.txt
@@ -0,0 +1,13 @@
+People coming to the Leipzig sprint June 2012
+=============================================
+
+People who have a ``?`` in their arrive/depart or accomodation
+column are known to be coming but there are no details 
+available yet from them.
+
+==================== ============== =====================
+    Name              Arrive/Depart     Accomodation 
+==================== ============== =====================
+Armin Rigo           ?              ?
+==================== ============== =====================
+

From noreply at buildbot.pypy.org  Mon Apr  2 14:10:28 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 14:10:28 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Initial goals
Message-ID: <20120402121028.3562946E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4170:4ae5466dbaf3
Date: 2012-04-02 14:10 +0200
http://bitbucket.org/pypy/extradoc/changeset/4ae5466dbaf3/

Log:	Initial goals

diff --git a/sprintinfo/leipzig2012/announce.txt b/sprintinfo/leipzig2012/announce.txt
--- a/sprintinfo/leipzig2012/announce.txt
+++ b/sprintinfo/leipzig2012/announce.txt
@@ -8,7 +8,10 @@
 Topics and goals
 ----------------
 
-Open. (to be defined more precisely)
+Open.  Here are some goals:
+
+- numpy: progress towards completing the ``numpypy`` module
+- stm: try out the ``transaction`` module on real code
 
 Location
 --------

From noreply at buildbot.pypy.org  Mon Apr  2 14:15:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 14:15:00 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Make the dates more precise.
Message-ID: <20120402121500.1462146E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4171:e2291545b923
Date: 2012-04-02 14:14 +0200
http://bitbucket.org/pypy/extradoc/changeset/e2291545b923/

Log:	Make the dates more precise.

diff --git a/sprintinfo/leipzig2012/announce.txt b/sprintinfo/leipzig2012/announce.txt
--- a/sprintinfo/leipzig2012/announce.txt
+++ b/sprintinfo/leipzig2012/announce.txt
@@ -3,7 +3,8 @@
 
 The next PyPy sprint will be held in the Python Academy of Leipzig,
 Germany, from the 22nd to the 27th of June 2012.  This is a fully public
-sprint, everyone is welcome to join us.
+sprint, everyone is welcome to join us.  All days are full sprint days,
+so it is recommended to arrive the 21st and leave the 28th.
 
 Topics and goals
 ----------------

From noreply at buildbot.pypy.org  Mon Apr  2 15:40:56 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 15:40:56 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Update.
Message-ID: <20120402134056.62DB446E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4172:71246f66f47a
Date: 2012-04-02 15:40 +0200
http://bitbucket.org/pypy/extradoc/changeset/71246f66f47a/

Log:	Update.

diff --git a/sprintinfo/leipzig2012/announce.txt b/sprintinfo/leipzig2012/announce.txt
--- a/sprintinfo/leipzig2012/announce.txt
+++ b/sprintinfo/leipzig2012/announce.txt
@@ -1,18 +1,31 @@
 Leipzig PyPy sprint June 22 - 27, 2012
 ======================================
 
-The next PyPy sprint will be held in the Python Academy of Leipzig,
-Germany, from the 22nd to the 27th of June 2012.  This is a fully public
-sprint, everyone is welcome to join us.  All days are full sprint days,
-so it is recommended to arrive the 21st and leave the 28th.
+The next PyPy sprint will be held --- for the first time in a while ---
+in a place where we haven't been so far: Leipzig, Germany, at the
+`Python Academy`_'s Teaching Center.  It will take place from the 22nd
+to the 27th of June 2012, before EuroPython.  Thanks to Mike M&#252;ller for
+organizing it!
+
+.. _`Python Academy`: http://www.python-academy.com/
+
+This is a fully public sprint, everyone is welcome to join us.  All days are
+full sprint days, so it is recommended to arrive the 21st and leave the 28th.
+
 
 Topics and goals
 ----------------
 
 Open.  Here are some goals:
 
-- numpy: progress towards completing the ``numpypy`` module
-- stm: try out the ``transaction`` module on real code
+- numpy: progress towards completing the ``numpypy`` module; try to
+  use it in real code
+
+- stm: progress on Transactional Memory; try out the ``transaction``
+  module on real code.
+
+- any other PyPy-related topic is fine too.
+
 
 Location
 --------
@@ -31,6 +44,7 @@
 are fine too. The tram needs only 16 minutes from the
 central station to the Academy location.
 
+
 Grants
 ------
 
@@ -38,6 +52,7 @@
 funds.  Additionally, we can support you applying for grants from the
 PSF and other sources.
 
+
 Registration
 ------------
 
@@ -48,4 +63,4 @@
 questions.
 
 .. _`pypy-dev`: http://mail.python.org/mailman/listinfo/pypy-dev
-.. _`list of poeple`: https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leipzig2012/people.txt
+.. _`list of people`: https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leipzig2012/people.txt

From noreply at buildbot.pypy.org  Mon Apr  2 15:40:57 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 15:40:57 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Also add the html version,
 to be linked from the blog post.
Message-ID: <20120402134057.94DBC46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4173:477481830a61
Date: 2012-04-02 15:40 +0200
http://bitbucket.org/pypy/extradoc/changeset/477481830a61/

Log:	Also add the html version, to be linked from the blog post.

diff --git a/sprintinfo/leipzig2012/announce.html b/sprintinfo/leipzig2012/announce.html
new file mode 100644
--- /dev/null
+++ b/sprintinfo/leipzig2012/announce.html
@@ -0,0 +1,342 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<meta name="generator" content="Docutils 0.4: http://docutils.sourceforge.net/" />
+<title>Leipzig PyPy sprint June 22 - 27, 2012</title>
+<style type="text/css">
+
+/*
+:Author: David Goodger
+:Contact: goodger at users.sourceforge.net
+:Date: $Date: 2005-12-18 01:56:14 +0100 (Sun, 18 Dec 2005) $
+:Revision: $Revision: 4224 $
+:Copyright: This stylesheet has been placed in the public domain.
+
+Default cascading style sheet for the HTML output of Docutils.
+
+See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to
+customize this style sheet.
+*/
+
+/* used to remove borders from tables and images */
+.borderless, table.borderless td, table.borderless th {
+  border: 0 }
+
+table.borderless td, table.borderless th {
+  /* Override padding for "table.docutils td" with "! important".
+     The right padding separates the table cells. */
+  padding: 0 0.5em 0 0 ! important }
+
+.first {
+  /* Override more specific margin styles with "! important". */
+  margin-top: 0 ! important }
+
+.last, .with-subtitle {
+  margin-bottom: 0 ! important }
+
+.hidden {
+  display: none }
+
+a.toc-backref {
+  text-decoration: none ;
+  color: black }
+
+blockquote.epigraph {
+  margin: 2em 5em ; }
+
+dl.docutils dd {
+  margin-bottom: 0.5em }
+
+/* Uncomment (and remove this text!) to get bold-faced definition list terms
+dl.docutils dt {
+  font-weight: bold }
+*/
+
+div.abstract {
+  margin: 2em 5em }
+
+div.abstract p.topic-title {
+  font-weight: bold ;
+  text-align: center }
+
+div.admonition, div.attention, div.caution, div.danger, div.error,
+div.hint, div.important, div.note, div.tip, div.warning {
+  margin: 2em ;
+  border: medium outset ;
+  padding: 1em }
+
+div.admonition p.admonition-title, div.hint p.admonition-title,
+div.important p.admonition-title, div.note p.admonition-title,
+div.tip p.admonition-title {
+  font-weight: bold ;
+  font-family: sans-serif }
+
+div.attention p.admonition-title, div.caution p.admonition-title,
+div.danger p.admonition-title, div.error p.admonition-title,
+div.warning p.admonition-title {
+  color: red ;
+  font-weight: bold ;
+  font-family: sans-serif }
+
+/* Uncomment (and remove this text!) to get reduced vertical space in
+   compound paragraphs.
+div.compound .compound-first, div.compound .compound-middle {
+  margin-bottom: 0.5em }
+
+div.compound .compound-last, div.compound .compound-middle {
+  margin-top: 0.5em }
+*/
+
+div.dedication {
+  margin: 2em 5em ;
+  text-align: center ;
+  font-style: italic }
+
+div.dedication p.topic-title {
+  font-weight: bold ;
+  font-style: normal }
+
+div.figure {
+  margin-left: 2em ;
+  margin-right: 2em }
+
+div.footer, div.header {
+  clear: both;
+  font-size: smaller }
+
+div.line-block {
+  display: block ;
+  margin-top: 1em ;
+  margin-bottom: 1em }
+
+div.line-block div.line-block {
+  margin-top: 0 ;
+  margin-bottom: 0 ;
+  margin-left: 1.5em }
+
+div.sidebar {
+  margin-left: 1em ;
+  border: medium outset ;
+  padding: 1em ;
+  background-color: #ffffee ;
+  width: 40% ;
+  float: right ;
+  clear: right }
+
+div.sidebar p.rubric {
+  font-family: sans-serif ;
+  font-size: medium }
+
+div.system-messages {
+  margin: 5em }
+
+div.system-messages h1 {
+  color: red }
+
+div.system-message {
+  border: medium outset ;
+  padding: 1em }
+
+div.system-message p.system-message-title {
+  color: red ;
+  font-weight: bold }
+
+div.topic {
+  margin: 2em }
+
+h1.section-subtitle, h2.section-subtitle, h3.section-subtitle,
+h4.section-subtitle, h5.section-subtitle, h6.section-subtitle {
+  margin-top: 0.4em }
+
+h1.title {
+  text-align: center }
+
+h2.subtitle {
+  text-align: center }
+
+hr.docutils {
+  width: 75% }
+
+img.align-left {
+  clear: left }
+
+img.align-right {
+  clear: right }
+
+ol.simple, ul.simple {
+  margin-bottom: 1em }
+
+ol.arabic {
+  list-style: decimal }
+
+ol.loweralpha {
+  list-style: lower-alpha }
+
+ol.upperalpha {
+  list-style: upper-alpha }
+
+ol.lowerroman {
+  list-style: lower-roman }
+
+ol.upperroman {
+  list-style: upper-roman }
+
+p.attribution {
+  text-align: right ;
+  margin-left: 50% }
+
+p.caption {
+  font-style: italic }
+
+p.credits {
+  font-style: italic ;
+  font-size: smaller }
+
+p.label {
+  white-space: nowrap }
+
+p.rubric {
+  font-weight: bold ;
+  font-size: larger ;
+  color: maroon ;
+  text-align: center }
+
+p.sidebar-title {
+  font-family: sans-serif ;
+  font-weight: bold ;
+  font-size: larger }
+
+p.sidebar-subtitle {
+  font-family: sans-serif ;
+  font-weight: bold }
+
+p.topic-title {
+  font-weight: bold }
+
+pre.address {
+  margin-bottom: 0 ;
+  margin-top: 0 ;
+  font-family: serif ;
+  font-size: 100% }
+
+pre.literal-block, pre.doctest-block {
+  margin-left: 2em ;
+  margin-right: 2em ;
+  background-color: #eeeeee }
+
+span.classifier {
+  font-family: sans-serif ;
+  font-style: oblique }
+
+span.classifier-delimiter {
+  font-family: sans-serif ;
+  font-weight: bold }
+
+span.interpreted {
+  font-family: sans-serif }
+
+span.option {
+  white-space: nowrap }
+
+span.pre {
+  white-space: pre }
+
+span.problematic {
+  color: red }
+
+span.section-subtitle {
+  /* font-size relative to parent (h1..h6 element) */
+  font-size: 80% }
+
+table.citation {
+  border-left: solid 1px gray;
+  margin-left: 1px }
+
+table.docinfo {
+  margin: 2em 4em }
+
+table.docutils {
+  margin-top: 0.5em ;
+  margin-bottom: 0.5em }
+
+table.footnote {
+  border-left: solid 1px black;
+  margin-left: 1px }
+
+table.docutils td, table.docutils th,
+table.docinfo td, table.docinfo th {
+  padding-left: 0.5em ;
+  padding-right: 0.5em ;
+  vertical-align: top }
+
+table.docutils th.field-name, table.docinfo th.docinfo-name {
+  font-weight: bold ;
+  text-align: left ;
+  white-space: nowrap ;
+  padding-left: 0 }
+
+h1 tt.docutils, h2 tt.docutils, h3 tt.docutils,
+h4 tt.docutils, h5 tt.docutils, h6 tt.docutils {
+  font-size: 100% }
+
+tt.docutils {
+  background-color: #eeeeee }
+
+ul.auto-toc {
+  list-style-type: none }
+
+</style>
+</head>
+<body>
+<div class="document" id="leipzig-pypy-sprint-june-22-27-2012">
+<h1 class="title">Leipzig PyPy sprint June 22 - 27, 2012</h1>
+<p>The next PyPy sprint will be held --- for the first time in a while ---
+in a place where we haven't been so far: Leipzig, Germany, at the
+<a class="reference" href="http://www.python-academy.com/">Python Academy</a>'s Teaching Center, from the 22nd to the 27th of June
+2012, before EuroPython.  Thanks to Mike M&#252;ller for organizing it!</p>
+<p>This is a fully public sprint, everyone is welcome to join us.  All days are
+full sprint days, so it is recommended to arrive the 21st and leave the 28th.</p>
+<div class="section">
+<h1><a id="topics-and-goals" name="topics-and-goals">Topics and goals</a></h1>
+<p>Open.  Here are some goals:</p>
+<ul class="simple">
+<li>numpy: progress towards completing the <tt class="docutils literal"><span class="pre">numpypy</span></tt> module; try to
+use it in real code</li>
+<li>stm: progress on Transactional Memory; try out the <tt class="docutils literal"><span class="pre">transaction</span></tt>
+module on real code.</li>
+<li>any other PyPy-related topic is fine too.</li>
+</ul>
+</div>
+<div class="section">
+<h1><a id="location" name="location">Location</a></h1>
+<p>Python Academy Leipzig, Germany.
+<a class="reference" href="http://www.python-academy.com/center/find.html">http://www.python-academy.com/center/find.html</a>
+Thanks to Mike M&#252;ller for inviting us.</p>
+<p>The room holds 8 - 12 people depending how much space each individual
+needs.  Drinks, snacks and a simple lunch, e.g. pizza or other fast food
+is provided.  Dinner would be extra.</p>
+<p>Some accommodations:
+<a class="reference" href="http://www.python-academy.com/center/accommodation.html">http://www.python-academy.com/center/accommodation.html</a>
+Pretty much all hotels and hostels in the city center
+are fine too. The tram needs only 16 minutes from the
+central station to the Academy location.</p>
+</div>
+<div class="section">
+<h1><a id="grants" name="grants">Grants</a></h1>
+<p>For students, we have the possibility to support some costs via PyPy
+funds.  Additionally, we can support you applying for grants from the
+PSF and other sources.</p>
+</div>
+<div class="section">
+<h1><a id="registration" name="registration">Registration</a></h1>
+<p>If you'd like to come, please <em>sign up</em> either by announcing yourself on
+<a class="reference" href="http://mail.python.org/mailman/listinfo/pypy-dev">pypy-dev</a>, or by directly adding yourself to the <a class="reference" href="https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leipzig2012/people.txt">list of people</a>.
+(We need to have a head count for the organization.)  If you are new to
+the project please drop a note about your interests and post any
+questions.</p>
+</div>
+</div>
+</body>
+</html>

From noreply at buildbot.pypy.org  Mon Apr  2 15:44:38 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 15:44:38 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Remove the html version;
 we can't seem to link to it anyway.
Message-ID: <20120402134438.3027346E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4174:13d4f71f9018
Date: 2012-04-02 15:43 +0200
http://bitbucket.org/pypy/extradoc/changeset/13d4f71f9018/

Log:	Remove the html version; we can't seem to link to it anyway.

diff --git a/sprintinfo/leipzig2012/announce.html b/sprintinfo/leipzig2012/announce.html
deleted file mode 100644
--- a/sprintinfo/leipzig2012/announce.html
+++ /dev/null
@@ -1,342 +0,0 @@
-<?xml version="1.0" encoding="utf-8" ?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-<meta name="generator" content="Docutils 0.4: http://docutils.sourceforge.net/" />
-<title>Leipzig PyPy sprint June 22 - 27, 2012</title>
-<style type="text/css">
-
-/*
-:Author: David Goodger
-:Contact: goodger at users.sourceforge.net
-:Date: $Date: 2005-12-18 01:56:14 +0100 (Sun, 18 Dec 2005) $
-:Revision: $Revision: 4224 $
-:Copyright: This stylesheet has been placed in the public domain.
-
-Default cascading style sheet for the HTML output of Docutils.
-
-See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to
-customize this style sheet.
-*/
-
-/* used to remove borders from tables and images */
-.borderless, table.borderless td, table.borderless th {
-  border: 0 }
-
-table.borderless td, table.borderless th {
-  /* Override padding for "table.docutils td" with "! important".
-     The right padding separates the table cells. */
-  padding: 0 0.5em 0 0 ! important }
-
-.first {
-  /* Override more specific margin styles with "! important". */
-  margin-top: 0 ! important }
-
-.last, .with-subtitle {
-  margin-bottom: 0 ! important }
-
-.hidden {
-  display: none }
-
-a.toc-backref {
-  text-decoration: none ;
-  color: black }
-
-blockquote.epigraph {
-  margin: 2em 5em ; }
-
-dl.docutils dd {
-  margin-bottom: 0.5em }
-
-/* Uncomment (and remove this text!) to get bold-faced definition list terms
-dl.docutils dt {
-  font-weight: bold }
-*/
-
-div.abstract {
-  margin: 2em 5em }
-
-div.abstract p.topic-title {
-  font-weight: bold ;
-  text-align: center }
-
-div.admonition, div.attention, div.caution, div.danger, div.error,
-div.hint, div.important, div.note, div.tip, div.warning {
-  margin: 2em ;
-  border: medium outset ;
-  padding: 1em }
-
-div.admonition p.admonition-title, div.hint p.admonition-title,
-div.important p.admonition-title, div.note p.admonition-title,
-div.tip p.admonition-title {
-  font-weight: bold ;
-  font-family: sans-serif }
-
-div.attention p.admonition-title, div.caution p.admonition-title,
-div.danger p.admonition-title, div.error p.admonition-title,
-div.warning p.admonition-title {
-  color: red ;
-  font-weight: bold ;
-  font-family: sans-serif }
-
-/* Uncomment (and remove this text!) to get reduced vertical space in
-   compound paragraphs.
-div.compound .compound-first, div.compound .compound-middle {
-  margin-bottom: 0.5em }
-
-div.compound .compound-last, div.compound .compound-middle {
-  margin-top: 0.5em }
-*/
-
-div.dedication {
-  margin: 2em 5em ;
-  text-align: center ;
-  font-style: italic }
-
-div.dedication p.topic-title {
-  font-weight: bold ;
-  font-style: normal }
-
-div.figure {
-  margin-left: 2em ;
-  margin-right: 2em }
-
-div.footer, div.header {
-  clear: both;
-  font-size: smaller }
-
-div.line-block {
-  display: block ;
-  margin-top: 1em ;
-  margin-bottom: 1em }
-
-div.line-block div.line-block {
-  margin-top: 0 ;
-  margin-bottom: 0 ;
-  margin-left: 1.5em }
-
-div.sidebar {
-  margin-left: 1em ;
-  border: medium outset ;
-  padding: 1em ;
-  background-color: #ffffee ;
-  width: 40% ;
-  float: right ;
-  clear: right }
-
-div.sidebar p.rubric {
-  font-family: sans-serif ;
-  font-size: medium }
-
-div.system-messages {
-  margin: 5em }
-
-div.system-messages h1 {
-  color: red }
-
-div.system-message {
-  border: medium outset ;
-  padding: 1em }
-
-div.system-message p.system-message-title {
-  color: red ;
-  font-weight: bold }
-
-div.topic {
-  margin: 2em }
-
-h1.section-subtitle, h2.section-subtitle, h3.section-subtitle,
-h4.section-subtitle, h5.section-subtitle, h6.section-subtitle {
-  margin-top: 0.4em }
-
-h1.title {
-  text-align: center }
-
-h2.subtitle {
-  text-align: center }
-
-hr.docutils {
-  width: 75% }
-
-img.align-left {
-  clear: left }
-
-img.align-right {
-  clear: right }
-
-ol.simple, ul.simple {
-  margin-bottom: 1em }
-
-ol.arabic {
-  list-style: decimal }
-
-ol.loweralpha {
-  list-style: lower-alpha }
-
-ol.upperalpha {
-  list-style: upper-alpha }
-
-ol.lowerroman {
-  list-style: lower-roman }
-
-ol.upperroman {
-  list-style: upper-roman }
-
-p.attribution {
-  text-align: right ;
-  margin-left: 50% }
-
-p.caption {
-  font-style: italic }
-
-p.credits {
-  font-style: italic ;
-  font-size: smaller }
-
-p.label {
-  white-space: nowrap }
-
-p.rubric {
-  font-weight: bold ;
-  font-size: larger ;
-  color: maroon ;
-  text-align: center }
-
-p.sidebar-title {
-  font-family: sans-serif ;
-  font-weight: bold ;
-  font-size: larger }
-
-p.sidebar-subtitle {
-  font-family: sans-serif ;
-  font-weight: bold }
-
-p.topic-title {
-  font-weight: bold }
-
-pre.address {
-  margin-bottom: 0 ;
-  margin-top: 0 ;
-  font-family: serif ;
-  font-size: 100% }
-
-pre.literal-block, pre.doctest-block {
-  margin-left: 2em ;
-  margin-right: 2em ;
-  background-color: #eeeeee }
-
-span.classifier {
-  font-family: sans-serif ;
-  font-style: oblique }
-
-span.classifier-delimiter {
-  font-family: sans-serif ;
-  font-weight: bold }
-
-span.interpreted {
-  font-family: sans-serif }
-
-span.option {
-  white-space: nowrap }
-
-span.pre {
-  white-space: pre }
-
-span.problematic {
-  color: red }
-
-span.section-subtitle {
-  /* font-size relative to parent (h1..h6 element) */
-  font-size: 80% }
-
-table.citation {
-  border-left: solid 1px gray;
-  margin-left: 1px }
-
-table.docinfo {
-  margin: 2em 4em }
-
-table.docutils {
-  margin-top: 0.5em ;
-  margin-bottom: 0.5em }
-
-table.footnote {
-  border-left: solid 1px black;
-  margin-left: 1px }
-
-table.docutils td, table.docutils th,
-table.docinfo td, table.docinfo th {
-  padding-left: 0.5em ;
-  padding-right: 0.5em ;
-  vertical-align: top }
-
-table.docutils th.field-name, table.docinfo th.docinfo-name {
-  font-weight: bold ;
-  text-align: left ;
-  white-space: nowrap ;
-  padding-left: 0 }
-
-h1 tt.docutils, h2 tt.docutils, h3 tt.docutils,
-h4 tt.docutils, h5 tt.docutils, h6 tt.docutils {
-  font-size: 100% }
-
-tt.docutils {
-  background-color: #eeeeee }
-
-ul.auto-toc {
-  list-style-type: none }
-
-</style>
-</head>
-<body>
-<div class="document" id="leipzig-pypy-sprint-june-22-27-2012">
-<h1 class="title">Leipzig PyPy sprint June 22 - 27, 2012</h1>
-<p>The next PyPy sprint will be held --- for the first time in a while ---
-in a place where we haven't been so far: Leipzig, Germany, at the
-<a class="reference" href="http://www.python-academy.com/">Python Academy</a>'s Teaching Center, from the 22nd to the 27th of June
-2012, before EuroPython.  Thanks to Mike M&#252;ller for organizing it!</p>
-<p>This is a fully public sprint, everyone is welcome to join us.  All days are
-full sprint days, so it is recommended to arrive the 21st and leave the 28th.</p>
-<div class="section">
-<h1><a id="topics-and-goals" name="topics-and-goals">Topics and goals</a></h1>
-<p>Open.  Here are some goals:</p>
-<ul class="simple">
-<li>numpy: progress towards completing the <tt class="docutils literal"><span class="pre">numpypy</span></tt> module; try to
-use it in real code</li>
-<li>stm: progress on Transactional Memory; try out the <tt class="docutils literal"><span class="pre">transaction</span></tt>
-module on real code.</li>
-<li>any other PyPy-related topic is fine too.</li>
-</ul>
-</div>
-<div class="section">
-<h1><a id="location" name="location">Location</a></h1>
-<p>Python Academy Leipzig, Germany.
-<a class="reference" href="http://www.python-academy.com/center/find.html">http://www.python-academy.com/center/find.html</a>
-Thanks to Mike M&#252;ller for inviting us.</p>
-<p>The room holds 8 - 12 people depending how much space each individual
-needs.  Drinks, snacks and a simple lunch, e.g. pizza or other fast food
-is provided.  Dinner would be extra.</p>
-<p>Some accommodations:
-<a class="reference" href="http://www.python-academy.com/center/accommodation.html">http://www.python-academy.com/center/accommodation.html</a>
-Pretty much all hotels and hostels in the city center
-are fine too. The tram needs only 16 minutes from the
-central station to the Academy location.</p>
-</div>
-<div class="section">
-<h1><a id="grants" name="grants">Grants</a></h1>
-<p>For students, we have the possibility to support some costs via PyPy
-funds.  Additionally, we can support you applying for grants from the
-PSF and other sources.</p>
-</div>
-<div class="section">
-<h1><a id="registration" name="registration">Registration</a></h1>
-<p>If you'd like to come, please <em>sign up</em> either by announcing yourself on
-<a class="reference" href="http://mail.python.org/mailman/listinfo/pypy-dev">pypy-dev</a>, or by directly adding yourself to the <a class="reference" href="https://bitbucket.org/pypy/extradoc/raw/extradoc/sprintinfo/leipzig2012/people.txt">list of people</a>.
-(We need to have a head count for the organization.)  If you are new to
-the project please drop a note about your interests and post any
-questions.</p>
-</div>
-</div>
-</body>
-</html>

From noreply at buildbot.pypy.org  Mon Apr  2 16:10:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 16:10:00 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add a minimal usage example.
Message-ID: <20120402141000.31DD846E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54151:ae91dbd26901
Date: 2012-04-02 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/ae91dbd26901/

Log:	Add a minimal usage example.

diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -1,3 +1,20 @@
+"""
+Minimal example of usage:
+
+    for i in range(10):
+        transaction.add(do_stuff, i)
+    transaction.run()
+
+This schedules and runs all ten do_stuff(i), each in its own transaction.
+Each one can also add more transactions to run afterwards, and so on.
+The call to run() returns when all transactions have completed.
+
+From the API point of view it is as if the do_stuff(i) were run serially
+in some random order.  If you use a real implementation instead of this
+one (which is here for trying things out), then the transactions can
+actually run in parallel on multiple cores.
+"""
+
 import sys
 import random
 

From noreply at buildbot.pypy.org  Mon Apr  2 16:11:20 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 16:11:20 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: Update with a link to the
 documentation -- so far only in transaction.py.
Message-ID: <20120402141120.0BFFE46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r351:0bffaf37ab63
Date: 2012-04-02 16:11 +0200
http://bitbucket.org/pypy/pypy.org/changeset/0bffaf37ab63/

Log:	Update with a link to the documentation -- so far only in
	transaction.py.

diff --git a/source/tmdonate.txt b/source/tmdonate.txt
--- a/source/tmdonate.txt
+++ b/source/tmdonate.txt
@@ -189,6 +189,10 @@
 detail once they are known).  But the point is that your program is
 always correct.
 
+See some `concrete example`__ of the API.
+
+.. __: https://bitbucket.org/pypy/pypy/raw/stm-gc/lib_pypy/transaction.py
+
 
 Speed
 -----
diff --git a/tmdonate.html b/tmdonate.html
--- a/tmdonate.html
+++ b/tmdonate.html
@@ -183,6 +183,7 @@
 transactions into several parts; the exact rules will be published in
 detail once they are known).  But the point is that your program is
 always correct.</p>
+<p>See some <a class="reference external" href="https://bitbucket.org/pypy/pypy/raw/stm-gc/lib_pypy/transaction.py">concrete example</a> of the API.</p>
 </div>
 <div class="section" id="speed">
 <h2>Speed</h2>

From noreply at buildbot.pypy.org  Mon Apr  2 17:21:51 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 17:21:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix: apply this change,
 even if it's incompatible with CPython 2.2(!).
Message-ID: <20120402152151.F26C246E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54152:49d683bbd92b
Date: 2012-04-02 17:17 +0200
http://bitbucket.org/pypy/pypy/changeset/49d683bbd92b/

Log:	Fix: apply this change, even if it's incompatible with CPython
	2.2(!). It fixes an issue reported by adoven on irc. Add a test.

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -968,8 +968,7 @@
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
+        return NotImplemented    # note that this doesn't work on CPython 2.2
 
     __radd__ = __add__
 
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -44,3 +44,9 @@
     assert type(dt.microsecond) is int
 
     copy.copy(dt)
+
+def test_radd():
+    class X(object):
+        def __radd__(self, other):
+            return "radd"
+    assert datetime.date(10, 10, 10) + X() == "radd"

From noreply at buildbot.pypy.org  Mon Apr  2 17:32:59 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 17:32:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for issue1100.
Message-ID: <20120402153259.F0D0846E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54153:ae4a7ab99a76
Date: 2012-04-02 17:32 +0200
http://bitbucket.org/pypy/pypy/changeset/ae4a7ab99a76/

Log:	Fix for issue1100.

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -247,7 +247,7 @@
         return 0
 
     def iter(self, w_dict):
-        return EmptyIteratorImplementation(self.space, w_dict)
+        return EmptyIteratorImplementation(self.space, self, w_dict)
 
     def clear(self, w_dict):
         return
@@ -263,8 +263,9 @@
 # Iterator Implementation base classes
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.dictimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -272,7 +273,8 @@
     def next(self):
         if self.dictimplementation is None:
             return None, None
-        if self.len != self.dictimplementation.length():
+        if (self.len != self.dictimplementation.length()
+              or self.strategy is not self.dictimplementation.strategy):
             self.len = -1   # Make this error state sticky
             raise OperationError(self.space.w_RuntimeError,
                      self.space.wrap("dictionary changed size during iteration"))
@@ -489,7 +491,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -503,7 +505,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -804,6 +804,21 @@
         assert "IntDictStrategy" in self.get_strategy(d)
         assert d[1L] == "hi"
 
+    def test_iter_dict_length_change(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d[7] = 8
+        # 'd' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_dict_strategy_only_change(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d['foo'] = 'bar'
+        del d[1]
+        # 'd' is still length 3, but its strategy changed
+        raises(RuntimeError, it.next)
+
 
 class FakeString(str):
     hash_count = 0

From noreply at buildbot.pypy.org  Mon Apr  2 18:15:13 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 18:15:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: Another case: iterating should work if
 the dict strategy changed "just"
Message-ID: <20120402161513.5403146E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54154:7e37a82840df
Date: 2012-04-02 18:14 +0200
http://bitbucket.org/pypy/pypy/changeset/7e37a82840df/

Log:	Another case: iterating should work if the dict strategy changed
	"just" because of a __getitem__. It's workaroundesque but enough, I
	hope.

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -273,8 +273,7 @@
     def next(self):
         if self.dictimplementation is None:
             return None, None
-        if (self.len != self.dictimplementation.length()
-              or self.strategy is not self.dictimplementation.strategy):
+        if self.len != self.dictimplementation.length():
             self.len = -1   # Make this error state sticky
             raise OperationError(self.space.w_RuntimeError,
                      self.space.wrap("dictionary changed size during iteration"))
@@ -282,7 +281,20 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.dictimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the dict.  The (key, value) pair in 'result'
+                # might be out-of-date.  We try to explicitly look up
+                # the key in the dict.
+                w_key = result[0]
+                w_value = self.dictimplementation.getitem(w_key)
+                if w_value is None:
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return (w_key, w_value)
         # no more entries
         self.dictimplementation = None
         return None, None
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -811,7 +811,17 @@
         # 'd' is now length 4
         raises(RuntimeError, it.next)
 
-    def test_iter_dict_strategy_only_change(self):
+    def test_iter_dict_strategy_only_change_1(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        d.get(Foo())    # this changes the strategy of 'd'
+        lst = list(it)  # but iterating still works
+        assert sorted(lst) == [(1, 2), (3, 4), (5, 6)]
+
+    def test_iter_dict_strategy_only_change_2(self):
         d = {1: 2, 3: 4, 5: 6}
         it = d.iteritems()
         d['foo'] = 'bar'

From pullrequests-noreply at bitbucket.org  Mon Apr  2 18:22:13 2012
From: pullrequests-noreply at bitbucket.org (unbit)
Date: Mon, 02 Apr 2012 16:22:13 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize
	(pull request #67)
Message-ID: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>

A new pull request has been opened by unbit.

unbit/pypy has changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize

Title: added implementation of Py_Initialize

Py_Initialize() is the function called by apps wanting to embed a python intepreter. The commit adds Py_Initialize implementation, its definition in pythonrun header file and exports the symbol in the translator scripts.

Changes to be pulled:

4c1e64afbe9a by rob... at goyle: "added implementation of Py_Initialize"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Mon Apr  2 18:22:25 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 18:22:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: Same issue and same fix for sets.
Message-ID: <20120402162225.6152046E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54155:f16be5ae31e6
Date: 2012-04-02 18:22 +0200
http://bitbucket.org/pypy/pypy/changeset/f16be5ae31e6/

Log:	Same issue and same fix for sets.

diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -359,7 +359,7 @@
         w_set.sstorage = w_other.get_storage_copy()
 
     def iter(self, w_set):
-        return EmptyIteratorImplementation(self.space, w_set)
+        return EmptyIteratorImplementation(self.space, self, w_set)
 
     def popitem(self, w_set):
         raise OperationError(self.space.w_KeyError,
@@ -784,8 +784,9 @@
             d_obj[w_item] = None
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.setimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -801,7 +802,17 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.setimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the set.  The 'result' might be out-of-date.
+                # We try to explicitly look it up in the set.
+                if not self.setimplementation.has_key(result):
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return result
         # no more entries
         self.setimplementation = None
         return None
@@ -823,7 +834,7 @@
 
 class StringIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, w_set):
-        IteratorImplementation.__init__(self, space, w_set)
+        IteratorImplementation.__init__(self, space, strategy, w_set)
         d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
@@ -835,9 +846,9 @@
 
 class IntegerIteratorImplementation(IteratorImplementation):
     #XXX same implementation in dictmultiobject on dictstrategy-branch
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
@@ -848,9 +859,9 @@
             return None
 
 class RDictIteratorImplementation(IteratorImplementation):
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -817,7 +817,7 @@
         class Foo(object):
             def __eq__(self, other):
                 return False
-        d.get(Foo())    # this changes the strategy of 'd'
+        assert d.get(Foo()) is None    # this changes the strategy of 'd'
         lst = list(it)  # but iterating still works
         assert sorted(lst) == [(1, 2), (3, 4), (5, 6)]
 
@@ -826,8 +826,10 @@
         it = d.iteritems()
         d['foo'] = 'bar'
         del d[1]
-        # 'd' is still length 3, but its strategy changed
-        raises(RuntimeError, it.next)
+        # 'd' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us (1, 2), but 1 is not in the dict any longer.
+        raises(RuntimeError, list, it)
 
 
 class FakeString(str):
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -907,3 +907,30 @@
                 return [5, 3, 4][i]
         s = set([10,3,2]).intersection(Obj())
         assert list(s) == [3]
+
+    def test_iter_set_length_change(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add(7)
+        # 's' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_set_strategy_only_change_1(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert Foo() not in s      # this changes the strategy of 'd'
+        lst = list(s)  # but iterating still works
+        assert sorted(lst) == [1, 3, 5]
+
+    def test_iter_set_strategy_only_change_2(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add('foo')
+        s.remove(1)
+        # 's' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us 1, but 1 is not in the set any longer.
+        raises(RuntimeError, list, it)

From noreply at buildbot.pypy.org  Mon Apr  2 19:59:15 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  2 Apr 2012 19:59:15 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add a point
Message-ID: <20120402175915.322BA46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4175:1c44b080a0e5
Date: 2012-04-02 19:58 +0200
http://bitbucket.org/pypy/extradoc/changeset/1c44b080a0e5/

Log:	Add a point

diff --git a/sprintinfo/leipzig2012/announce.txt b/sprintinfo/leipzig2012/announce.txt
--- a/sprintinfo/leipzig2012/announce.txt
+++ b/sprintinfo/leipzig2012/announce.txt
@@ -24,6 +24,9 @@
 - stm: progress on Transactional Memory; try out the ``transaction``
   module on real code.
 
+- jit optimizations: there are a number of optimizations we can still
+  try out or refactor.
+
 - any other PyPy-related topic is fine too.
 
 

From noreply at buildbot.pypy.org  Mon Apr  2 20:04:08 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  2 Apr 2012 20:04:08 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: add a point
Message-ID: <20120402180408.831A446E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r4176:1bdfa84745f2
Date: 2012-04-02 20:03 +0200
http://bitbucket.org/pypy/extradoc/changeset/1bdfa84745f2/

Log:	add a point

diff --git a/sprintinfo/leipzig2012/announce.txt b/sprintinfo/leipzig2012/announce.txt
--- a/sprintinfo/leipzig2012/announce.txt
+++ b/sprintinfo/leipzig2012/announce.txt
@@ -27,6 +27,10 @@
 - jit optimizations: there are a number of optimizations we can still
   try out or refactor.
 
+- work on various, more efficient data structures for Python language.
+  A good example would be lazy string slicing/concatenation or more efficient
+  objects.
+
 - any other PyPy-related topic is fine too.
 
 
diff --git a/talk/uct2012/talk.rst b/talk/uct2012/talk.rst
--- a/talk/uct2012/talk.rst
+++ b/talk/uct2012/talk.rst
@@ -6,7 +6,7 @@
 
 * PyPy developer since 2006
 
-XXX
+* Worked on many areas, including the GC, the JIT, etc.
 
 What is PyPy?
 -------------

From noreply at buildbot.pypy.org  Mon Apr  2 21:08:53 2012
From: noreply at buildbot.pypy.org (ctismer)
Date: Mon,  2 Apr 2012 21:08:53 +0200 (CEST)
Subject: [pypy-commit] pypy win64-stage1: Merge from default
Message-ID: <20120402190853.3009346E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Christian Tismer  <tismer at stackless.com>
Branch: win64-stage1
Changeset: r54156:c624ccedd559
Date: 2012-04-02 21:08 +0200
http://bitbucket.org/pypy/pypy/changeset/c624ccedd559/

Log:	Merge from default

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -968,8 +968,7 @@
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
+        return NotImplemented    # note that this doesn't work on CPython 2.2
 
     __radd__ = __add__
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -167,6 +167,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1797,6 +1797,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1813,6 +1814,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -432,7 +432,8 @@
                     raise _ssl_seterror(self.space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return self.space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return self.space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                              length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -11,7 +11,7 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.objectmodel import specialize
+from pypy.rlib.objectmodel import specialize, keepalive_until_here
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
@@ -145,18 +145,24 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(RWBuffer):
-    def __init__(self, data, bytes):
-        self.data = data
-        self.len = bytes
+    def __init__(self, array):
+        self.array = array
 
     def getlength(self):
-        return self.len
+        return self.array.len * self.array.itemsize
 
     def getitem(self, index):
-        return self.data[index]
+        array = self.array
+        data = array._charbuf_start()
+        char = data[index]
+        array._charbuf_stop()
+        return char
 
     def setitem(self, index, char):
-        self.data[index] = char
+        array = self.array
+        data = array._charbuf_start()
+        data[index] = char
+        array._charbuf_stop()
 
 
 def make_array(mytype):
@@ -278,9 +284,10 @@
             oldlen = self.len
             new = len(s) / mytype.bytes
             self.setlen(oldlen + new)
-            cbuf = self.charbuf()
+            cbuf = self._charbuf_start()
             for i in range(len(s)):
                 cbuf[oldlen * mytype.bytes + i] = s[i]
+            self._charbuf_stop()
 
         def fromlist(self, w_lst):
             s = self.len
@@ -310,8 +317,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def charbuf(self):
-            return  rffi.cast(rffi.CCHARP, self.buffer)
+        def _charbuf_start(self):
+            return rffi.cast(rffi.CCHARP, self.buffer)
+
+        def _charbuf_stop(self):
+            keepalive_until_here(self)
 
         def w_getitem(self, space, idx):
             item = self.buffer[idx]
@@ -530,8 +540,10 @@
         self.fromstring(space.str_w(w_s))
 
     def array_tostring__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
+        self._charbuf_stop()
+        return self.space.wrap(s)
 
     def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
         if not isinstance(w_f, W_File):
@@ -613,8 +625,7 @@
     # Misc methods
 
     def buffer__Array(space, self):
-        b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes)
-        return space.wrap(b)
+        return space.wrap(ArrayBuffer(self))
 
     def array_buffer_info__Array(space, self):
         w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer))
@@ -649,7 +660,7 @@
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
-        bytes = self.charbuf()
+        bytes = self._charbuf_start()
         tmp = [bytes[0]] * mytype.bytes
         for start in range(0, self.len * mytype.bytes, mytype.bytes):
             stop = start + mytype.bytes - 1
@@ -657,6 +668,7 @@
                 tmp[i] = bytes[start + i]
             for i in range(mytype.bytes):
                 bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
 
     def repr__Array(space, self):
         if self.len == 0:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -433,7 +433,25 @@
         a = self.array('h', 'Hi')
         buf = buffer(a)
         assert buf[1] == 'i'
-        #raises(TypeError, buf.__setitem__, 1, 'o')
+
+    def test_buffer_write(self):
+        a = self.array('c', 'hello')
+        buf = buffer(a)
+        print repr(buf)
+        try:
+            buf[3] = 'L'
+        except TypeError:
+            skip("buffer(array) returns a read-only buffer on CPython")
+        assert a.tostring() == 'helLo'
+
+    def test_buffer_keepalive(self):
+        buf = buffer(self.array('c', 'text'))
+        assert buf[2] == 'x'
+        #
+        a = self.array('c', 'foobarbaz')
+        buf = buffer(a)
+        a.fromstring('some extra text')
+        assert buf[:] == 'foobarbazsome extra text'
 
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,7 +2,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
     PyObjectFields, PyObject)
-from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
+from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
 from pypy.interpreter.error import OperationError
 from pypy.module.array.interp_array import ArrayBuffer
@@ -43,13 +43,18 @@
         py_buf.c_b_offset = w_obj.offset
         w_obj = w_obj.buffer
 
+    # If w_obj already allocated a fixed buffer, use it, and keep a
+    # reference to w_obj.
+    # Otherwise, b_base stays NULL, and we own the b_ptr.
+
     if isinstance(w_obj, StringBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
+        py_buf.c_b_base = lltype.nullptr(PyObject.TO)
         py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
         py_buf.c_b_size = w_obj.getlength()
     elif isinstance(w_obj, ArrayBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.data)
+        w_base = w_obj.array
+        py_buf.c_b_base = make_ref(space, w_base)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start())
         py_buf.c_b_size = w_obj.getlength()
     else:
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -60,14 +65,16 @@
     """
     Creates the buffer in the PyPy interpreter from a cpyext representation.
     """
-    raise Exception("realize fail fail fail")
-
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "Don't know how to realize a buffer"))
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
-    Py_DecRef(space, py_buf.c_b_base)
-    rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
+    if py_buf.c_b_base:
+        Py_DecRef(space, py_buf.c_b_base)
+    else:
+        rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -315,3 +316,65 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,14 +167,16 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -183,7 +185,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
@@ -414,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -425,6 +429,36 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
+
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -307,6 +307,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -624,7 +626,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -673,6 +673,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -1261,6 +1265,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1386,6 +1415,7 @@
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -107,6 +107,10 @@
         arr.compute_first_step(self, f)
         return f
 
+    def debug_repr(self):
+        # should be overridden, but in case it isn't, provide a default
+        return str(self)
+
 class ConcreteSignature(Signature):
     _immutable_fields_ = ['dtype']
 
@@ -207,7 +211,7 @@
     def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        transforms = transforms + [ViewTransform(arr.chunks)]
+        transforms = [ViewTransform(arr.chunks)] + transforms 
         self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
@@ -215,6 +219,9 @@
         assert isinstance(arr, VirtualSlice)
         return self.child.eval(frame, arr.child)
 
+    def debug_repr(self):
+        return 'VirtualSlice(%s)' % self.child.debug_repr()
+
 class Call1(Signature):
     _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
 
@@ -270,7 +277,7 @@
         from pypy.module.micronumpy.interp_numarray import Call1
 
         assert isinstance(arr, Call1)
-        vtransforms = transforms + [BroadcastTransform(arr.values.shape)]
+        vtransforms = [BroadcastTransform(arr.values.shape)] + transforms
         self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
         self.res._create_iter(iterlist, arraylist, arr.res, transforms)
 
@@ -348,7 +355,7 @@
         from pypy.module.micronumpy.interp_numarray import ResultArray
 
         assert isinstance(arr, ResultArray)
-        rtransforms = transforms + [BroadcastTransform(arr.left.shape)]
+        rtransforms = [BroadcastTransform(arr.left.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -375,7 +382,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
@@ -388,7 +395,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -401,8 +408,8 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -424,7 +431,7 @@
         frame.cur_value = self.binfunc(self.calc_dtype, frame.cur_value, rval)
 
     def debug_repr(self):
-        return 'ReduceSig(%s)' % (self.name, self.right.debug_repr())
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
 
 class SliceloopSignature(Call2):
     def eval(self, frame, arr):
@@ -448,7 +455,7 @@
         from pypy.module.micronumpy.interp_numarray import SliceArray
 
         assert isinstance(arr, SliceArray)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,9 +1,9 @@
 from pypy.rlib import jit
 
 
- at jit.look_inside_iff(lambda s: jit.isconstant(len(s)))
+ at jit.unroll_safe
 def product(s):
     i = 1
     for x in s:
         i *= x
-    return i
\ No newline at end of file
+    return i
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -395,11 +395,19 @@
         assert a[3] == 0.
 
     def test_newaxis(self):
-        from _numpypy import array
+        import math
+        from _numpypy import array, cos, zeros
         from numpypy.core.numeric import newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
+        a = array(range(3))
+        b = array([1, 3])
+        expected = zeros((3, 2))
+        for x in range(3):
+            for y in range(2):
+                expected[x, y] = math.cos(a[x]) * math.cos(b[y])
+        assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):
         from _numpypy import array
@@ -1338,6 +1346,10 @@
         dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0)
         assert str(dims_disagree.value) == \
             "array dimensions must agree except for axis being concatenated"
+        a = array([1, 2, 3, 4, 5, 6])
+        a = (a + a)[::2]
+        b = concatenate((a[:3], a[-3:]))
+        assert (b == [2, 6, 10, 2, 6, 10]).all()
 
     def test_std(self):
         from _numpypy import array
@@ -1387,6 +1399,16 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat, array
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -212,7 +212,7 @@
             i19 = int_add(i12, 1)
             setfield_gc(p9, i19, descr=<FieldS .*W_AbstractSeqIterObject.inst_index .*>)
             guard_nonnull_class(p17, 146982464, descr=...)
-            i21 = getfield_gc(p17, descr=<FieldS .*W_ArrayTypei.inst_len .*>)
+            i21 = getfield_gc(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
             i24 = getfield_gc(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -198,3 +198,37 @@
             i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>)
             guard_value(i49, 1, descr=...)
             ''')
+
+    def test_remove_duplicate_method_calls(self):
+        def main(n):
+            lst = []
+            for i in range(n):
+                s = 'Hello %d' % i
+                t = s.lower()   # ID: callone
+                u = s.lower()   # ID: calltwo
+                lst.append(t)
+                lst.append(u)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '''
+            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr . r EF=3>)
+            guard_no_exception(descr=...)
+            ''')
+        loop.match_by_id('calltwo', '')    # nothing
+
+    def test_move_method_call_out_of_loop(self):
+        def main(n):
+            lst = []
+            s = 'Hello %d' % n
+            for i in range(n):
+                t = s.lower()   # ID: callone
+                lst.append(t)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '')    # nothing
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -600,30 +600,33 @@
 class AppTestSysExcInfoDirect:
 
     def setup_method(self, meth):
-        self.seen = []
-        from pypy.module.sys import vm
-        def exc_info_with_tb(*args):
-            self.seen.append("n")     # not optimized
-            return self.old[0](*args)
-        def exc_info_without_tb(*args):
-            self.seen.append("y")     # optimized
-            return self.old[1](*args)
-        self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
-        vm.exc_info_with_tb = exc_info_with_tb
-        vm.exc_info_without_tb = exc_info_without_tb
-        #
-        from pypy.rlib import jit
-        self.old2 = [jit.we_are_jitted]
-        jit.we_are_jitted = lambda: True
+        self.checking = not option.runappdirect
+        if self.checking:
+            self.seen = []
+            from pypy.module.sys import vm
+            def exc_info_with_tb(*args):
+                self.seen.append("n")     # not optimized
+                return self.old[0](*args)
+            def exc_info_without_tb(*args):
+                self.seen.append("y")     # optimized
+                return self.old[1](*args)
+            self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
+            vm.exc_info_with_tb = exc_info_with_tb
+            vm.exc_info_without_tb = exc_info_without_tb
+            #
+            from pypy.rlib import jit
+            self.old2 = [jit.we_are_jitted]
+            jit.we_are_jitted = lambda: True
 
     def teardown_method(self, meth):
-        from pypy.module.sys import vm
-        from pypy.rlib import jit
-        vm.exc_info_with_tb = self.old[0]
-        vm.exc_info_without_tb = self.old[1]
-        jit.we_are_jitted = self.old2[0]
-        #
-        assert ''.join(self.seen) == meth.expected
+        if self.checking:
+            from pypy.module.sys import vm
+            from pypy.rlib import jit
+            vm.exc_info_with_tb = self.old[0]
+            vm.exc_info_without_tb = self.old[1]
+            jit.we_are_jitted = self.old2[0]
+            #
+            assert ''.join(self.seen) == meth.expected
 
     def test_returns_none(self):
         import sys
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -44,3 +44,9 @@
     assert type(dt.microsecond) is int
 
     copy.copy(dt)
+
+def test_radd():
+    class X(object):
+        def __radd__(self, other):
+            return "radd"
+    assert datetime.date(10, 10, 10) + X() == "radd"
diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py
--- a/pypy/module/thread/test/test_ll_thread.py
+++ b/pypy/module/thread/test/test_ll_thread.py
@@ -66,7 +66,6 @@
     def test_gc_locking(self):
         import time
         from pypy.rlib.objectmodel import invoke_around_extcall
-        from pypy.rlib.objectmodel import we_are_translated
         from pypy.rlib.debug import ll_assert
 
         class State:
@@ -129,8 +128,6 @@
             state.finished = 0
             # the next line installs before_extcall() and after_extcall()
             # to be called automatically around external function calls.
-            # When not translated it does not work around time.sleep(),
-            # so we have to call them manually for this test.
             invoke_around_extcall(before_extcall, after_extcall)
 
             g(10, 1)
@@ -142,13 +139,9 @@
                 willing_to_wait_more -= 1
                 done = len(state.answers) == expected
 
-                if not we_are_translated(): before_extcall()
                 time.sleep(0.01)
-                if not we_are_translated(): after_extcall()
 
-            if not we_are_translated(): before_extcall()
             time.sleep(0.1)
-            if not we_are_translated(): after_extcall()
 
             return len(state.answers)
 
@@ -160,12 +153,11 @@
         answers = fn()
         assert answers == expected
 
-class TestRunDirectly(AbstractThreadTests):
-    def getcompiled(self, f, argtypes):
-        return f
-
-    def test_start_new_thread(self):
-        py.test.skip("deadlocks occasionally -- why???")
+#class TestRunDirectly(AbstractThreadTests):
+#    def getcompiled(self, f, argtypes):
+#        return f
+# These are disabled because they crash occasionally for bad reasons
+# related to the fact that ll2ctypes is not at all thread-safe
 
 class TestUsingBoehm(AbstractThreadTests):
     gcpolicy = 'boehm'
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -43,6 +43,13 @@
     return w_eq
 type_eq._annspecialcase_ = 'specialize:memo'
 
+def list_iter(space):
+    "Utility that returns the app-level descriptor list.__iter__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_list,
+                                               '__iter__')
+    return w_iter
+list_iter._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     w_type = space.type(w_obj)
     typename = w_type.getname(space)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -86,6 +86,7 @@
         return s_None
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -247,7 +247,7 @@
         return 0
 
     def iter(self, w_dict):
-        return EmptyIteratorImplementation(self.space, w_dict)
+        return EmptyIteratorImplementation(self.space, self, w_dict)
 
     def clear(self, w_dict):
         return
@@ -263,8 +263,9 @@
 # Iterator Implementation base classes
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.dictimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -280,7 +281,20 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.dictimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the dict.  The (key, value) pair in 'result'
+                # might be out-of-date.  We try to explicitly look up
+                # the key in the dict.
+                w_key = result[0]
+                w_value = self.dictimplementation.getitem(w_key)
+                if w_value is None:
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return (w_key, w_value)
         # no more entries
         self.dictimplementation = None
         return None, None
@@ -489,7 +503,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -503,7 +517,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -20,7 +20,17 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_w(w_lookup_type, space.w_str):
+        if (space.is_w(w_lookup_type, space.w_str) or  # Most common path first
+            space.abstract_issubclass_w(w_lookup_type, space.w_str)):
+            return self.getitem_str(w_dict, space.str_w(w_key))
+        elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode):
+            try:
+                w_key = space.str(w_key)
+            except OperationError, e:
+                if not e.match(space, space.w_UnicodeEncodeError):
+                    raise
+                # non-ascii unicode is never equal to a byte string
+                return None
             return self.getitem_str(w_dict, space.str_w(w_key))
         else:
             return None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -439,6 +439,8 @@
             t = w_obj.getitems()
         elif isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
+        elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            t = w_obj.getitems()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
         if expected_length != -1 and len(t) != expected_length:
@@ -456,6 +458,8 @@
             return w_obj.listview_str()
         if isinstance(w_obj, W_StringObject):
             return w_obj.listview_str()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_str()
         return None
 
     def listview_int(self, w_obj):
@@ -465,8 +469,14 @@
             return w_obj.listview_int()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_int()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_int()
         return None
 
+    def _uses_list_iter(self, w_obj):
+        from pypy.objspace.descroperation import list_iter
+        return self.lookup(w_obj, '__iter__') is list_iter(self)
+
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -359,7 +359,7 @@
         w_set.sstorage = w_other.get_storage_copy()
 
     def iter(self, w_set):
-        return EmptyIteratorImplementation(self.space, w_set)
+        return EmptyIteratorImplementation(self.space, self, w_set)
 
     def popitem(self, w_set):
         raise OperationError(self.space.w_KeyError,
@@ -784,8 +784,9 @@
             d_obj[w_item] = None
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.setimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -801,7 +802,17 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.setimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the set.  The 'result' might be out-of-date.
+                # We try to explicitly look it up in the set.
+                if not self.setimplementation.has_key(result):
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return result
         # no more entries
         self.setimplementation = None
         return None
@@ -823,7 +834,7 @@
 
 class StringIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, w_set):
-        IteratorImplementation.__init__(self, space, w_set)
+        IteratorImplementation.__init__(self, space, strategy, w_set)
         d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
@@ -835,9 +846,9 @@
 
 class IntegerIteratorImplementation(IteratorImplementation):
     #XXX same implementation in dictmultiobject on dictstrategy-branch
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
@@ -848,9 +859,9 @@
             return None
 
 class RDictIteratorImplementation(IteratorImplementation):
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -804,6 +804,33 @@
         assert "IntDictStrategy" in self.get_strategy(d)
         assert d[1L] == "hi"
 
+    def test_iter_dict_length_change(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d[7] = 8
+        # 'd' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_dict_strategy_only_change_1(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert d.get(Foo()) is None    # this changes the strategy of 'd'
+        lst = list(it)  # but iterating still works
+        assert sorted(lst) == [(1, 2), (3, 4), (5, 6)]
+
+    def test_iter_dict_strategy_only_change_2(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d['foo'] = 'bar'
+        del d[1]
+        # 'd' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us (1, 2), but 1 is not in the dict any longer.
+        raises(RuntimeError, list, it)
+
 
 class FakeString(str):
     hash_count = 0
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -25,6 +25,16 @@
         key, value = NotEmpty.__dict__.popitem()
         assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
 
+    def test_dictproxy_getitem(self):
+        class NotEmpty(object):
+            a = 1
+        assert 'a' in NotEmpty.__dict__
+        class substr(str): pass
+        assert substr('a') in NotEmpty.__dict__
+        assert u'a' in NotEmpty.__dict__
+        assert NotEmpty.__dict__[u'a'] == 1
+        assert u'\xe9' not in NotEmpty.__dict__
+
     def test_dictproxyeq(self):
         class a(object):
             pass
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1186,14 +1186,23 @@
         # of dicts, because the OrderedDict in the stdlib relies on this.
         # we extend the use case to lists and sets, i.e. all types that have
         # strategies, to avoid surprizes depending on the strategy.
-        for base, arg in [(list, []), (list, [5]), (list, ['x']),
-                          (set, []),  (set,  [5]), (set,  ['x']),
-                          (dict, []), (dict, [(5,6)]), (dict, [('x',7)])]:
+        class X: pass
+        for base, arg in [
+                (list, []), (list, [5]), (list, ['x']), (list, [X]),
+                (set, []),  (set,  [5]), (set,  ['x']), (set, [X]),
+                (dict, []), (dict, [(5,6)]), (dict, [('x',7)]), (dict, [(X,8)]),
+                ]:
             print base, arg
             class SubClass(base):
                 def __iter__(self):
                     return iter("foobar")
             assert list(SubClass(arg)) == ['f', 'o', 'o', 'b', 'a', 'r']
+            class Sub2(base):
+                pass
+            assert list(Sub2(arg)) == list(base(arg))
+            s = set()
+            s.update(Sub2(arg))
+            assert s == set(base(arg))
 
 class AppTestForRangeLists(AppTestW_ListObject):
 
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -907,3 +907,30 @@
                 return [5, 3, 4][i]
         s = set([10,3,2]).intersection(Obj())
         assert list(s) == [3]
+
+    def test_iter_set_length_change(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add(7)
+        # 's' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_set_strategy_only_change_1(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert Foo() not in s      # this changes the strategy of 'd'
+        lst = list(s)  # but iterating still works
+        assert sorted(lst) == [1, 3, 5]
+
+    def test_iter_set_strategy_only_change_2(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add('foo')
+        s.remove(1)
+        # 's' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us 1, but 1 is not in the set any longer.
+        raises(RuntimeError, list, it)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -382,7 +382,7 @@
         pass
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 vref_None = non_virtual_ref(None)
 
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -22,6 +22,7 @@
                 c_name = hop.inputconst(lltype.Void, 'access_helper')
                 args_v = [hop.inputarg(arg, arg=i)
                           for i, arg in enumerate(hop.args_r)]
+                hop.exception_cannot_occur()
                 return hop.genop('jit_marker', [c_name, c_func] + args_v,
                                  resulttype=hop.r_result)
         return helper
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -215,6 +215,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
@@ -397,6 +398,7 @@
         r_obj, = hop.args_r
         v_obj, = hop.inputargs(r_obj)
         ll_fn = r_obj.get_ll_hash_function()
+        hop.exception_is_here()
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
@@ -419,6 +421,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_identity_hash() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_identityhash', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -441,6 +444,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_id', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -452,6 +456,7 @@
 
     def specialize_call(self, hop):
         vobj, = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         if hop.rtyper.type_system.name == 'lltypesystem':
             from pypy.rpython.lltypesystem import lltype
             if isinstance(vobj.concretetype, lltype.Ptr):
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -85,7 +85,7 @@
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 
 class rbigint(object):
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -100,6 +100,7 @@
         def specialize_call(self, hop):
             bk = hop.rtyper.annotator.bookkeeper
             s_obj = identity.get_input_annotation(bk)
+            hop.exception_cannot_occur()
             return hop.r_result.rtype_erase(hop, s_obj)
 
     class Entry(ExtRegistryEntry):
@@ -110,6 +111,7 @@
             return identity.leave_tunnel(self.bookkeeper)
 
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
@@ -214,6 +216,7 @@
         return hop.genop('cast_opaque_ptr', [v], resulttype=hop.r_result)
 
     def rtype_unerase_int(self, hop, v):
+        hop.exception_cannot_occur()
         return hop.gendirectcall(ll_unerase_int, v)
 
     def rtype_erase_int(self, hop):
@@ -264,6 +267,7 @@
 
     def rtype_unerase_int(self, hop, v):
         c_one = hop.inputconst(lltype.Signed, 1)
+        hop.exception_cannot_occur()
         v2 = hop.genop('oounbox_int', [v], resulttype=hop.r_result)
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -382,6 +382,7 @@
     def compute_result_annotation(self):
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_roots', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
@@ -392,6 +393,7 @@
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_referents', vlist,
                          resulttype = hop.r_result)
 
@@ -402,6 +404,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_memory_usage', vlist,
                          resulttype = hop.r_result)
 
@@ -412,6 +415,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
@@ -430,6 +434,7 @@
         return annmodel.SomeBool()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_is_rpy_instance', vlist,
                          resulttype = hop.r_result)
 
@@ -449,6 +454,7 @@
         classrepr = getclassrepr(hop.rtyper, classdef)
         vtable = classrepr.getvtable()
         assert lltype.typeOf(vtable) == rclass.CLASSTYPE
+        hop.exception_cannot_occur()
         return Constant(vtable, concretetype=rclass.CLASSTYPE)
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py
--- a/pypy/rlib/rstring.py
+++ b/pypy/rlib/rstring.py
@@ -245,5 +245,5 @@
             raise ValueError("Value is not no_nul")
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
diff --git a/pypy/rpython/controllerentry.py b/pypy/rpython/controllerentry.py
--- a/pypy/rpython/controllerentry.py
+++ b/pypy/rpython/controllerentry.py
@@ -201,6 +201,7 @@
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
         assert hop.s_result.is_constant()
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -130,6 +130,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 def enum_ops_without_sideeffects(raising_is_ok=False):
diff --git a/pypy/rpython/lltypesystem/rbuiltin.py b/pypy/rpython/lltypesystem/rbuiltin.py
--- a/pypy/rpython/lltypesystem/rbuiltin.py
+++ b/pypy/rpython/lltypesystem/rbuiltin.py
@@ -9,6 +9,7 @@
 from pypy.rpython.rbool import bool_repr
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr or hop.args_r[1] == pyobj_repr:
@@ -33,6 +34,7 @@
     return my_instantiate()
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     s_class = hop.args_s[0]
     assert isinstance(s_class, annmodel.SomePBC)
     if len(s_class.descriptions) != 1:
@@ -46,6 +48,7 @@
     return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops)
 
 def rtype_builtin_hasattr(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr:
@@ -56,6 +59,7 @@
     raise TyperError("hasattr is only suported on a constant or on PyObject")
 
 def rtype_builtin___import__(hop):
+    xxx    # should not be used any more
     args_v = hop.inputargs(*[pyobj_repr for ign in hop.args_r])
     c = hop.inputconst(pyobj_repr, __import__)
     return hop.genop('simple_call', [c] + args_v, resulttype = pyobj_repr)
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -746,4 +746,5 @@
         assert isinstance(TYPE, GcStruct)
         assert lltype._castdepth(TYPE, OBJECT) > 0
         hop.rtyper.set_type_for_typeptr(vtable, TYPE)
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -55,6 +55,7 @@
         vtup = hop.inputarg(self, 0)
         LIST = hop.r_result.lowleveltype.TO
         cno = inputconst(Signed, nitems)
+        hop.exception_is_here()
         vlist = hop.gendirectcall(LIST.ll_newlist, cno)
         v_func = hop.inputconst(Void, rlist.dum_nocheck)
         for index in range(nitems):
diff --git a/pypy/rpython/module/r_os_stat.py b/pypy/rpython/module/r_os_stat.py
--- a/pypy/rpython/module/r_os_stat.py
+++ b/pypy/rpython/module/r_os_stat.py
@@ -65,4 +65,5 @@
     r_StatResult = hop.rtyper.getrepr(ll_os_stat.s_StatResult)
     [v_result] = hop.inputargs(r_StatResult.r_tuple)
     # no-op conversion from r_StatResult.r_tuple to r_StatResult
+    hop.exception_cannot_occur()
     return v_result
diff --git a/pypy/rpython/ootypesystem/ooregistry.py b/pypy/rpython/ootypesystem/ooregistry.py
--- a/pypy/rpython/ootypesystem/ooregistry.py
+++ b/pypy/rpython/ootypesystem/ooregistry.py
@@ -22,6 +22,7 @@
                                          annmodel.SomeOOInstance,
                                          annmodel.SomeString))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oostring', vlist, resulttype = ootype.String)
 
 class Entry_oounicode(ExtRegistryEntry):
@@ -38,6 +39,7 @@
         assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint,
                                           annmodel.SomeOOInstance))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oounicode', vlist, resulttype = ootype.Unicode)
     
 
diff --git a/pypy/rpython/ootypesystem/rbuiltin.py b/pypy/rpython/ootypesystem/rbuiltin.py
--- a/pypy/rpython/ootypesystem/rbuiltin.py
+++ b/pypy/rpython/ootypesystem/rbuiltin.py
@@ -7,12 +7,14 @@
 from pypy.rpython.error import TyperError
 
 def rtype_new(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputargs(ootype.Void)
     return hop.genop('new', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_oonewarray(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputarg(ootype.Void, arg=0)
     vlength = hop.inputarg(ootype.Signed, arg=1)
@@ -20,23 +22,27 @@
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_null(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     TYPE = hop.args_s[0].const
     nullvalue = ootype.null(TYPE)
     return hop.inputconst(TYPE, nullvalue)
 
 def rtype_classof(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
     vlist = hop.inputargs(hop.args_r[0])
     return hop.genop('classof', vlist,
                      resulttype = ootype.Class)
 
 def rtype_subclassof(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr, rootype.ooclass_repr)
     return hop.genop('subclassof', vlist,
                      resulttype = ootype.Bool)
 
 def rtype_instanceof(hop):
+    hop.exception_cannot_occur()
     INSTANCE = hop.args_v[1].value
     v_inst = hop.inputarg(hop.args_r[0], arg=0)
     c_cls = hop.inputconst(ootype.Void, INSTANCE)
@@ -44,23 +50,27 @@
                      resulttype=ootype.Bool)
 
 def rtype_runtimenew(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr)
     return hop.genop('runtimenew', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_ooupcast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_oodowncast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_to_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOStaticMeth) or \
            isinstance(hop.args_s[0], annmodel.SomeOOClass) or \
            isinstance(hop.args_s[0].ootype, ootype.OOType)
@@ -68,12 +78,14 @@
     return hop.genop('cast_to_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_from_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.OOType)
     assert isinstance(hop.args_s[1], annmodel.SomeOOObject)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('cast_from_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(ootype.Bool, hop.s_result.const)
 
@@ -99,6 +111,7 @@
     return ootype.subclassof(c1, class_)
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     if hop.args_s[0].is_constant():
 ##        INSTANCE = hop.s_result.rtyper_makerepr(hop.rtyper).lowleveltype
 ##        v_instance = hop.inputconst(ootype.Void, INSTANCE)
diff --git a/pypy/rpython/ootypesystem/rtuple.py b/pypy/rpython/ootypesystem/rtuple.py
--- a/pypy/rpython/ootypesystem/rtuple.py
+++ b/pypy/rpython/ootypesystem/rtuple.py
@@ -39,6 +39,7 @@
         RESULT = hop.r_result.lowleveltype
         c_resulttype = inputconst(ootype.Void, RESULT)
         c_length = inputconst(ootype.Signed, len(self.items_r))
+        hop.exception_is_here()
         if isinstance(RESULT, ootype.Array):
             v_list = hop.genop('oonewarray', [c_resulttype, c_length], resulttype=RESULT)
         else:
diff --git a/pypy/rpython/rbool.py b/pypy/rpython/rbool.py
--- a/pypy/rpython/rbool.py
+++ b/pypy/rpython/rbool.py
@@ -34,6 +34,7 @@
 
     def rtype_float(_, hop):
         vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
         return vlist[0]
 
 #
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -111,25 +111,32 @@
         raise TyperError("don't know about built-in function %r" % (
             self.builtinfunc,))
 
+    def _call(self, hop2, **kwds_i):
+        bltintyper = self.findbltintyper(hop2.rtyper)
+        hop2.llops._called_exception_is_here_or_cannot_occur = False
+        v_result = bltintyper(hop2, **kwds_i)
+        if not hop2.llops._called_exception_is_here_or_cannot_occur:
+            raise TyperError("missing hop.exception_cannot_occur() or "
+                             "hop.exception_is_here() in %s" % bltintyper)
+        return v_result
+
     def rtype_simple_call(self, hop):
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
-        return bltintyper(hop2)
+        return self._call(hop2)
 
     def rtype_call_args(self, hop):
         # calling a built-in function with keyword arguments:
         # mostly for rpython.objectmodel.hint()
         hop, kwds_i = call_args_expand(hop)
 
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
         hop2.r_s_popfirstarg()
         # the RPython-level keyword args are passed with an 'i_' prefix and
         # the corresponding value is an *index* in the hop2 arguments,
         # to be used with hop.inputarg(arg=..)
-        return bltintyper(hop2, **kwds_i)
+        return self._call(hop2, **kwds_i)
 
 
 class BuiltinMethodRepr(Repr):
@@ -198,6 +205,7 @@
 # ____________________________________________________________
 
 def rtype_builtin_bool(hop):
+    # not called any more?
     assert hop.nb_args == 1
     return hop.args_r[0].rtype_is_true(hop)
 
@@ -241,6 +249,7 @@
 
 def rtype_builtin_min(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_min, v1, v2)
 
 def ll_min(i1, i2):
@@ -250,6 +259,7 @@
 
 def rtype_builtin_max(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_max, v1, v2)
 
 def ll_max(i1, i2):
@@ -264,6 +274,7 @@
     pass
 
 def rtype_OSError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("OSError() should not be called with "
                          "a single argument")
@@ -274,6 +285,7 @@
         r_self.setfield(v_self, 'errno', v_errno, hop.llops)
 
 def rtype_WindowsError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("WindowsError() should not be called with "
                          "a single argument")
@@ -442,6 +454,7 @@
     assert hop.args_s[0].is_constant()
     TGT = hop.args_s[0].const
     v_type, v_value = hop.inputargs(lltype.Void, hop.args_r[1])
+    hop.exception_cannot_occur()
     return gen_cast(hop.llops, TGT, v_value)
 
 _cast_to_Signed = {
@@ -523,11 +536,13 @@
 
 def rtype_identity_hash(hop):
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('gc_identityhash', vlist, resulttype=lltype.Signed)
 
 def rtype_runtime_type_info(hop):
     assert isinstance(hop.args_r[0], rptr.PtrRepr)
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('runtime_type_info', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
@@ -558,6 +573,7 @@
 
 def rtype_raw_malloc(hop):
     v_size, = hop.inputargs(lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_malloc', [v_size], resulttype=llmemory.Address)
 
 def rtype_raw_malloc_usage(hop):
@@ -586,6 +602,7 @@
     if s_addr.is_null_address():
         raise TyperError("raw_memclear(x, n) where x is the constant NULL")
     v_list = hop.inputargs(llmemory.Address, lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_memclear', v_list)
 
 BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc
@@ -596,6 +613,7 @@
 
 def rtype_offsetof(hop):
     TYPE, field = hop.inputargs(lltype.Void, lltype.Void)
+    hop.exception_cannot_occur()
     return hop.inputconst(lltype.Signed,
                           llmemory.offsetof(TYPE.value, field.value))
 
@@ -605,6 +623,7 @@
 # non-gc objects
 
 def rtype_free_non_gc_object(hop):
+    hop.exception_cannot_occur()
     vinst, = hop.inputargs(hop.args_r[0])
     flavor = hop.args_r[0].gcflavor
     assert flavor != 'gc'
@@ -617,6 +636,7 @@
 # keepalive_until_here
 
 def rtype_keepalive_until_here(hop):
+    hop.exception_cannot_occur()
     for v in hop.args_v:
         hop.genop('keepalive', [v], resulttype=lltype.Void)
     return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/rfloat.py b/pypy/rpython/rfloat.py
--- a/pypy/rpython/rfloat.py
+++ b/pypy/rpython/rfloat.py
@@ -136,7 +136,10 @@
         hop.exception_cannot_occur()
         return hop.genop('cast_float_to_int', vlist, resulttype=Signed)
 
-    rtype_float = rtype_pos
+    def rtype_float(_, hop):
+        vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
+        return vlist[0]
 
     # version picked by specialisation based on which
     # type system rtyping is using, from <type_system>.ll_str module
diff --git a/pypy/rpython/rint.py b/pypy/rpython/rint.py
--- a/pypy/rpython/rint.py
+++ b/pypy/rpython/rint.py
@@ -310,6 +310,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_chr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_char', vlist, resulttype=Char)
 
     def rtype_unichr(_, hop):
@@ -317,6 +319,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_unichr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_unichar', vlist, resulttype=UniChar)
 
     def rtype_is_true(self, hop):
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -115,6 +115,7 @@
     def rtype_bltn_list(self, hop):
         v_lst = hop.inputarg(self, 0)
         cRESLIST = hop.inputconst(Void, hop.r_result.LIST)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_copy, cRESLIST, v_lst)
 
     def rtype_len(self, hop):
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -107,8 +107,10 @@
     if isinstance(hop.r_result, AbstractRangeRepr):
         if hop.r_result.step != 0:
             c_rng = hop.inputconst(Void, hop.r_result.RANGE)
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrange, c_rng, vstart, vstop)
         else:
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrangest, vstart, vstop, vstep)
     else:
         # cannot build a RANGE object, needs a real list
@@ -117,6 +119,7 @@
         if isinstance(ITEMTYPE, Ptr):
             ITEMTYPE = ITEMTYPE.TO
         cLIST = hop.inputconst(Void, ITEMTYPE)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_range2list, cLIST, vstart, vstop, vstep)
 
 rtype_builtin_xrange = rtype_builtin_range
@@ -212,4 +215,5 @@
                                      [v_index, v_item])
 
 def rtype_builtin_enumerate(hop):
+    hop.exception_cannot_occur()
     return hop.r_result.r_baseiter.newiter(hop)
diff --git a/pypy/rpython/rstr.py b/pypy/rpython/rstr.py
--- a/pypy/rpython/rstr.py
+++ b/pypy/rpython/rstr.py
@@ -288,6 +288,8 @@
 
     def rtype_unicode(self, hop):
         if hop.args_s[0].is_constant():
+            # convertion errors occur during annotation, so cannot any more:
+            hop.exception_cannot_occur()
             return hop.inputconst(hop.r_result, hop.s_result.const)
         repr = hop.args_r[0].repr
         v_str = hop.inputarg(repr, 0)
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -846,6 +846,7 @@
         return result
 
     def exception_is_here(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
@@ -861,6 +862,7 @@
         self.llops.llop_raising_exceptions = len(self.llops)
 
     def exception_cannot_occur(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
diff --git a/pypy/rpython/test/test_extregistry.py b/pypy/rpython/test/test_extregistry.py
--- a/pypy/rpython/test/test_extregistry.py
+++ b/pypy/rpython/test/test_extregistry.py
@@ -114,6 +114,7 @@
         _about_ = dummy_func
         s_result_annotation = annmodel.SomeInteger()
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             return hop.inputconst(lltype.Signed, 42)
 
     def func():
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1085,6 +1085,7 @@
                 return annmodel.SomeInteger()
             def specialize_call(self, hop):
                 [v_instance] = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.gendirectcall(ll_my_gethash, v_instance)
 
         def f(n):
diff --git a/pypy/translator/c/test/test_extfunc.py b/pypy/translator/c/test/test_extfunc.py
--- a/pypy/translator/c/test/test_extfunc.py
+++ b/pypy/translator/c/test/test_extfunc.py
@@ -919,4 +919,5 @@
             t, cbuilder = self.compile(does_stuff)
             data = cbuilder.cmdexec('')
             res = os.nice(0) + 3
+            if res > 19: res = 19    # xxx Linux specific, probably
             assert data.startswith('os.nice returned %d\n' % res)
diff --git a/pypy/translator/cli/dotnet.py b/pypy/translator/cli/dotnet.py
--- a/pypy/translator/cli/dotnet.py
+++ b/pypy/translator/cli/dotnet.py
@@ -459,6 +459,7 @@
             
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         assert hop.args_s[1].is_constant()
         TYPE = hop.args_s[1].const
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
@@ -507,6 +508,7 @@
 
     def specialize_call(self, hop):
         v_obj, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('same_as', [v_obj], hop.r_result.lowleveltype)
 
 def new_array(type, length):
@@ -608,6 +610,7 @@
 
     def specialize_call(self, hop):
         v_type, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('cli_typeof', [v_type], hop.r_result.lowleveltype)
 
 
@@ -626,6 +629,7 @@
         v_obj, = hop.inputargs(*hop.args_r)
         methodname = hop.args_r[0].methodname
         c_methodname = hop.inputconst(ootype.Void, methodname)
+        hop.exception_cannot_occur()
         return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype)
 
 
@@ -647,6 +651,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -668,6 +673,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -701,6 +707,7 @@
 
     def specialize_call(self, hop):
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype)
 
 

From noreply at buildbot.pypy.org  Mon Apr  2 21:18:30 2012
From: noreply at buildbot.pypy.org (ctismer)
Date: Mon,  2 Apr 2012 21:18:30 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Added myself,
	without exact dates yet
Message-ID: <20120402191830.5DB4746E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Christian Tismer  <tismer at stackless.com>
Branch: extradoc
Changeset: r4177:5099f4763dbb
Date: 2012-04-02 21:17 +0200
http://bitbucket.org/pypy/extradoc/changeset/5099f4763dbb/

Log:	Added myself, without exact dates yet

diff --git a/sprintinfo/leipzig2012/people.txt b/sprintinfo/leipzig2012/people.txt
--- a/sprintinfo/leipzig2012/people.txt
+++ b/sprintinfo/leipzig2012/people.txt
@@ -9,5 +9,6 @@
     Name              Arrive/Depart     Accomodation 
 ==================== ============== =====================
 Armin Rigo           ?              ?
+Christian Tismer     ?              ?
 ==================== ============== =====================
 

From noreply at buildbot.pypy.org  Tue Apr  3 00:32:37 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue,  3 Apr 2012 00:32:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix test_celldict,
	and translation as well.
Message-ID: <20120402223237.D091346E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54157:3f8bd87cfc04
Date: 2012-04-03 00:31 +0200
http://bitbucket.org/pypy/pypy/changeset/3f8bd87cfc04/

Log:	Fix test_celldict, and translation as well.

diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -163,7 +163,8 @@
 
 class ModuleDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         dict_w = strategy.unerase(dictimplementation.dstorage)
         self.iterator = dict_w.iteritems()
 

From noreply at buildbot.pypy.org  Tue Apr  3 00:51:59 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue,  3 Apr 2012 00:51:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Same fix on other dict implementations
	:(
Message-ID: <20120402225159.2BFDF46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54158:cc56984630fc
Date: 2012-04-03 00:51 +0200
http://bitbucket.org/pypy/pypy/changeset/cc56984630fc/

Log:	Same fix on other dict implementations :(

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -108,7 +108,8 @@
 
 class DictProxyIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_type = strategy.unerase(dictimplementation.dstorage)
         self.iterator = w_type.dict_w.iteritems()
 
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -703,7 +703,8 @@
 
 class MapDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_obj = strategy.unerase(dictimplementation.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:36 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:36 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: groundwork for lazy lookup on
 namespaces (for now, only for updates)
Message-ID: <20120402234336.3761746E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54159:228864642d3d
Date: 2012-03-30 13:51 -0700
http://bitbucket.org/pypy/pypy/changeset/228864642d3d/

Log:	groundwork for lazy lookup on namespaces (for now, only for updates)

diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py
--- a/pypy/module/cppyy/capi/__init__.py
+++ b/pypy/module/cppyy/capi/__init__.py
@@ -293,6 +293,12 @@
 def c_method_signature(cppscope, method_index):
     return charp2str_free(_c_method_signature(cppscope, method_index))
 
+c_method_index = rffi.llexternal(
+    "cppyy_method_index",
+    [C_SCOPE, rffi.CCHARP], rffi.INT,
+    threadsafe=threadsafe,
+    compilation_info=backend.eci)
+
 c_get_method = rffi.llexternal(
     "cppyy_get_method",
     [C_SCOPE, rffi.INT], C_METHOD,
@@ -337,6 +343,12 @@
     threadsafe=threadsafe,
     compilation_info=backend.eci)
 
+c_data_member_index = rffi.llexternal(
+    "cppyy_data_member_index",
+    [C_SCOPE, rffi.CCHARP], rffi.INT,
+    threadsafe=threadsafe,
+    compilation_info=backend.eci)
+
 # data member properties -----------------------------------------------------
 c_is_publicdata = rffi.llexternal(
     "cppyy_is_publicdata",
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -71,6 +71,8 @@
     char* cppyy_method_arg_default(cppyy_scope_t scope, int method_index, int arg_index);
     char* cppyy_method_signature(cppyy_scope_t scope, int method_index);
 
+    int cppyy_method_index(cppyy_scope_t scope, const char* name);
+
     cppyy_method_t cppyy_get_method(cppyy_scope_t scope, int method_index);
 
     /* method properties -----------------------------------------------------  */
@@ -83,6 +85,8 @@
     char* cppyy_data_member_type(cppyy_scope_t scope, int data_member_index);
     size_t cppyy_data_member_offset(cppyy_scope_t scope, int data_member_index);
 
+    int cppyy_data_member_index(cppyy_scope_t scope, const char* name);
+
     /* data member properties ------------------------------------------------  */
     int cppyy_is_publicdata(cppyy_type_t type, int data_member_index);
     int cppyy_is_staticdata(cppyy_type_t type, int data_member_index);
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -413,7 +413,8 @@
         try:
             return self.methods[name]
         except KeyError:
-            raise self.missing_attribute_error(name)
+            pass
+        return self.find_overload(name)
 
     def get_data_member_names(self):
         return self.space.newlist([self.space.wrap(name) for name in self.data_members])
@@ -423,7 +424,8 @@
         try:
             return self.data_members[name]
         except KeyError:
-            raise self.missing_attribute_error(name)
+            pass
+        return self.find_data_member(name)
 
     def missing_attribute_error(self, name):
         return OperationError(
@@ -451,6 +453,13 @@
             arg_defs.append((arg_type, arg_dflt))
         return CPPFunction(self, method_index, result_type, arg_defs, args_required)
 
+    def _make_data_member(self, dm_name, dm_idx):
+        type_name = capi.c_data_member_type(self.handle, dm_idx)
+        offset = capi.c_data_member_offset(self.handle, dm_idx)
+        data_member = W_CPPDataMember(self.space, self.handle, type_name, offset, True)
+        self.data_members[dm_name] = data_member
+        return data_member
+
     def _find_data_members(self):
         num_data_members = capi.c_num_data_members(self.handle)
         for i in range(num_data_members):
@@ -458,10 +467,24 @@
                 continue
             data_member_name = capi.c_data_member_name(self.handle, i)
             if not data_member_name in self.data_members:
-                type_name = capi.c_data_member_type(self.handle, i)
-                offset = capi.c_data_member_offset(self.handle, i)
-                data_member = W_CPPDataMember(self.space, self.handle, type_name, offset, True)
-                self.data_members[data_member_name] = data_member
+                self._make_data_member(data_member_name, i)
+
+    def find_overload(self, meth_name):
+        # TODO: collect all overloads, not just the non-overloaded version
+        meth_idx = capi.c_method_index(self.handle, meth_name)
+        if meth_idx < 0:
+            raise self.missing_attribute_error(meth_name)
+        cppfunction = self._make_cppfunction(meth_idx)
+        overload = W_CPPOverload(self.space, self.handle, meth_name, [cppfunction])
+        self.methods[meth_name] = overload
+        return overload
+
+    def find_data_member(self, dm_name):
+        dm_idx = capi.c_data_member_index(self.handle, dm_name)
+        if dm_idx < 0:
+            raise self.missing_attribute_error(dm_name)
+        data_member = self._make_data_member(dm_name, dm_idx)
+        return data_member
 
     def update(self):
         self._find_methods()
@@ -517,6 +540,12 @@
             data_member = W_CPPDataMember(self.space, self.handle, type_name, offset, is_static)
             self.data_members[data_member_name] = data_member
 
+    def find_overload(self, name):
+        raise self.missing_attribute_error(name)
+
+    def find_data_member(self, name):
+        raise self.missing_attribute_error(name)
+
     def get_cppthis(self, cppinstance, scope_handle):
         assert self.handle == cppinstance.cppclass.handle
         return cppinstance.get_rawobject()
diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -207,13 +207,6 @@
 
     pycppitem = None
 
-    # namespaces are "open"; TODO: classes are too (template methods, inner classes ...)
-    if isinstance(scope, CppyyNamespaceMeta):
-        global _loaded_dictionaries_isdirty
-        if _loaded_dictionaries_isdirty:  # TODO: this should be per namespace
-            scope._cpp_proxy.update()     # TODO: this is currently quadratic
-            _loaded_dictionaries_isdirty = False
-
     # classes
     cppitem = cppyy._type_byname(true_name)
     if cppitem:
@@ -365,15 +358,12 @@
         pyclass.__getitem__ = pyclass.__setitem__
 
 _loaded_dictionaries = {}
-_loaded_dictionaries_isdirty = True     # should be per namespace
 def load_reflection_info(name):
     try:
         return _loaded_dictionaries[name]
     except KeyError:
         dct = cppyy._load_dictionary(name)
         _loaded_dictionaries[name] = dct
-        global _loaded_dictionaries_isdirty
-        _loaded_dictionaries_isdirty = True
         return dct
     
 
diff --git a/pypy/module/cppyy/src/reflexcwrapper.cxx b/pypy/module/cppyy/src/reflexcwrapper.cxx
--- a/pypy/module/cppyy/src/reflexcwrapper.cxx
+++ b/pypy/module/cppyy/src/reflexcwrapper.cxx
@@ -372,6 +372,23 @@
     return cppstring_to_cstring(sig.str());
 }
 
+int cppyy_method_index(cppyy_scope_t handle, const char* name) {
+    Reflex::Scope s = scope_from_handle(handle);
+    // the following appears dumb, but the internal storage for Reflex is an
+    // unsorted std::vector anyway, so there's no gain to be had in using the
+    // Scope::FunctionMemberByName() function
+    int num_meth = s.FunctionMemberSize();
+    for (int imeth = 0; imeth < num_meth; ++imeth) {
+        Reflex::Member m = s.FunctionMemberAt(imeth);
+        if (m.Name() == name) {
+            if (m.IsPublic())
+                return imeth;
+            return -1;
+        }
+    }
+    return -1;
+}
+
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, int method_index) {
     Reflex::Scope s = scope_from_handle(handle);
     Reflex::Member m = s.FunctionMemberAt(method_index);
@@ -436,6 +453,23 @@
     return m.Offset();
 }
 
+int cppyy_data_member_index(cppyy_scope_t handle, const char* name) {
+    Reflex::Scope s = scope_from_handle(handle);
+    // the following appears dumb, but the internal storage for Reflex is an
+    // unsorted std::vector anyway, so there's no gain to be had in using the
+    // Scope::DataMemberByName() function (which returns Member, not an index)
+    int num_dm = cppyy_num_data_members(handle);
+    for (int idm = 0; idm < num_dm; ++idm) {
+        Reflex::Member m = s.DataMemberAt(idm);
+        if (m.Name() == name || m.Name(Reflex::FINAL) == name) {
+            if (m.IsPublic())
+                return idm;
+            return -1;
+        }
+    }
+    return -1;
+}
+
 
 /* data member properties ------------------------------------------------  */
 int cppyy_is_publicdata(cppyy_scope_t handle, int data_member_index) {
diff --git a/pypy/module/cppyy/test/Makefile b/pypy/module/cppyy/test/Makefile
--- a/pypy/module/cppyy/test/Makefile
+++ b/pypy/module/cppyy/test/Makefile
@@ -1,5 +1,6 @@
-dicts = example01Dict.so datatypesDict.so advancedcppDict.so overloadsDict.so \
-stltypesDict.so operatorsDict.so fragileDict.so std_streamsDict.so
+dicts = example01Dict.so datatypesDict.so advancedcppDict.so advancedcpp2Dict.so \
+overloadsDict.so stltypesDict.so operatorsDict.so fragileDict.so \
+std_streamsDict.so
 all : $(dicts)
 
 ROOTSYS := ${ROOTSYS}
diff --git a/pypy/module/cppyy/test/advancedcpp.cxx b/pypy/module/cppyy/test/advancedcpp.cxx
--- a/pypy/module/cppyy/test/advancedcpp.cxx
+++ b/pypy/module/cppyy/test/advancedcpp.cxx
@@ -24,6 +24,9 @@
 int a_ns::d_ns::e_class::s_e          = 55;
 int a_ns::d_ns::e_class::f_class::s_f = 66;
 
+int a_ns::get_g_a() { return g_a; }
+int a_ns::d_ns::get_g_d() { return g_d; }
+
 
 // for template testing
 template class T1<int>;
diff --git a/pypy/module/cppyy/test/advancedcpp.h b/pypy/module/cppyy/test/advancedcpp.h
--- a/pypy/module/cppyy/test/advancedcpp.h
+++ b/pypy/module/cppyy/test/advancedcpp.h
@@ -100,6 +100,7 @@
 //===========================================================================
 namespace a_ns {                   // for namespace testing
    extern int g_a;
+   int get_g_a();
 
    struct b_class {
       b_class() { m_b = -2; }
@@ -115,6 +116,7 @@
 
    namespace d_ns {
       extern int g_d;
+      int get_g_d();
 
       struct e_class {
          e_class() { m_e = -5; }
diff --git a/pypy/module/cppyy/test/advancedcpp.xml b/pypy/module/cppyy/test/advancedcpp.xml
--- a/pypy/module/cppyy/test/advancedcpp.xml
+++ b/pypy/module/cppyy/test/advancedcpp.xml
@@ -22,7 +22,9 @@
   <namespace pattern="a_ns::*" />
   <class pattern="a_ns::*" />
   <variable name="a_ns::g_a" />
+  <function name="a_ns::get_g_a" />
   <variable name="a_ns::d_ns::g_d" />
+  <function name="a_ns::d_ns::get_g_d" />
 
   <class name="some_abstract_class" />
   <class name="some_concrete_class" />
diff --git a/pypy/module/cppyy/test/test_advancedcpp.py b/pypy/module/cppyy/test/test_advancedcpp.py
--- a/pypy/module/cppyy/test/test_advancedcpp.py
+++ b/pypy/module/cppyy/test/test_advancedcpp.py
@@ -10,9 +10,10 @@
 def setup_module(mod):
     if sys.platform == 'win32':
         py.test.skip("win32 not supported so far")
-    err = os.system("cd '%s' && make advancedcppDict.so" % currpath)
-    if err:
-        raise OSError("'make' failed (see stderr)")
+    for refl_dict in ["advancedcppDict.so", "advancedcpp2Dict.so"]:
+        err = os.system("cd '%s' && make %s" % (currpath, refl_dict))
+        if err:
+            raise OSError("'make' failed (see stderr)")
 
 class AppTestADVANCEDCPP:
     def setup_class(cls):
@@ -113,23 +114,55 @@
         import cppyy
         gbl = cppyy.gbl
 
+        assert gbl.a_ns      is gbl.a_ns
+        assert gbl.a_ns.d_ns is gbl.a_ns.d_ns
+
+        assert gbl.a_ns.b_class              is gbl.a_ns.b_class
+        assert gbl.a_ns.b_class.c_class      is gbl.a_ns.b_class.c_class
+        assert gbl.a_ns.d_ns.e_class         is gbl.a_ns.d_ns.e_class
+        assert gbl.a_ns.d_ns.e_class.f_class is gbl.a_ns.d_ns.e_class.f_class
+
         assert gbl.a_ns.g_a                           == 11
+        assert gbl.a_ns.get_g_a()                     == 11
         assert gbl.a_ns.b_class.s_b                   == 22
         assert gbl.a_ns.b_class().m_b                 == -2
         assert gbl.a_ns.b_class.c_class.s_c           == 33
         assert gbl.a_ns.b_class.c_class().m_c         == -3
         assert gbl.a_ns.d_ns.g_d                      == 44
+        assert gbl.a_ns.d_ns.get_g_d()                == 44
         assert gbl.a_ns.d_ns.e_class.s_e              == 55
         assert gbl.a_ns.d_ns.e_class().m_e            == -5
         assert gbl.a_ns.d_ns.e_class.f_class.s_f      == 66
         assert gbl.a_ns.d_ns.e_class.f_class().m_f    == -6
 
+    def test03a_namespace_lookup_on_update(self):
+        """Test whether namespaces can be shared across dictionaries."""
+
+        import cppyy
+        gbl = cppyy.gbl
+
+        lib2 = cppyy.load_reflection_info("advancedcpp2Dict.so")
+
         assert gbl.a_ns      is gbl.a_ns
         assert gbl.a_ns.d_ns is gbl.a_ns.d_ns
 
-        assert gbl.a_ns.b_class              is gbl.a_ns.b_class
-        assert gbl.a_ns.d_ns.e_class         is gbl.a_ns.d_ns.e_class
-        assert gbl.a_ns.d_ns.e_class.f_class is gbl.a_ns.d_ns.e_class.f_class
+        assert gbl.a_ns.g_class              is gbl.a_ns.g_class
+        assert gbl.a_ns.g_class.h_class      is gbl.a_ns.g_class.h_class
+        assert gbl.a_ns.d_ns.i_class         is gbl.a_ns.d_ns.i_class
+        assert gbl.a_ns.d_ns.i_class.j_class is gbl.a_ns.d_ns.i_class.j_class
+
+        assert gbl.a_ns.g_g                           ==  77
+        assert gbl.a_ns.get_g_g()                     ==  77
+        assert gbl.a_ns.g_class.s_g                   ==  88
+        assert gbl.a_ns.g_class().m_g                 ==  -7
+        assert gbl.a_ns.g_class.h_class.s_h           ==  99
+        assert gbl.a_ns.g_class.h_class().m_h         ==  -8
+        assert gbl.a_ns.d_ns.g_i                      == 111
+        assert gbl.a_ns.d_ns.get_g_i()                == 111
+        assert gbl.a_ns.d_ns.i_class.s_i              == 222
+        assert gbl.a_ns.d_ns.i_class().m_i            ==  -9
+        assert gbl.a_ns.d_ns.i_class.j_class.s_j      == 333
+        assert gbl.a_ns.d_ns.i_class.j_class().m_j    == -10
 
     def test04_template_types(self):
         """Test bindings of templated types"""
diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py
--- a/pypy/module/cppyy/test/test_pythonify.py
+++ b/pypy/module/cppyy/test/test_pythonify.py
@@ -24,7 +24,7 @@
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
 
-    def test01_load_dicionary_cache(self):
+    def test01_load_dictionary_cache(self):
         """Test whether loading a dictionary twice results in the same object."""
         import cppyy
         lib2 = cppyy.load_reflection_info(self.test_dct)

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:37 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:37 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: files needed for update-handling
	testing
Message-ID: <20120402234337.7155946E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54160:f2d76f0952a9
Date: 2012-03-30 14:03 -0700
http://bitbucket.org/pypy/pypy/changeset/f2d76f0952a9/

Log:	files needed for update-handling testing

diff --git a/pypy/module/cppyy/test/advancedcpp2.cxx b/pypy/module/cppyy/test/advancedcpp2.cxx
new file mode 100644
--- /dev/null
+++ b/pypy/module/cppyy/test/advancedcpp2.cxx
@@ -0,0 +1,13 @@
+#include "advancedcpp2.h"
+
+
+// for namespace testing
+int a_ns::g_g                         =  77;
+int a_ns::g_class::s_g                =  88;
+int a_ns::g_class::h_class::s_h       =  99;
+int a_ns::d_ns::g_i                   = 111;
+int a_ns::d_ns::i_class::s_i          = 222;
+int a_ns::d_ns::i_class::j_class::s_j = 333;
+
+int a_ns::get_g_g() { return g_g; }
+int a_ns::d_ns::get_g_i() { return g_i; }
diff --git a/pypy/module/cppyy/test/advancedcpp2.h b/pypy/module/cppyy/test/advancedcpp2.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cppyy/test/advancedcpp2.h
@@ -0,0 +1,36 @@
+//===========================================================================
+namespace a_ns {                   // for namespace testing
+   extern int g_g;
+   int get_g_g();
+
+   struct g_class {
+      g_class() { m_g = -7; }
+      int m_g;
+      static int s_g;
+
+      struct h_class {
+         h_class() { m_h = -8; }
+         int m_h;
+         static int s_h;
+      };
+   };
+
+   namespace d_ns {
+      extern int g_i;
+      int get_g_i();
+
+      struct i_class {
+         i_class() { m_i = -9; }
+         int m_i;
+         static int s_i;
+
+         struct j_class {
+            j_class() { m_j = -10; }
+            int m_j;
+            static int s_j;
+         };
+      };
+
+   } // namespace d_ns
+
+} // namespace a_ns
diff --git a/pypy/module/cppyy/test/advancedcpp2.xml b/pypy/module/cppyy/test/advancedcpp2.xml
new file mode 100644
--- /dev/null
+++ b/pypy/module/cppyy/test/advancedcpp2.xml
@@ -0,0 +1,11 @@
+<lcgdict>
+
+  <namespace name="a_ns" />
+  <namespace pattern="a_ns::*" />
+  <class pattern="a_ns::*" />
+  <variable name="a_ns::g_g" />
+  <function name="a_ns::get_g_g" />
+  <variable name="a_ns::d_ns::g_i" />
+  <function name="a_ns::d_ns::get_g_i" />
+
+</lcgdict>

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:38 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:38 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: likewise,
	lazy lookups for CINT back-end
Message-ID: <20120402234338.BF81F46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54161:8277dedafc32
Date: 2012-03-30 14:57 -0700
http://bitbucket.org/pypy/pypy/changeset/8277dedafc32/

Log:	likewise, lazy lookups for CINT back-end

diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx
--- a/pypy/module/cppyy/src/cintcwrapper.cxx
+++ b/pypy/module/cppyy/src/cintcwrapper.cxx
@@ -36,6 +36,23 @@
 extern "C" void G__LockCriticalSection();
 extern "C" void G__UnlockCriticalSection();
 
+/* ROOT meta internals ---------------------------------------------------- */
+namespace {
+
+class Cppyy_OpenedTClass : public TDictionary {
+public:
+   mutable TObjArray *fStreamerInfo;    //Array of TVirtualStreamerInfo
+   mutable std::map<std::string, TObjArray*> *fConversionStreamerInfo; //Array of the streamer infos derived from another class.
+   TList             *fRealData;        //linked list for persistent members including base classes
+   TList             *fBase;            //linked list for base classes
+   TList             *fData;            //linked list for data members
+   TList             *fMethod;          //linked list for methods
+   TList             *fAllPubData;      //all public data members (including from base classes)
+   TList             *fAllPubMethod;    //all public methods (including from base classes)
+};
+
+} // unnamed namespace
+
 
 /* data for life time management ------------------------------------------ */
 #define GLOBAL_HANDLE 1l
@@ -540,6 +557,30 @@
     return cppstring_to_cstring(sig.str());
 }
 
+int cppyy_method_index(cppyy_scope_t handle, const char* name) {
+    TClassRef cr = type_from_handle(handle);
+    if (cr.GetClass()) {
+        gInterpreter->UpdateListOfMethods(cr.GetClass());
+        int imeth = 0;
+        TFunction* func;
+        TIter next(cr->GetListOfMethods());
+        while ((func = (TFunction*)next())) {
+            if (strcmp(name, func->GetName()) == 0) {
+                if (func->Property() & G__BIT_ISPUBLIC)
+                    return imeth;
+                return -1;
+            }
+            ++imeth;
+        }
+    }
+    TFunction* func = gROOT->GetGlobalFunction(name, NULL, kTRUE);
+    if (!func)
+        return -1;
+    int idx = g_globalfuncs.size();
+    g_globalfuncs.push_back(*func);
+    return idx;
+}
+
 cppyy_method_t cppyy_get_method(cppyy_scope_t handle, int method_index) {
     TFunction* f = type_get_method(handle, method_index);
     return (cppyy_method_t)f->InterfaceMethod();
@@ -621,6 +662,39 @@
     return (size_t)gbl.GetAddress();
 }
 
+int cppyy_data_member_index(cppyy_scope_t handle, const char* name) {
+    TClassRef cr = type_from_handle(handle);
+    if (cr.GetClass()) {
+        // called from updates; add a hard reset as the code itself caches in
+        // Class (TODO: by-pass ROOT/meta)
+        Cppyy_OpenedTClass* c = (Cppyy_OpenedTClass*)cr.GetClass();
+        if (c->fData) {
+            c->fData->Delete();
+            delete c->fData; c->fData = 0;
+            delete c->fAllPubData; c->fAllPubData = 0;
+        }
+        // the following appears dumb, but TClass::GetDataMember() does a linear
+        // search itself, so there is no gain
+        int idm = 0;
+        TDataMember* dm;
+        TIter next(cr->GetListOfDataMembers());
+        while ((dm = (TDataMember*)next())) {
+            if (strcmp(name, dm->GetName()) == 0) {
+                if (dm->Property() & G__BIT_ISPUBLIC)
+                    return idm;
+                return -1;
+            }
+            ++idm;
+        }
+    }
+    TGlobal* gbl = (TGlobal*)gROOT->GetListOfGlobals(kTRUE)->FindObject(name);
+    if (!gbl)
+        return -1;
+    int idx = g_globalvars.size();
+    g_globalvars.push_back(*gbl);
+    return idx;
+}
+
 
 /* data member properties ------------------------------------------------  */
 int cppyy_is_publicdata(cppyy_scope_t handle, int data_member_index) {
diff --git a/pypy/module/cppyy/test/advancedcpp_LinkDef.h b/pypy/module/cppyy/test/advancedcpp_LinkDef.h
--- a/pypy/module/cppyy/test/advancedcpp_LinkDef.h
+++ b/pypy/module/cppyy/test/advancedcpp_LinkDef.h
@@ -36,7 +36,9 @@
 #pragma link C++ struct a_ns::d_ns::e_class;
 #pragma link C++ struct a_ns::d_ns::e_class::f_class;
 #pragma link C++ variable a_ns::g_a;
+#pragma link C++ function a_ns::get_g_a;
 #pragma link C++ variable a_ns::d_ns::g_d;
+#pragma link C++ function a_ns::d_ns::get_g_d;
 
 #pragma link C++ class some_abstract_class;
 #pragma link C++ class some_concrete_class;

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:40 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:40 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: rollback (translation fails in
	array)
Message-ID: <20120402234340.940B446E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54162:a869f2ef3eb2
Date: 2012-03-30 16:20 -0700
http://bitbucket.org/pypy/pypy/changeset/a869f2ef3eb2/

Log:	rollback (translation fails in array)


From noreply at buildbot.pypy.org  Tue Apr  3 01:43:41 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:41 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: needed for CINT update tests
Message-ID: <20120402234341.CB58E46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54163:4794b0f265d6
Date: 2012-03-30 16:21 -0700
http://bitbucket.org/pypy/pypy/changeset/4794b0f265d6/

Log:	needed for CINT update tests

diff --git a/pypy/module/cppyy/test/advancedcpp2_LinkDef.h b/pypy/module/cppyy/test/advancedcpp2_LinkDef.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cppyy/test/advancedcpp2_LinkDef.h
@@ -0,0 +1,18 @@
+#ifdef __CINT__
+
+#pragma link off all globals;
+#pragma link off all classes;
+#pragma link off all functions;
+
+#pragma link C++ namespace a_ns;
+#pragma link C++ namespace a_ns::d_ns;
+#pragma link C++ struct a_ns::g_class;
+#pragma link C++ struct a_ns::g_class::h_class;
+#pragma link C++ struct a_ns::d_ns::i_class;
+#pragma link C++ struct a_ns::d_ns::i_class::j_class;
+#pragma link C++ variable a_ns::g_g;
+#pragma link C++ function a_ns::get_g_g;
+#pragma link C++ variable a_ns::d_ns::g_i;
+#pragma link C++ function a_ns::d_ns::get_g_i;
+
+#endif

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:43 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:43 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: easier use for testing the
	various executables
Message-ID: <20120402234343.7740046E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54164:ea7e2c6e0d5b
Date: 2012-03-30 16:52 -0700
http://bitbucket.org/pypy/pypy/changeset/ea7e2c6e0d5b/

Log:	easier use for testing the various executables

diff --git a/pypy/module/cppyy/bench/bench02.h b/pypy/module/cppyy/bench/bench02.h
--- a/pypy/module/cppyy/bench/bench02.h
+++ b/pypy/module/cppyy/bench/bench02.h
@@ -7,12 +7,15 @@
 #include "TH1F.h"
 #include "TH2F.h"
 #include "TRandom.h"
+#include "TRandom3.h"
 
 #include "TROOT.h"
 #include "TApplication.h"
+#include "TSystem.h"
 
 #include "TArchiveFile.h"
 #include "TBasket.h"
+#include "TBenchmark.h"
 #include "TBox.h"
 #include "TBranchRef.h"
 #include "TBrowser.h"
@@ -67,10 +70,3 @@
    void report();
    void close_file(TFile* f);
 };
-
-/*
-gROOT      = cppyy.gbl.gROOT
-gBenchmark = cppyy.gbl.gBenchmark
-gRandom    = cppyy.gbl.gRandom
-gSystem    = cppyy.gbl.gSystem
-*/
diff --git a/pypy/module/cppyy/bench/bench02.xml b/pypy/module/cppyy/bench/bench02.xml
--- a/pypy/module/cppyy/bench/bench02.xml
+++ b/pypy/module/cppyy/bench/bench02.xml
@@ -7,6 +7,11 @@
      <class pattern="ROOT::T[A-Z]*" />
      <class pattern="ROOT::Fit::*" />
 
+     <!-- ROOT globals -->
+     <variable name="gROOT" />
+     <variable name="gSystem" />
+     <variable name="gRandom" />
+
      <!-- STL classes actually used -->
      <class name="std::string" />
      <class name="std::ostream" />
diff --git a/pypy/module/cppyy/bench/hsimple.py b/pypy/module/cppyy/bench/hsimple.py
--- a/pypy/module/cppyy/bench/hsimple.py
+++ b/pypy/module/cppyy/bench/hsimple.py
@@ -10,9 +10,15 @@
 #*-*
 #*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*
 
+_reflex = True     # to keep things equal, set to False for full macro
+
 try:
     import cppyy, random
 
+    if not hasattr(cppyy.gbl, 'gROOT'):
+        cppyy.load_reflection_info('bench02Dict_reflex.so')
+        _reflex = True
+
     TCanvas  = cppyy.gbl.TCanvas
     TFile    = cppyy.gbl.TFile
     TProfile = cppyy.gbl.TProfile
@@ -30,22 +36,27 @@
     from ROOT import gROOT, gBenchmark, gSystem
     import random
 
+if _reflex:
+   gROOT.SetBatch(True)
+
 # Create a new ROOT binary machine independent file.
 # Note that this file may contain any kind of ROOT objects, histograms,
 # pictures, graphics objects, detector geometries, tracks, events, etc..
 # This file is now becoming the current directory.
 
-hfile = gROOT.FindObject('hsimple.root')
-if hfile:
-    hfile.Close()
-hfile = TFile('hsimple.root', 'RECREATE', 'Demo ROOT file with histograms' )
+if not _reflex:
+    hfile = gROOT.FindObject('hsimple.root')
+    if hfile:
+        hfile.Close()
+    hfile = TFile('hsimple.root', 'RECREATE', 'Demo ROOT file with histograms' )
 
 # Create some histograms, a profile histogram and an ntuple
 hpx    = TH1F('hpx', 'This is the px distribution', 100, -4, 4)
 hpx.SetFillColor(48)
 hpxpy  = TH2F('hpxpy', 'py vs px', 40, -4, 4, 40, -4, 4)
 hprof  = TProfile('hprof', 'Profile of pz versus px', 100, -4, 4, 0, 20)
-ntuple = TNtuple('ntuple', 'Demo ntuple', 'px:py:pz:random:i')
+if not _reflex:
+    ntuple = TNtuple('ntuple', 'Demo ntuple', 'px:py:pz:random:i')
 
 gBenchmark.Start('hsimple')
 
@@ -71,14 +82,15 @@
     hpx.Fill(px)
     hpxpy.Fill(px, py)
     hprof.Fill(px, pz)
-    ntuple.Fill(px, py, pz, rnd, i)
+    if not _reflex:
+        ntuple.Fill(px, py, pz, rnd, i)
 
     # Update display every kUPDATE events
     if i and i%kUPDATE == 0:
         if i == kUPDATE:
             hpx.Draw()
 
-        c1.Modified()
+        c1.Modified(True)
         c1.Update()
 
         if gSystem.ProcessEvents():          # allow user interrupt
@@ -88,9 +100,10 @@
 
 # Save all objects in this file
 hpx.SetFillColor(0)
-hfile.Write()
+if not _reflex:
+    hfile.Write()
 hpx.SetFillColor(48)
-c1.Modified()
+c1.Modified(True)
 c1.Update()
 
 # Note that the file is automatically closed when application terminates

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:44 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:44 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: ignore reflection information
	by-products
Message-ID: <20120402234344.B2B3E46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54165:42cf027dd5ea
Date: 2012-04-02 15:44 -0700
http://bitbucket.org/pypy/pypy/changeset/42cf027dd5ea/

Log:	ignore reflection information by-products

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -19,10 +19,14 @@
 ^pypy/module/cpyext/test/.+\.manifest$
 ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
 ^pypy/module/cppyy/src/.+\.o$
+^pypy/module/cppyy/bench/.+\.so$
+^pypy/module/cppyy/bench/.+\.root$
+^pypy/module/cppyy/bench/.+\.d$
 ^pypy/module/cppyy/src/.+\.errors$
 ^pypy/module/cppyy/test/.+_rflx\.cpp$
 ^pypy/module/cppyy/test/.+\.so$
 ^pypy/module/cppyy/test/.+\.exe$
+^pypy/module/cppyy/test/.+_cint.h$
 ^pypy/doc/.+\.html$
 ^pypy/doc/config/.+\.rst$
 ^pypy/doc/basicblock\.asc$

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:46 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:46 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: o) merge default into branch
Message-ID: <20120402234346.3A42646E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54166:ec9d1aee9f1a
Date: 2012-04-02 15:55 -0700
http://bitbucket.org/pypy/pypy/changeset/ec9d1aee9f1a/

Log:	o) merge default into branch o) fix on my side the problems due to
	changes in array

diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -149,6 +149,22 @@
 exported.  This would give us a one-size-fits-all generic .so file to be
 imported by any application that wants to load .so files :-)
 
+Optimising cpyext (CPython C-API compatibility layer)
+-----------------------------------------------------
+
+A lot of work has gone into PyPy's implementation of CPython's C-API over
+the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However,
+there are still many edges and corner cases where it misbehaves, and it has
+not received any substantial optimisation so far.
+
+The objective of this project is to fix bugs in cpyext and to optimise
+several performance critical parts of it, such as the reference counting
+support and other heavily used C-API functions. The net result would be to
+have CPython extensions run much faster on PyPy than they currently do, or
+to make them work at all if they currently don't. A part of this work would
+be to get cpyext into a shape where it supports running Cython generated
+extensions.
 
 .. _`issue tracker`: http://bugs.pypy.org
 .. _`mailing list`: http://mail.python.org/mailman/listinfo/pypy-dev
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -199,17 +199,11 @@
 The following features (present in some past Stackless version of PyPy)
 are for the time being not supported any more:
 
-* Tasklets and channels (currently ``stackless.py`` seems to import,
-  but you have tasklets on top of coroutines on top of greenlets on
-  top of continulets on top of stacklets, and it's probably not too
-  hard to cut two of these levels by adapting ``stackless.py`` to
-  use directly continulets)
-
 * Coroutines (could be rewritten at app-level)
 
-* Pickling and unpickling continulets (*)
-
-* Continuing execution of a continulet in a different thread (*)
+* Continuing execution of a continulet in a different thread
+  (but if it is "simple enough", you can pickle it and unpickle it
+  in the other thread).
 
 * Automatic unlimited stack (must be emulated__ so far)
 
@@ -217,15 +211,6 @@
 
 .. __: `recursion depth limit`_
 
-(*) Pickling, as well as changing threads, could be implemented by using
-a "soft" stack switching mode again.  We would get either "hard" or
-"soft" switches, similarly to Stackless Python 3rd version: you get a
-"hard" switch (like now) when the C stack contains non-trivial C frames
-to save, and a "soft" switch (like previously) when it contains only
-simple calls from Python to Python.  Soft-switched continulets would
-also consume a bit less RAM, and the switch might be a bit faster too
-(unsure about that; what is the Stackless Python experience?).
-
 
 Recursion depth limit
 +++++++++++++++++++++
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -296,6 +296,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self.frame_trace_action = FrameTraceAction(self)
+        self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
         self.our_magic = default_magic
@@ -467,9 +468,9 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: time2 or rctime take precedence over time
+        # a bit of custom logic: rctime take precedence over time
         # XXX this could probably be done as a "requires" in the config
-        if ('time2' in modules or 'rctime' in modules) and 'time' in modules:
+        if 'rctime' in modules and 'time' in modules:
             modules.remove('time')
 
         if not self.config.objspace.nofaking:
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -154,6 +154,7 @@
         #operationerr.print_detailed_traceback(self.space)
 
     def _convert_exc(self, operr):
+        # Only for the flow object space
         return operr
 
     def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -113,6 +113,12 @@
         from pypy.interpreter.pycode import PyCode
 
         code = self.getcode() # hook for the jit
+        #
+        if (jit.we_are_jitted() and code is self.space._code_of_sys_exc_info
+                                and nargs == 0):
+            from pypy.module.sys.vm import exc_info_direct
+            return exc_info_direct(self.space, frame)
+        #
         fast_natural_arity = code.fast_natural_arity
         if nargs == fast_natural_arity:
             if nargs == 0:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -874,6 +874,12 @@
             fn.add_to_table()
         if gateway.as_classmethod:
             fn = ClassMethod(space.wrap(fn))
+        #
+        from pypy.module.sys.vm import exc_info
+        if code._bltin is exc_info:
+            assert space._code_of_sys_exc_info is None
+            space._code_of_sys_exc_info = code
+        #
         return fn
 
 
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -2,34 +2,39 @@
 from pypy.interpreter import unicodehelper
 from pypy.rlib.rstring import StringBuilder
 
-def parsestr(space, encoding, s, unicode_literals=False):
-    # compiler.transformer.Transformer.decode_literal depends on what 
-    # might seem like minor details of this function -- changes here 
-    # must be reflected there.
+def parsestr(space, encoding, s, unicode_literal=False):
+    """Parses a string or unicode literal, and return a wrapped value.
+
+    If encoding=iso8859-1, the source string is also in this encoding.
+    If encoding=None, the source string is ascii only.
+    In other cases, the source string is in utf-8 encoding.
+
+    When a bytes string is returned, it will be encoded with the
+    original encoding.
+
+    Yes, it's very inefficient.
+    Yes, CPython has very similar code.
+    """
 
     # we use ps as "pointer to s"
     # q is the virtual last char index of the string
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = unicode_literals
 
     # string decoration handling
-    o = ord(quote)
-    isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
-    if isalpha or quote == '_':
-        if quote == 'b' or quote == 'B':
-            ps += 1
-            quote = s[ps]
-            unicode = False
-        elif quote == 'u' or quote == 'U':
-            ps += 1
-            quote = s[ps]
-            unicode = True
-        if quote == 'r' or quote == 'R':
-            ps += 1
-            quote = s[ps]
-            rawmode = True
+    if quote == 'b' or quote == 'B':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = False
+    elif quote == 'u' or quote == 'U':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = True
+    if quote == 'r' or quote == 'R':
+        ps += 1
+        quote = s[ps]
+        rawmode = True
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
                              'Internal error: parser passed unquoted literal')
@@ -46,21 +51,28 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
-    if unicode: # XXX Py_UnicodeFlag is ignored for now
+    if unicode_literal: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None or encoding == "iso-8859-1":
+            # 'unicode_escape' expects latin-1 bytes, string is ready.
             buf = s
             bufp = ps
             bufq = q
             u = None
         else:
-            # "\XX" may become "\u005c\uHHLL" (12 bytes)
+            # String is utf8-encoded, but 'unicode_escape' expects
+            # latin-1; So multibyte sequences must be escaped.
             lis = [] # using a list to assemble the value
             end = q
+            # Worst case: "\XX" may become "\u005c\uHHLL" (12 bytes)
             while ps < end:
                 if s[ps] == '\\':
                     lis.append(s[ps])
                     ps += 1
                     if ord(s[ps]) & 0x80:
+                        # A multibyte sequence will follow, it will be
+                        # escaped like \u1234. To avoid confusion with
+                        # the backslash we just wrote, we emit "\u005c"
+                        # instead.
                         lis.append("u005c")
                 if ord(s[ps]) & 0x80: # XXX inefficient
                     w, ps = decode_utf8(space, s, ps, end, "utf-16-be")
@@ -86,13 +98,11 @@
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
-    # XXX add strchr like interface to rtyper
     assert 0 <= ps <= q
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
             w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
-            #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -27,6 +27,12 @@
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
 
+def boxlonglong(ll):
+    if longlong.is_64_bit:
+        return BoxInt(ll)
+    else:
+        return BoxFloat(ll)
+
 
 class Runner(object):
 
@@ -1623,6 +1629,11 @@
                                      [boxfloat(2.5)], t).value
         assert res == longlong2float.float2longlong(2.5)
 
+        bytes = longlong2float.float2longlong(2.5)
+        res = self.execute_operation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT,
+                                     [boxlonglong(res)], 'float').value
+        assert longlong.getrealfloat(res) == 2.5
+
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -328,6 +328,15 @@
     def produce_into(self, builder, r):
         self.put(builder, [r.choice(builder.intvars)])
 
+class CastLongLongToFloatOperation(AbstractFloatOperation):
+    def produce_into(self, builder, r):
+        if longlong.is_64_bit:
+            self.put(builder, [r.choice(builder.intvars)])
+        else:
+            if not builder.floatvars:
+                raise CannotProduceOperation
+            self.put(builder, [r.choice(builder.floatvars)])
+
 class CastFloatToIntOperation(AbstractFloatOperation):
     def produce_into(self, builder, r):
         if not builder.floatvars:
@@ -450,6 +459,7 @@
 OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT))
 OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT))
 OPERATIONS.append(CastFloatToIntOperation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG))
+OPERATIONS.append(CastLongLongToFloatOperation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT))
 
 OperationBuilder.OPERATIONS = OPERATIONS
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1251,6 +1251,15 @@
         else:
             self.mov(loc0, resloc)
 
+    def genop_convert_longlong_bytes_to_float(self, op, arglocs, resloc):
+        loc0, = arglocs
+        if longlong.is_64_bit:
+            assert isinstance(resloc, RegLoc)
+            assert isinstance(loc0, RegLoc)
+            self.mc.MOVD(resloc, loc0)
+        else:
+            self.mov(loc0, resloc)
+
     def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
         guard_opnum = guard_op.getopnum()
         self.mc.CMP(arglocs[0], imm0)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -773,10 +773,24 @@
             self.Perform(op, [loc0], loc1)
             self.xrm.possibly_free_var(op.getarg(0))
         else:
-            loc0 = self.xrm.loc(op.getarg(0))
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.loc(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
+
+    def consider_convert_longlong_bytes_to_float(self, op):
+        if longlong.is_64_bit:
+            loc0 = self.rm.make_sure_var_in_reg(op.getarg(0))
             loc1 = self.xrm.force_allocate_reg(op.result)
             self.Perform(op, [loc0], loc1)
-            self.xrm.possibly_free_var(op.getarg(0))
+            self.rm.possibly_free_var(op.getarg(0))
+        else:
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.make_sure_var_in_reg(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
 
     def _consider_llong_binop_xx(self, op):
         # must force both arguments into xmm registers, because we don't
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -295,6 +295,7 @@
         return op
 
     rewrite_op_convert_float_bytes_to_longlong = _noop_rewrite
+    rewrite_op_convert_longlong_bytes_to_float = _noop_rewrite
 
     # ----------
     # Various kinds of calls
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -968,20 +968,22 @@
             int_return %i2
         """, transform=True)
 
-    def test_convert_float_bytes_to_int(self):
-        from pypy.rlib.longlong2float import float2longlong
+    def test_convert_float_bytes(self):
+        from pypy.rlib.longlong2float import float2longlong, longlong2float
         def f(x):
-            return float2longlong(x)
+            ll = float2longlong(x)
+            return longlong2float(ll)
         if longlong.is_64_bit:
-            result_var = "%i0"
-            return_op = "int_return"
+            tmp_var = "%i0"
+            result_var = "%f1"
         else:
-            result_var = "%f1"
-            return_op = "float_return"
+            tmp_var = "%f1"
+            result_var = "%f2"
         self.encoding_test(f, [25.0], """
-            convert_float_bytes_to_longlong %%f0 -> %(result_var)s
-            %(return_op)s %(result_var)s
-        """ % {"result_var": result_var, "return_op": return_op})
+            convert_float_bytes_to_longlong %%f0 -> %(tmp_var)s
+            convert_longlong_bytes_to_float %(tmp_var)s -> %(result_var)s
+            float_return %(result_var)s
+        """ % {"result_var": result_var, "tmp_var": tmp_var}, transform=True)
 
 
 def check_force_cast(FROM, TO, operations, value):
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -672,6 +672,11 @@
         a = longlong.getrealfloat(a)
         return longlong2float.float2longlong(a)
 
+    @arguments(LONGLONG_TYPECODE, returns="f")
+    def bhimpl_convert_longlong_bytes_to_float(a):
+        a = longlong2float.longlong2float(a)
+        return longlong.getfloatstorage(a)
+
     # ----------
     # control flow operations
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -224,6 +224,7 @@
                     'float_neg', 'float_abs',
                     'cast_ptr_to_int', 'cast_int_to_ptr',
                     'convert_float_bytes_to_longlong',
+                    'convert_longlong_bytes_to_float',
                     ]:
         exec py.code.Source('''
             @arguments("box")
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -420,6 +420,7 @@
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     'CONVERT_FLOAT_BYTES_TO_LONGLONG/1',
+    'CONVERT_LONGLONG_BYTES_TO_FLOAT/1',
     #
     'INT_LT/2b',
     'INT_LE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,3 +1,4 @@
+import math
 import sys
 
 import py
@@ -15,7 +16,7 @@
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
     isconstant, isvirtual, promote_string, set_param, record_known_class)
-from pypy.rlib.longlong2float import float2longlong
+from pypy.rlib.longlong2float import float2longlong, longlong2float
 from pypy.rlib.rarithmetic import ovfcheck, is_valid_int
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3795,15 +3796,15 @@
         res = self.interp_operations(g, [1])
         assert res == 3
 
-    def test_float2longlong(self):
+    def test_float_bytes(self):
         def f(n):
-            return float2longlong(n)
+            ll = float2longlong(n)
+            return longlong2float(ll)
 
         for x in [2.5, float("nan"), -2.5, float("inf")]:
             # There are tests elsewhere to verify the correctness of this.
-            expected = float2longlong(x)
             res = self.interp_operations(f, [x])
-            assert longlong.getfloatstorage(res) == expected
+            assert res == x or math.isnan(x) and math.isnan(res)
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -11,7 +11,7 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.objectmodel import specialize
+from pypy.rlib.objectmodel import specialize, keepalive_until_here
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
@@ -145,21 +145,27 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(RWBuffer):
-    def __init__(self, data, bytes):
-        self.data = data
-        self.len = bytes
+    def __init__(self, array):
+        self.array = array
 
     def getlength(self):
-        return self.len
+        return self.array.len * self.array.itemsize
 
     def getitem(self, index):
-        return self.data[index]
+        array = self.array
+        data = array._charbuf_start()
+        char = data[index]
+        array._charbuf_stop()
+        return char
 
     def setitem(self, index, char):
-        self.data[index] = char
+        array = self.array
+        data = array._charbuf_start()
+        data[index] = char
+        array._charbuf_stop()
 
     def get_raw_address(self):
-        return self.data
+        return self.array._charbuf_start()
 
 
 def make_array(mytype):
@@ -281,9 +287,10 @@
             oldlen = self.len
             new = len(s) / mytype.bytes
             self.setlen(oldlen + new)
-            cbuf = self.charbuf()
+            cbuf = self._charbuf_start()
             for i in range(len(s)):
                 cbuf[oldlen * mytype.bytes + i] = s[i]
+            self._charbuf_stop()
 
         def fromlist(self, w_lst):
             s = self.len
@@ -313,8 +320,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def charbuf(self):
-            return  rffi.cast(rffi.CCHARP, self.buffer)
+        def _charbuf_start(self):
+            return rffi.cast(rffi.CCHARP, self.buffer)
+
+        def _charbuf_stop(self):
+            keepalive_until_here(self)
 
         def w_getitem(self, space, idx):
             item = self.buffer[idx]
@@ -533,8 +543,10 @@
         self.fromstring(space.str_w(w_s))
 
     def array_tostring__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
+        self._charbuf_stop()
+        return self.space.wrap(s)
 
     def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
         if not isinstance(w_f, W_File):
@@ -616,8 +628,7 @@
     # Misc methods
 
     def buffer__Array(space, self):
-        b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes)
-        return space.wrap(b)
+        return space.wrap(ArrayBuffer(self))
 
     def array_buffer_info__Array(space, self):
         w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer))
@@ -652,7 +663,7 @@
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
-        bytes = self.charbuf()
+        bytes = self._charbuf_start()
         tmp = [bytes[0]] * mytype.bytes
         for start in range(0, self.len * mytype.bytes, mytype.bytes):
             stop = start + mytype.bytes - 1
@@ -660,6 +671,7 @@
                 tmp[i] = bytes[start + i]
             for i in range(mytype.bytes):
                 bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
 
     def repr__Array(space, self):
         if self.len == 0:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -433,7 +433,25 @@
         a = self.array('h', 'Hi')
         buf = buffer(a)
         assert buf[1] == 'i'
-        #raises(TypeError, buf.__setitem__, 1, 'o')
+
+    def test_buffer_write(self):
+        a = self.array('c', 'hello')
+        buf = buffer(a)
+        print repr(buf)
+        try:
+            buf[3] = 'L'
+        except TypeError:
+            skip("buffer(array) returns a read-only buffer on CPython")
+        assert a.tostring() == 'helLo'
+
+    def test_buffer_keepalive(self):
+        buf = buffer(self.array('c', 'text'))
+        assert buf[2] == 'x'
+        #
+        a = self.array('c', 'foobarbaz')
+        buf = buffer(a)
+        a.fromstring('some extra text')
+        assert buf[:] == 'foobarbazsome extra text'
 
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -4,6 +4,8 @@
     PyObjectFields, PyObject)
 from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
+from pypy.interpreter.error import OperationError
+from pypy.module.array.interp_array import ArrayBuffer
 
 
 PyBufferObjectStruct = lltype.ForwardReference()
@@ -43,10 +45,15 @@
 
     if isinstance(w_obj, StringBuffer):
         py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.as_str()))
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
+        py_buf.c_b_size = w_obj.getlength()
+    elif isinstance(w_obj, ArrayBuffer):
+        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.data)
         py_buf.c_b_size = w_obj.getlength()
     else:
-        raise Exception("Fail fail fail fail fail")
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "buffer flavor not supported"))
 
 
 def buffer_realize(space, py_obj):
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -48,3 +48,17 @@
             ])
         b = module.buffer_new()
         raises(AttributeError, getattr, b, 'x')
+
+    def test_array_buffer(self):
+        module = self.import_extension('foo', [
+            ("roundtrip", "METH_O",
+             """
+                 PyBufferObject *buf = (PyBufferObject *)args;
+                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
+             """),
+            ])
+        import array
+        a = array.array('c', 'text')
+        b = buffer(a)
+        assert module.roundtrip(b) == 'text'
+        
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -99,6 +99,8 @@
         ("exp2", "exp2"),
         ("expm1", "expm1"),
         ("fabs", "fabs"),
+        ("fmax", "fmax"),
+        ("fmin", "fmin"),
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
@@ -122,12 +124,14 @@
         ("sinh", "sinh"),
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
+        ('square', 'square'),
         ("tan", "tan"),
         ("tanh", "tanh"),
         ('bitwise_and', 'bitwise_and'),
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
         ('isneginf', 'isneginf'),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,9 +3,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype, support, loop
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
+
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
@@ -28,7 +30,7 @@
         return self.identity
 
     def descr_call(self, space, __args__):
-	from interp_numarray import BaseArray
+        from interp_numarray import BaseArray
         args_w, kwds_w = __args__.unpack()
         # it occurs to me that we don't support any datatypes that
         # require casting, change it later when we do
@@ -179,7 +181,7 @@
                 elif out.shape != shape:
                     raise operationerrfmt(space.w_ValueError,
                         'output parameter shape mismatch, expecting [%s]' +
-                        ' , got [%s]', 
+                        ' , got [%s]',
                         ",".join([str(x) for x in shape]),
                         ",".join([str(x) for x in out.shape]),
                         )
@@ -204,7 +206,7 @@
         else:
             arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
             val = loop.compute(arr)
-        return val 
+        return val
 
     def do_axis_reduce(self, obj, dtype, axis, result):
         from pypy.module.micronumpy.interp_numarray import AxisReduce
@@ -253,7 +255,7 @@
         if isinstance(w_obj, Scalar):
             arr = self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))
             if isinstance(out,Scalar):
-                out.value=arr
+                out.value = arr
             elif isinstance(out, BaseArray):
                 out.fill(space, arr)
             else:
@@ -265,7 +267,7 @@
             if not broadcast_shape or broadcast_shape != out.shape:
                 raise operationerrfmt(space.w_ValueError,
                     'output parameter shape mismatch, could not broadcast [%s]' +
-                    ' to [%s]', 
+                    ' to [%s]',
                     ",".join([str(x) for x in w_obj.shape]),
                     ",".join([str(x) for x in out.shape]),
                     )
@@ -292,10 +294,11 @@
         self.func = func
         self.comparison_func = comparison_func
 
+    @jit.unroll_safe
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
             convert_to_array, Scalar, shape_agreement, BaseArray)
-        if len(args_w)>2:
+        if len(args_w) > 2:
             [w_lhs, w_rhs, w_out] = args_w
         else:
             [w_lhs, w_rhs] = args_w
@@ -326,7 +329,7 @@
                 w_rhs.value.convert_to(calc_dtype)
             )
             if isinstance(out,Scalar):
-                out.value=arr
+                out.value = arr
             elif isinstance(out, BaseArray):
                 out.fill(space, arr)
             else:
@@ -337,7 +340,7 @@
         if out and out.shape != shape_agreement(space, new_shape, out.shape):
             raise operationerrfmt(space.w_ValueError,
                 'output parameter shape mismatch, could not broadcast [%s]' +
-                ' to [%s]', 
+                ' to [%s]',
                 ",".join([str(x) for x in new_shape]),
                 ",".join([str(x) for x in out.shape]),
                 )
@@ -347,7 +350,6 @@
         w_lhs.add_invalidates(w_res)
         w_rhs.add_invalidates(w_res)
         if out:
-            #out.add_invalidates(w_res) #causes a recursion loop
             w_res.get_concrete()
         return w_res
 
@@ -539,6 +541,8 @@
             ("reciprocal", "reciprocal", 1),
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
+            ("fmax", "fmax", 2, {"promote_to_float": True}),
+            ("fmin", "fmin", 2, {"promote_to_float": True}),
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
@@ -547,6 +551,7 @@
             ("expm1", "expm1", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
+            ('square', 'square', 1, {'promote_to_float': True}),
 
             ("sin", "sin", 1, {"promote_to_float": True}),
             ("cos", "cos", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,6 +1,7 @@
 from pypy.rlib import jit
 from pypy.interpreter.error import OperationError
 
+ at jit.look_inside_iff(lambda chunks: jit.isconstant(len(chunks)))
 def enumerate_chunks(chunks):
     result = []
     i = -1
@@ -85,9 +86,9 @@
         space.isinstance_w(w_item_or_slice, space.w_slice)):
         raise OperationError(space.w_IndexError,
                              space.wrap('unsupported iterator index'))
-            
+
     start, stop, step, lngth = space.decode_index4(w_item_or_slice, size)
-        
+
     coords = [0] * len(shape)
     i = start
     if order == 'C':
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,5 +1,9 @@
+from pypy.rlib import jit
+
+
+ at jit.unroll_safe
 def product(s):
     i = 1
     for x in s:
         i *= x
-    return i
\ No newline at end of file
+    return i
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -135,6 +135,38 @@
         assert fabs(float('-inf')) == float('inf')
         assert isnan(fabs(float('nan')))
 
+    def test_fmax(self):
+        from _numpypy import fmax
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmax(a, [ninf]*5) == a).all()
+        assert (fmax(a, [inf]*5) == [inf]*5).all()
+        assert (fmax(a, [1]*5) == [1, 1, 1, 5, inf]).all()
+        assert math.isnan(fmax(nan, 0))
+        assert math.isnan(fmax(0, nan))
+        assert math.isnan(fmax(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+
+    def test_fmin(self):
+        from _numpypy import fmin
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmin(a, [ninf]*5) == [ninf]*5).all()
+        assert (fmin(a, [inf]*5) == a).all()
+        assert (fmin(a, [1]*5) == [ninf, -5, 0, 1, 1]).all()
+        assert math.isnan(fmin(nan, 0))
+        assert math.isnan(fmin(0, nan))
+        assert math.isnan(fmin(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+
     def test_fmod(self):
         from _numpypy import fmod
         import math
@@ -455,6 +487,19 @@
         assert math.isnan(sqrt(-1))
         assert math.isnan(sqrt(nan))
 
+    def test_square(self):
+        import math
+        from _numpypy import square
+
+        nan, inf, ninf = float("nan"), float("inf"), float("-inf")
+
+        assert math.isnan(square(nan))
+        assert math.isinf(square(inf))
+        assert math.isinf(square(ninf))
+        assert square(ninf) > 0
+        assert [square(x) for x in range(-5, 5)] == [x*x for x in range(-5, 5)]
+        assert math.isinf(square(1e300))
+
     def test_radians(self):
         import math
         from _numpypy import radians, array
@@ -546,10 +591,11 @@
         raises(TypeError, 'array([1.0]) & 1')
 
     def test_unary_bitops(self):
-        from _numpypy import bitwise_not, array
+        from _numpypy import bitwise_not, invert, array
         a = array([1, 2, 3, 4])
         assert (~a == [-2, -3, -4, -5]).all()
         assert (bitwise_not(a) == ~a).all()
+        assert (invert(a) == ~a).all()
 
     def test_comparisons(self):
         import operator
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -631,6 +631,22 @@
         return math.fabs(v)
 
     @simple_binary_op
+    def fmax(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return max(v1, v2)
+
+    @simple_binary_op
+    def fmin(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return min(v1, v2)
+
+    @simple_binary_op
     def fmod(self, v1, v2):
         try:
             return math.fmod(v1, v2)
@@ -741,6 +757,10 @@
         except ValueError:
             return rfloat.NAN
 
+    @simple_unary_op
+    def square(self, v):
+        return v*v
+
     @raw_unary_op
     def isnan(self, v):
         return rfloat.isnan(v)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -351,3 +351,23 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(log.loops_by_id("eval")) == 1
+
+    def test_sys_exc_info(self):
+        def main():
+            i = 1
+            lst = [i]
+            while i < 1000:
+                try:
+                    return lst[i]
+                except:
+                    e = sys.exc_info()[1]    # ID: exc_info
+                    if not isinstance(e, IndexError):
+                        raise
+                i += 1
+            return 42
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("exc_info")) == 1
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -595,3 +595,121 @@
         assert len(frames) == 1
         _, other_frame = frames.popitem()
         assert other_frame.f_code.co_name in ('other_thread', '?')
+
+
+class AppTestSysExcInfoDirect:
+
+    def setup_method(self, meth):
+        self.seen = []
+        from pypy.module.sys import vm
+        def exc_info_with_tb(*args):
+            self.seen.append("n")     # not optimized
+            return self.old[0](*args)
+        def exc_info_without_tb(*args):
+            self.seen.append("y")     # optimized
+            return self.old[1](*args)
+        self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
+        vm.exc_info_with_tb = exc_info_with_tb
+        vm.exc_info_without_tb = exc_info_without_tb
+        #
+        from pypy.rlib import jit
+        self.old2 = [jit.we_are_jitted]
+        jit.we_are_jitted = lambda: True
+
+    def teardown_method(self, meth):
+        from pypy.module.sys import vm
+        from pypy.rlib import jit
+        vm.exc_info_with_tb = self.old[0]
+        vm.exc_info_without_tb = self.old[1]
+        jit.we_are_jitted = self.old2[0]
+        #
+        assert ''.join(self.seen) == meth.expected
+
+    def test_returns_none(self):
+        import sys
+        assert sys.exc_info() == (None, None, None)
+        assert sys.exc_info()[0] is None
+        assert sys.exc_info()[1] is None
+        assert sys.exc_info()[2] is None
+        assert sys.exc_info()[:2] == (None, None)
+        assert sys.exc_info()[:3] == (None, None, None)
+        assert sys.exc_info()[0:2] == (None, None)
+        assert sys.exc_info()[2:4] == (None,)
+    test_returns_none.expected = 'nnnnnnnn'
+
+    def test_returns_subscr(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert sys.exc_info()[0] is KeyError  # y
+            assert sys.exc_info()[1] is e         # y
+            assert sys.exc_info()[2] is not None  # n
+            assert sys.exc_info()[-3] is KeyError # y
+            assert sys.exc_info()[-2] is e        # y
+            assert sys.exc_info()[-1] is not None # n
+    test_returns_subscr.expected = 'yynyyn'
+
+    def test_returns_slice_2(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                  # n
+            assert sys.exc_info()[:0] == ()       # y
+            assert sys.exc_info()[:1] == foo[:1]  # y
+            assert sys.exc_info()[:2] == foo[:2]  # y
+            assert sys.exc_info()[:3] == foo      # n
+            assert sys.exc_info()[:4] == foo      # n
+            assert sys.exc_info()[:-1] == foo[:2] # y
+            assert sys.exc_info()[:-2] == foo[:1] # y
+            assert sys.exc_info()[:-3] == ()      # y
+    test_returns_slice_2.expected = 'nyyynnyyy'
+
+    def test_returns_slice_3(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                   # n
+            assert sys.exc_info()[2:2] == ()       # y
+            assert sys.exc_info()[0:1] == foo[:1]  # y
+            assert sys.exc_info()[1:2] == foo[1:2] # y
+            assert sys.exc_info()[0:3] == foo      # n
+            assert sys.exc_info()[2:4] == foo[2:]  # n
+            assert sys.exc_info()[0:-1] == foo[:2] # y
+            assert sys.exc_info()[0:-2] == foo[:1] # y
+            assert sys.exc_info()[5:-3] == ()      # y
+    test_returns_slice_3.expected = 'nyyynnyyy'
+
+    def test_strange_invocation(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            a = []; k = {}
+            assert sys.exc_info(*a)[:0] == ()
+            assert sys.exc_info(**k)[:0] == ()
+    test_strange_invocation.expected = 'nn'
+
+    def test_call_in_subfunction(self):
+        import sys
+        def g():
+            # this case is not optimized, because we need to search the
+            # frame chain.  it's probably not worth the complications
+            return sys.exc_info()[1]
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert g() is e
+    test_call_in_subfunction.expected = 'n'
+
+
+class AppTestSysExcInfoDirectCallMethod(AppTestSysExcInfoDirect):
+    def setup_class(cls):
+        cls.space = gettestobjspace(**{"objspace.opcodes.CALL_METHOD": True})
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -89,6 +89,9 @@
     """Return the (type, value, traceback) of the most recent exception
 caught by an except clause in the current stack frame or in an older stack
 frame."""
+    return exc_info_with_tb(space)    # indirection for the tests
+
+def exc_info_with_tb(space):
     operror = space.getexecutioncontext().sys_exc_info()
     if operror is None:
         return space.newtuple([space.w_None,space.w_None,space.w_None])
@@ -96,6 +99,59 @@
         return space.newtuple([operror.w_type, operror.get_w_value(space),
                                space.wrap(operror.get_traceback())])
 
+def exc_info_without_tb(space, frame):
+    operror = frame.last_exception
+    return space.newtuple([operror.w_type, operror.get_w_value(space),
+                           space.w_None])
+
+def exc_info_direct(space, frame):
+    from pypy.tool import stdlib_opcode
+    # In order to make the JIT happy, we try to return (exc, val, None)
+    # instead of (exc, val, tb).  We can do that only if we recognize
+    # the following pattern in the bytecode:
+    #       CALL_FUNCTION/CALL_METHOD         <-- invoking me
+    #       LOAD_CONST 0, 1, -2 or -3
+    #       BINARY_SUBSCR
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST <=2
+    #       SLICE_2
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST any integer
+    #       LOAD_CONST <=2
+    #       SLICE_3
+    need_all_three_args = True
+    co = frame.getcode().co_code
+    p = frame.last_instr
+    if (ord(co[p]) == stdlib_opcode.CALL_FUNCTION or
+        ord(co[p]) == stdlib_opcode.CALL_METHOD):
+        if ord(co[p+3]) == stdlib_opcode.LOAD_CONST:
+            lo = ord(co[p+4])
+            hi = ord(co[p+5])
+            w_constant = frame.getconstant_w((hi * 256) | lo)
+            if space.isinstance_w(w_constant, space.w_int):
+                constant = space.int_w(w_constant)
+                if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+                    if -3 <= constant <= 1 and constant != -1:
+                        need_all_three_args = False
+                elif ord(co[p+6]) == stdlib_opcode.SLICE+2:
+                    if constant <= 2:
+                        need_all_three_args = False
+                elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
+                      ord(co[p+9]) == stdlib_opcode.SLICE+3):
+                    lo = ord(co[p+7])
+                    hi = ord(co[p+8])
+                    w_constant = frame.getconstant_w((hi * 256) | lo)
+                    if space.isinstance_w(w_constant, space.w_int):
+                        if space.int_w(w_constant) <= 2:
+                            need_all_three_args = False
+    #
+    if need_all_three_args or frame.last_exception is None or frame.hide():
+        return exc_info_with_tb(space)
+    else:
+        return exc_info_without_tb(space, frame)
+
 def exc_clear(space):
     """Clear global information on the current exception.  Subsequent calls
 to exc_info() will return (None,None,None) until another exception is
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -21,7 +21,7 @@
 FLOAT_ARRAY_PTR = lltype.Ptr(lltype.Array(rffi.FLOAT))
 
 # these definitions are used only in tests, when not translated
-def longlong2float_emulator(llval):
+def longlong2float(llval):
     with lltype.scoped_alloc(DOUBLE_ARRAY_PTR.TO, 1) as d_array:
         ll_array = rffi.cast(LONGLONG_ARRAY_PTR, d_array)
         ll_array[0] = llval
@@ -51,12 +51,6 @@
 
 eci = ExternalCompilationInfo(includes=['string.h', 'assert.h'],
                               post_include_bits=["""
-static double pypy__longlong2float(long long x) {
-    double dd;
-    assert(sizeof(double) == 8 && sizeof(long long) == 8);
-    memcpy(&dd, &x, 8);
-    return dd;
-}
 static float pypy__uint2singlefloat(unsigned int x) {
     float ff;
     assert(sizeof(float) == 4 && sizeof(unsigned int) == 4);
@@ -71,12 +65,6 @@
 }
 """])
 
-longlong2float = rffi.llexternal(
-    "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
-    _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True,
-    oo_primitive="pypy__longlong2float")
-
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
@@ -99,4 +87,17 @@
 
     def specialize_call(self, hop):
         [v_float] = hop.inputargs(lltype.Float)
-        return hop.genop("convert_float_bytes_to_longlong", [v_float], resulttype=hop.r_result)
+        hop.exception_cannot_occur()
+        return hop.genop("convert_float_bytes_to_longlong", [v_float], resulttype=lltype.SignedLongLong)
+
+class LongLong2FloatEntry(ExtRegistryEntry):
+    _about_ = longlong2float
+
+    def compute_result_annotation(self, s_longlong):
+        assert annmodel.SomeInteger(knowntype=r_int64).contains(s_longlong)
+        return annmodel.SomeFloat()
+
+    def specialize_call(self, hop):
+        [v_longlong] = hop.inputargs(lltype.SignedLongLong)
+        hop.exception_cannot_occur()
+        return hop.genop("convert_longlong_bytes_to_float", [v_longlong], resulttype=lltype.Float)
diff --git a/pypy/rlib/test/test_longlong2float.py b/pypy/rlib/test/test_longlong2float.py
--- a/pypy/rlib/test/test_longlong2float.py
+++ b/pypy/rlib/test/test_longlong2float.py
@@ -2,6 +2,7 @@
 from pypy.rlib.longlong2float import longlong2float, float2longlong
 from pypy.rlib.longlong2float import uint2singlefloat, singlefloat2uint
 from pypy.rlib.rarithmetic import r_singlefloat
+from pypy.rpython.test.test_llinterp import interpret
 
 
 def fn(f1):
@@ -31,6 +32,18 @@
         res = fn2(x)
         assert repr(res) == repr(x)
 
+def test_interpreted():
+    def f(f1):
+        try:
+            ll = float2longlong(f1)
+            return longlong2float(ll)
+        except Exception:
+            return 500
+
+    for x in enum_floats():
+        res = interpret(f, [x])
+        assert repr(res) == repr(x)
+
 # ____________________________________________________________
 
 def fnsingle(f1):
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -350,6 +350,7 @@
     'truncate_longlong_to_int':LLOp(canfold=True),
     'force_cast':           LLOp(sideeffects=False),    # only for rffi.cast()
     'convert_float_bytes_to_longlong': LLOp(canfold=True),
+    'convert_longlong_bytes_to_float': LLOp(canfold=True),
 
     # __________ pointer operations __________
 
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -431,6 +431,10 @@
     from pypy.rlib.longlong2float import float2longlong
     return float2longlong(a)
 
+def op_convert_longlong_bytes_to_float(a):
+    from pypy.rlib.longlong2float import longlong2float
+    return longlong2float(a)
+
 
 def op_unichar_eq(x, y):
     assert isinstance(x, unicode) and len(x) == 1
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -765,7 +765,11 @@
     def _ll_stringslice(s1, start, stop):
         lgt = stop - start
         assert start >= 0
-        assert lgt >= 0
+        # If start > stop, return a empty string. This can happen if the start
+        # is greater than the length of the string. Use < instead of <= to avoid
+        # creating another path for the JIT when start == stop.
+        if lgt < 0:
+            return s1.empty()
         newstr = s1.malloc(lgt)
         s1.copy_contents(s1, newstr, start, 0, lgt)
         return newstr
diff --git a/pypy/rpython/ootypesystem/rstr.py b/pypy/rpython/ootypesystem/rstr.py
--- a/pypy/rpython/ootypesystem/rstr.py
+++ b/pypy/rpython/ootypesystem/rstr.py
@@ -222,6 +222,10 @@
         length = s.ll_strlen()
         if stop > length:
             stop = length
+        # If start > stop, return a empty string. This can happen if the start
+        # is greater than the length of the string.
+        if start > stop:
+            start = stop
         return s.ll_substring(start, stop-start)
 
     def ll_stringslice_minusone(s):
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -477,7 +477,11 @@
             s1 = s[:3]
             s2 = s[3:]
             s3 = s[3:10]
-            return s1+s2 == s and s2+s1 == const('lohel') and s1+s3 == s
+            s4 = s[42:44]
+            return (s1+s2 == s and
+                    s2+s1 == const('lohel') and
+                    s1+s3 == s and
+                    s4 == const(''))
         res = self.interpret(fn, [0])
         assert res
 
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -38,7 +38,7 @@
 closed_heads.reverse()
 
 for head, branch in closed_heads:
-    print '\t', branch
+    print '\t', head, '\t', branch
 print
 print 'The branches listed above will be merged to "closed-branches".'
 print 'You need to run this script in a clean working copy where you'
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -7,10 +7,17 @@
 from pypy import conftest
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.translator.platform import platform as compiler
+from pypy.rlib.rarithmetic import is_emulated_long
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib.entrypoint import entrypoint, secondary_entrypoints
 from pypy.rpython.lltypesystem.lloperation import llop
 
+_MSVC = compiler.name == "msvc"
+_MINGW = compiler.name == "mingw32"
+_WIN32 = _MSVC or _MINGW
+_WIN64 = _WIN32 and is_emulated_long
+# XXX get rid of 'is_emulated_long' and have a real config here.
+
 class AbstractTestAsmGCRoot:
     # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved
     # instructions:
@@ -18,8 +25,8 @@
 
     @classmethod
     def make_config(cls):
-        if compiler.name == "msvc":
-            py.test.skip("all asmgcroot tests disabled for MSVC")
+        if _MSVC and _WIN64:
+            py.test.skip("all asmgcroot tests disabled for MSVC X64")
         from pypy.config.pypyoption import get_pypy_config
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
diff --git a/pypy/translator/c/src/float.h b/pypy/translator/c/src/float.h
--- a/pypy/translator/c/src/float.h
+++ b/pypy/translator/c/src/float.h
@@ -43,5 +43,6 @@
 #define OP_CAST_FLOAT_TO_LONGLONG(x,r) r = (long long)(x)
 #define OP_CAST_FLOAT_TO_ULONGLONG(x,r) r = (unsigned long long)(x)
 #define OP_CONVERT_FLOAT_BYTES_TO_LONGLONG(x,r) memcpy(&r, &x, sizeof(double))
+#define OP_CONVERT_LONGLONG_BYTES_TO_FLOAT(x,r) memcpy(&r, &x, sizeof(long long))
 #endif
 
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -243,4 +243,5 @@
     'force_cast':               [PushAllArgs, CastPrimitive, StoreResult],
 
     'convert_float_bytes_to_longlong': jvm.PYPYDOUBLEBYTESTOLONG,
+    'convert_longlong_bytes_to_float': jvm.PYPYLONGBYTESTODOUBLE,
 })
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -942,6 +942,7 @@
 PYPYULONGTODOUBLE =     Method.s(jPyPy, 'ulong_to_double', (jLong,), jDouble)
 PYPYLONGBITWISENEGATE = Method.v(jPyPy, 'long_bitwise_negate', (jLong,), jLong)
 PYPYDOUBLEBYTESTOLONG = Method.v(jPyPy, 'pypy__float2longlong', (jDouble,), jLong)
+PYPYLONGBYTESTODOUBLE = Method.v(jPyPy, 'pypy__longlong2float', (jLong,), jDouble)
 PYPYSTRTOINT =          Method.v(jPyPy, 'str_to_int', (jString,), jInt)
 PYPYSTRTOUINT =         Method.v(jPyPy, 'str_to_uint', (jString,), jInt)
 PYPYSTRTOLONG =         Method.v(jPyPy, 'str_to_long', (jString,), jLong)

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:48 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:48 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: merge default
Message-ID: <20120402234348.8790046E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54167:748a53a1cc24
Date: 2012-04-02 15:59 -0700
http://bitbucket.org/pypy/pypy/changeset/748a53a1cc24/

Log:	merge default

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -968,8 +968,7 @@
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
+        return NotImplemented    # note that this doesn't work on CPython 2.2
 
     __radd__ = __add__
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -167,6 +167,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1797,6 +1797,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1813,6 +1814,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -432,7 +432,8 @@
                     raise _ssl_seterror(self.space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return self.space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return self.space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                              length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,7 +2,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
     PyObjectFields, PyObject)
-from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
+from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
 from pypy.interpreter.error import OperationError
 from pypy.module.array.interp_array import ArrayBuffer
@@ -43,13 +43,18 @@
         py_buf.c_b_offset = w_obj.offset
         w_obj = w_obj.buffer
 
+    # If w_obj already allocated a fixed buffer, use it, and keep a
+    # reference to w_obj.
+    # Otherwise, b_base stays NULL, and we own the b_ptr.
+
     if isinstance(w_obj, StringBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
+        py_buf.c_b_base = lltype.nullptr(PyObject.TO)
         py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
         py_buf.c_b_size = w_obj.getlength()
     elif isinstance(w_obj, ArrayBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.data)
+        w_base = w_obj.array
+        py_buf.c_b_base = make_ref(space, w_base)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start())
         py_buf.c_b_size = w_obj.getlength()
     else:
         raise OperationError(space.w_NotImplementedError, space.wrap(
@@ -60,14 +65,16 @@
     """
     Creates the buffer in the PyPy interpreter from a cpyext representation.
     """
-    raise Exception("realize fail fail fail")
-
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "Don't know how to realize a buffer"))
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
-    Py_DecRef(space, py_buf.c_b_base)
-    rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
+    if py_buf.c_b_base:
+        Py_DecRef(space, py_buf.c_b_base)
+    else:
+        rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -315,3 +316,65 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,14 +167,16 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -183,7 +185,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
@@ -414,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -425,6 +429,36 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
+
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -307,6 +307,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -624,7 +626,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -673,6 +673,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -1261,6 +1265,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1386,6 +1415,7 @@
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -107,6 +107,10 @@
         arr.compute_first_step(self, f)
         return f
 
+    def debug_repr(self):
+        # should be overridden, but in case it isn't, provide a default
+        return str(self)
+
 class ConcreteSignature(Signature):
     _immutable_fields_ = ['dtype']
 
@@ -207,7 +211,7 @@
     def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        transforms = transforms + [ViewTransform(arr.chunks)]
+        transforms = [ViewTransform(arr.chunks)] + transforms 
         self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
@@ -215,6 +219,9 @@
         assert isinstance(arr, VirtualSlice)
         return self.child.eval(frame, arr.child)
 
+    def debug_repr(self):
+        return 'VirtualSlice(%s)' % self.child.debug_repr()
+
 class Call1(Signature):
     _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
 
@@ -270,7 +277,7 @@
         from pypy.module.micronumpy.interp_numarray import Call1
 
         assert isinstance(arr, Call1)
-        vtransforms = transforms + [BroadcastTransform(arr.values.shape)]
+        vtransforms = [BroadcastTransform(arr.values.shape)] + transforms
         self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
         self.res._create_iter(iterlist, arraylist, arr.res, transforms)
 
@@ -348,7 +355,7 @@
         from pypy.module.micronumpy.interp_numarray import ResultArray
 
         assert isinstance(arr, ResultArray)
-        rtransforms = transforms + [BroadcastTransform(arr.left.shape)]
+        rtransforms = [BroadcastTransform(arr.left.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -375,7 +382,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
@@ -388,7 +395,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -401,8 +408,8 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -424,7 +431,7 @@
         frame.cur_value = self.binfunc(self.calc_dtype, frame.cur_value, rval)
 
     def debug_repr(self):
-        return 'ReduceSig(%s)' % (self.name, self.right.debug_repr())
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
 
 class SliceloopSignature(Call2):
     def eval(self, frame, arr):
@@ -448,7 +455,7 @@
         from pypy.module.micronumpy.interp_numarray import SliceArray
 
         assert isinstance(arr, SliceArray)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -395,11 +395,19 @@
         assert a[3] == 0.
 
     def test_newaxis(self):
-        from _numpypy import array
+        import math
+        from _numpypy import array, cos, zeros
         from numpypy.core.numeric import newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
+        a = array(range(3))
+        b = array([1, 3])
+        expected = zeros((3, 2))
+        for x in range(3):
+            for y in range(2):
+                expected[x, y] = math.cos(a[x]) * math.cos(b[y])
+        assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):
         from _numpypy import array
@@ -1338,6 +1346,10 @@
         dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0)
         assert str(dims_disagree.value) == \
             "array dimensions must agree except for axis being concatenated"
+        a = array([1, 2, 3, 4, 5, 6])
+        a = (a + a)[::2]
+        b = concatenate((a[:3], a[-3:]))
+        assert (b == [2, 6, 10, 2, 6, 10]).all()
 
     def test_std(self):
         from _numpypy import array
@@ -1387,6 +1399,16 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat, array
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -212,7 +212,7 @@
             i19 = int_add(i12, 1)
             setfield_gc(p9, i19, descr=<FieldS .*W_AbstractSeqIterObject.inst_index .*>)
             guard_nonnull_class(p17, 146982464, descr=...)
-            i21 = getfield_gc(p17, descr=<FieldS .*W_ArrayTypei.inst_len .*>)
+            i21 = getfield_gc(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
             i24 = getfield_gc(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -198,3 +198,37 @@
             i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>)
             guard_value(i49, 1, descr=...)
             ''')
+
+    def test_remove_duplicate_method_calls(self):
+        def main(n):
+            lst = []
+            for i in range(n):
+                s = 'Hello %d' % i
+                t = s.lower()   # ID: callone
+                u = s.lower()   # ID: calltwo
+                lst.append(t)
+                lst.append(u)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '''
+            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr . r EF=3>)
+            guard_no_exception(descr=...)
+            ''')
+        loop.match_by_id('calltwo', '')    # nothing
+
+    def test_move_method_call_out_of_loop(self):
+        def main(n):
+            lst = []
+            s = 'Hello %d' % n
+            for i in range(n):
+                t = s.lower()   # ID: callone
+                lst.append(t)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '')    # nothing
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -600,30 +600,33 @@
 class AppTestSysExcInfoDirect:
 
     def setup_method(self, meth):
-        self.seen = []
-        from pypy.module.sys import vm
-        def exc_info_with_tb(*args):
-            self.seen.append("n")     # not optimized
-            return self.old[0](*args)
-        def exc_info_without_tb(*args):
-            self.seen.append("y")     # optimized
-            return self.old[1](*args)
-        self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
-        vm.exc_info_with_tb = exc_info_with_tb
-        vm.exc_info_without_tb = exc_info_without_tb
-        #
-        from pypy.rlib import jit
-        self.old2 = [jit.we_are_jitted]
-        jit.we_are_jitted = lambda: True
+        self.checking = not option.runappdirect
+        if self.checking:
+            self.seen = []
+            from pypy.module.sys import vm
+            def exc_info_with_tb(*args):
+                self.seen.append("n")     # not optimized
+                return self.old[0](*args)
+            def exc_info_without_tb(*args):
+                self.seen.append("y")     # optimized
+                return self.old[1](*args)
+            self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
+            vm.exc_info_with_tb = exc_info_with_tb
+            vm.exc_info_without_tb = exc_info_without_tb
+            #
+            from pypy.rlib import jit
+            self.old2 = [jit.we_are_jitted]
+            jit.we_are_jitted = lambda: True
 
     def teardown_method(self, meth):
-        from pypy.module.sys import vm
-        from pypy.rlib import jit
-        vm.exc_info_with_tb = self.old[0]
-        vm.exc_info_without_tb = self.old[1]
-        jit.we_are_jitted = self.old2[0]
-        #
-        assert ''.join(self.seen) == meth.expected
+        if self.checking:
+            from pypy.module.sys import vm
+            from pypy.rlib import jit
+            vm.exc_info_with_tb = self.old[0]
+            vm.exc_info_without_tb = self.old[1]
+            jit.we_are_jitted = self.old2[0]
+            #
+            assert ''.join(self.seen) == meth.expected
 
     def test_returns_none(self):
         import sys
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -44,3 +44,9 @@
     assert type(dt.microsecond) is int
 
     copy.copy(dt)
+
+def test_radd():
+    class X(object):
+        def __radd__(self, other):
+            return "radd"
+    assert datetime.date(10, 10, 10) + X() == "radd"
diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py
--- a/pypy/module/thread/test/test_ll_thread.py
+++ b/pypy/module/thread/test/test_ll_thread.py
@@ -66,7 +66,6 @@
     def test_gc_locking(self):
         import time
         from pypy.rlib.objectmodel import invoke_around_extcall
-        from pypy.rlib.objectmodel import we_are_translated
         from pypy.rlib.debug import ll_assert
 
         class State:
@@ -129,8 +128,6 @@
             state.finished = 0
             # the next line installs before_extcall() and after_extcall()
             # to be called automatically around external function calls.
-            # When not translated it does not work around time.sleep(),
-            # so we have to call them manually for this test.
             invoke_around_extcall(before_extcall, after_extcall)
 
             g(10, 1)
@@ -142,13 +139,9 @@
                 willing_to_wait_more -= 1
                 done = len(state.answers) == expected
 
-                if not we_are_translated(): before_extcall()
                 time.sleep(0.01)
-                if not we_are_translated(): after_extcall()
 
-            if not we_are_translated(): before_extcall()
             time.sleep(0.1)
-            if not we_are_translated(): after_extcall()
 
             return len(state.answers)
 
@@ -160,12 +153,11 @@
         answers = fn()
         assert answers == expected
 
-class TestRunDirectly(AbstractThreadTests):
-    def getcompiled(self, f, argtypes):
-        return f
-
-    def test_start_new_thread(self):
-        py.test.skip("deadlocks occasionally -- why???")
+#class TestRunDirectly(AbstractThreadTests):
+#    def getcompiled(self, f, argtypes):
+#        return f
+# These are disabled because they crash occasionally for bad reasons
+# related to the fact that ll2ctypes is not at all thread-safe
 
 class TestUsingBoehm(AbstractThreadTests):
     gcpolicy = 'boehm'
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -43,6 +43,13 @@
     return w_eq
 type_eq._annspecialcase_ = 'specialize:memo'
 
+def list_iter(space):
+    "Utility that returns the app-level descriptor list.__iter__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_list,
+                                               '__iter__')
+    return w_iter
+list_iter._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     w_type = space.type(w_obj)
     typename = w_type.getname(space)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -86,6 +86,7 @@
         return s_None
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -163,7 +163,8 @@
 
 class ModuleDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         dict_w = strategy.unerase(dictimplementation.dstorage)
         self.iterator = dict_w.iteritems()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -247,7 +247,7 @@
         return 0
 
     def iter(self, w_dict):
-        return EmptyIteratorImplementation(self.space, w_dict)
+        return EmptyIteratorImplementation(self.space, self, w_dict)
 
     def clear(self, w_dict):
         return
@@ -263,8 +263,9 @@
 # Iterator Implementation base classes
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.dictimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -280,7 +281,20 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.dictimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the dict.  The (key, value) pair in 'result'
+                # might be out-of-date.  We try to explicitly look up
+                # the key in the dict.
+                w_key = result[0]
+                w_value = self.dictimplementation.getitem(w_key)
+                if w_value is None:
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return (w_key, w_value)
         # no more entries
         self.dictimplementation = None
         return None, None
@@ -489,7 +503,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -503,7 +517,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -20,7 +20,17 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_w(w_lookup_type, space.w_str):
+        if (space.is_w(w_lookup_type, space.w_str) or  # Most common path first
+            space.abstract_issubclass_w(w_lookup_type, space.w_str)):
+            return self.getitem_str(w_dict, space.str_w(w_key))
+        elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode):
+            try:
+                w_key = space.str(w_key)
+            except OperationError, e:
+                if not e.match(space, space.w_UnicodeEncodeError):
+                    raise
+                # non-ascii unicode is never equal to a byte string
+                return None
             return self.getitem_str(w_dict, space.str_w(w_key))
         else:
             return None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -439,6 +439,8 @@
             t = w_obj.getitems()
         elif isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
+        elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            t = w_obj.getitems()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
         if expected_length != -1 and len(t) != expected_length:
@@ -456,6 +458,8 @@
             return w_obj.listview_str()
         if isinstance(w_obj, W_StringObject):
             return w_obj.listview_str()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_str()
         return None
 
     def listview_int(self, w_obj):
@@ -465,8 +469,14 @@
             return w_obj.listview_int()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_int()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_int()
         return None
 
+    def _uses_list_iter(self, w_obj):
+        from pypy.objspace.descroperation import list_iter
+        return self.lookup(w_obj, '__iter__') is list_iter(self)
+
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -359,7 +359,7 @@
         w_set.sstorage = w_other.get_storage_copy()
 
     def iter(self, w_set):
-        return EmptyIteratorImplementation(self.space, w_set)
+        return EmptyIteratorImplementation(self.space, self, w_set)
 
     def popitem(self, w_set):
         raise OperationError(self.space.w_KeyError,
@@ -784,8 +784,9 @@
             d_obj[w_item] = None
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.setimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -801,7 +802,17 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.setimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the set.  The 'result' might be out-of-date.
+                # We try to explicitly look it up in the set.
+                if not self.setimplementation.has_key(result):
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return result
         # no more entries
         self.setimplementation = None
         return None
@@ -823,7 +834,7 @@
 
 class StringIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, w_set):
-        IteratorImplementation.__init__(self, space, w_set)
+        IteratorImplementation.__init__(self, space, strategy, w_set)
         d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
@@ -835,9 +846,9 @@
 
 class IntegerIteratorImplementation(IteratorImplementation):
     #XXX same implementation in dictmultiobject on dictstrategy-branch
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
@@ -848,9 +859,9 @@
             return None
 
 class RDictIteratorImplementation(IteratorImplementation):
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -804,6 +804,33 @@
         assert "IntDictStrategy" in self.get_strategy(d)
         assert d[1L] == "hi"
 
+    def test_iter_dict_length_change(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d[7] = 8
+        # 'd' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_dict_strategy_only_change_1(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert d.get(Foo()) is None    # this changes the strategy of 'd'
+        lst = list(it)  # but iterating still works
+        assert sorted(lst) == [(1, 2), (3, 4), (5, 6)]
+
+    def test_iter_dict_strategy_only_change_2(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d['foo'] = 'bar'
+        del d[1]
+        # 'd' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us (1, 2), but 1 is not in the dict any longer.
+        raises(RuntimeError, list, it)
+
 
 class FakeString(str):
     hash_count = 0
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -25,6 +25,16 @@
         key, value = NotEmpty.__dict__.popitem()
         assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
 
+    def test_dictproxy_getitem(self):
+        class NotEmpty(object):
+            a = 1
+        assert 'a' in NotEmpty.__dict__
+        class substr(str): pass
+        assert substr('a') in NotEmpty.__dict__
+        assert u'a' in NotEmpty.__dict__
+        assert NotEmpty.__dict__[u'a'] == 1
+        assert u'\xe9' not in NotEmpty.__dict__
+
     def test_dictproxyeq(self):
         class a(object):
             pass
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1186,14 +1186,23 @@
         # of dicts, because the OrderedDict in the stdlib relies on this.
         # we extend the use case to lists and sets, i.e. all types that have
         # strategies, to avoid surprizes depending on the strategy.
-        for base, arg in [(list, []), (list, [5]), (list, ['x']),
-                          (set, []),  (set,  [5]), (set,  ['x']),
-                          (dict, []), (dict, [(5,6)]), (dict, [('x',7)])]:
+        class X: pass
+        for base, arg in [
+                (list, []), (list, [5]), (list, ['x']), (list, [X]),
+                (set, []),  (set,  [5]), (set,  ['x']), (set, [X]),
+                (dict, []), (dict, [(5,6)]), (dict, [('x',7)]), (dict, [(X,8)]),
+                ]:
             print base, arg
             class SubClass(base):
                 def __iter__(self):
                     return iter("foobar")
             assert list(SubClass(arg)) == ['f', 'o', 'o', 'b', 'a', 'r']
+            class Sub2(base):
+                pass
+            assert list(Sub2(arg)) == list(base(arg))
+            s = set()
+            s.update(Sub2(arg))
+            assert s == set(base(arg))
 
 class AppTestForRangeLists(AppTestW_ListObject):
 
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -907,3 +907,30 @@
                 return [5, 3, 4][i]
         s = set([10,3,2]).intersection(Obj())
         assert list(s) == [3]
+
+    def test_iter_set_length_change(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add(7)
+        # 's' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_set_strategy_only_change_1(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert Foo() not in s      # this changes the strategy of 'd'
+        lst = list(s)  # but iterating still works
+        assert sorted(lst) == [1, 3, 5]
+
+    def test_iter_set_strategy_only_change_2(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add('foo')
+        s.remove(1)
+        # 's' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us 1, but 1 is not in the set any longer.
+        raises(RuntimeError, list, it)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -382,7 +382,7 @@
         pass
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 vref_None = non_virtual_ref(None)
 
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -22,6 +22,7 @@
                 c_name = hop.inputconst(lltype.Void, 'access_helper')
                 args_v = [hop.inputarg(arg, arg=i)
                           for i, arg in enumerate(hop.args_r)]
+                hop.exception_cannot_occur()
                 return hop.genop('jit_marker', [c_name, c_func] + args_v,
                                  resulttype=hop.r_result)
         return helper
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -215,6 +215,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
@@ -397,6 +398,7 @@
         r_obj, = hop.args_r
         v_obj, = hop.inputargs(r_obj)
         ll_fn = r_obj.get_ll_hash_function()
+        hop.exception_is_here()
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
@@ -419,6 +421,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_identity_hash() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_identityhash', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -441,6 +444,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_id', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -452,6 +456,7 @@
 
     def specialize_call(self, hop):
         vobj, = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         if hop.rtyper.type_system.name == 'lltypesystem':
             from pypy.rpython.lltypesystem import lltype
             if isinstance(vobj.concretetype, lltype.Ptr):
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -85,7 +85,7 @@
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 
 class rbigint(object):
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -100,6 +100,7 @@
         def specialize_call(self, hop):
             bk = hop.rtyper.annotator.bookkeeper
             s_obj = identity.get_input_annotation(bk)
+            hop.exception_cannot_occur()
             return hop.r_result.rtype_erase(hop, s_obj)
 
     class Entry(ExtRegistryEntry):
@@ -110,6 +111,7 @@
             return identity.leave_tunnel(self.bookkeeper)
 
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
@@ -214,6 +216,7 @@
         return hop.genop('cast_opaque_ptr', [v], resulttype=hop.r_result)
 
     def rtype_unerase_int(self, hop, v):
+        hop.exception_cannot_occur()
         return hop.gendirectcall(ll_unerase_int, v)
 
     def rtype_erase_int(self, hop):
@@ -264,6 +267,7 @@
 
     def rtype_unerase_int(self, hop, v):
         c_one = hop.inputconst(lltype.Signed, 1)
+        hop.exception_cannot_occur()
         v2 = hop.genop('oounbox_int', [v], resulttype=hop.r_result)
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -382,6 +382,7 @@
     def compute_result_annotation(self):
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_roots', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
@@ -392,6 +393,7 @@
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_referents', vlist,
                          resulttype = hop.r_result)
 
@@ -402,6 +404,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_memory_usage', vlist,
                          resulttype = hop.r_result)
 
@@ -412,6 +415,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
@@ -430,6 +434,7 @@
         return annmodel.SomeBool()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_is_rpy_instance', vlist,
                          resulttype = hop.r_result)
 
@@ -449,6 +454,7 @@
         classrepr = getclassrepr(hop.rtyper, classdef)
         vtable = classrepr.getvtable()
         assert lltype.typeOf(vtable) == rclass.CLASSTYPE
+        hop.exception_cannot_occur()
         return Constant(vtable, concretetype=rclass.CLASSTYPE)
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py
--- a/pypy/rlib/rstring.py
+++ b/pypy/rlib/rstring.py
@@ -245,5 +245,5 @@
             raise ValueError("Value is not no_nul")
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
diff --git a/pypy/rpython/controllerentry.py b/pypy/rpython/controllerentry.py
--- a/pypy/rpython/controllerentry.py
+++ b/pypy/rpython/controllerentry.py
@@ -201,6 +201,7 @@
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
         assert hop.s_result.is_constant()
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -130,6 +130,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 def enum_ops_without_sideeffects(raising_is_ok=False):
diff --git a/pypy/rpython/lltypesystem/rbuiltin.py b/pypy/rpython/lltypesystem/rbuiltin.py
--- a/pypy/rpython/lltypesystem/rbuiltin.py
+++ b/pypy/rpython/lltypesystem/rbuiltin.py
@@ -9,6 +9,7 @@
 from pypy.rpython.rbool import bool_repr
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr or hop.args_r[1] == pyobj_repr:
@@ -33,6 +34,7 @@
     return my_instantiate()
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     s_class = hop.args_s[0]
     assert isinstance(s_class, annmodel.SomePBC)
     if len(s_class.descriptions) != 1:
@@ -46,6 +48,7 @@
     return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops)
 
 def rtype_builtin_hasattr(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr:
@@ -56,6 +59,7 @@
     raise TyperError("hasattr is only suported on a constant or on PyObject")
 
 def rtype_builtin___import__(hop):
+    xxx    # should not be used any more
     args_v = hop.inputargs(*[pyobj_repr for ign in hop.args_r])
     c = hop.inputconst(pyobj_repr, __import__)
     return hop.genop('simple_call', [c] + args_v, resulttype = pyobj_repr)
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -746,4 +746,5 @@
         assert isinstance(TYPE, GcStruct)
         assert lltype._castdepth(TYPE, OBJECT) > 0
         hop.rtyper.set_type_for_typeptr(vtable, TYPE)
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -55,6 +55,7 @@
         vtup = hop.inputarg(self, 0)
         LIST = hop.r_result.lowleveltype.TO
         cno = inputconst(Signed, nitems)
+        hop.exception_is_here()
         vlist = hop.gendirectcall(LIST.ll_newlist, cno)
         v_func = hop.inputconst(Void, rlist.dum_nocheck)
         for index in range(nitems):
diff --git a/pypy/rpython/module/r_os_stat.py b/pypy/rpython/module/r_os_stat.py
--- a/pypy/rpython/module/r_os_stat.py
+++ b/pypy/rpython/module/r_os_stat.py
@@ -65,4 +65,5 @@
     r_StatResult = hop.rtyper.getrepr(ll_os_stat.s_StatResult)
     [v_result] = hop.inputargs(r_StatResult.r_tuple)
     # no-op conversion from r_StatResult.r_tuple to r_StatResult
+    hop.exception_cannot_occur()
     return v_result
diff --git a/pypy/rpython/ootypesystem/ooregistry.py b/pypy/rpython/ootypesystem/ooregistry.py
--- a/pypy/rpython/ootypesystem/ooregistry.py
+++ b/pypy/rpython/ootypesystem/ooregistry.py
@@ -22,6 +22,7 @@
                                          annmodel.SomeOOInstance,
                                          annmodel.SomeString))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oostring', vlist, resulttype = ootype.String)
 
 class Entry_oounicode(ExtRegistryEntry):
@@ -38,6 +39,7 @@
         assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint,
                                           annmodel.SomeOOInstance))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oounicode', vlist, resulttype = ootype.Unicode)
     
 
diff --git a/pypy/rpython/ootypesystem/rbuiltin.py b/pypy/rpython/ootypesystem/rbuiltin.py
--- a/pypy/rpython/ootypesystem/rbuiltin.py
+++ b/pypy/rpython/ootypesystem/rbuiltin.py
@@ -7,12 +7,14 @@
 from pypy.rpython.error import TyperError
 
 def rtype_new(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputargs(ootype.Void)
     return hop.genop('new', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_oonewarray(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputarg(ootype.Void, arg=0)
     vlength = hop.inputarg(ootype.Signed, arg=1)
@@ -20,23 +22,27 @@
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_null(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     TYPE = hop.args_s[0].const
     nullvalue = ootype.null(TYPE)
     return hop.inputconst(TYPE, nullvalue)
 
 def rtype_classof(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
     vlist = hop.inputargs(hop.args_r[0])
     return hop.genop('classof', vlist,
                      resulttype = ootype.Class)
 
 def rtype_subclassof(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr, rootype.ooclass_repr)
     return hop.genop('subclassof', vlist,
                      resulttype = ootype.Bool)
 
 def rtype_instanceof(hop):
+    hop.exception_cannot_occur()
     INSTANCE = hop.args_v[1].value
     v_inst = hop.inputarg(hop.args_r[0], arg=0)
     c_cls = hop.inputconst(ootype.Void, INSTANCE)
@@ -44,23 +50,27 @@
                      resulttype=ootype.Bool)
 
 def rtype_runtimenew(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr)
     return hop.genop('runtimenew', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_ooupcast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_oodowncast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_to_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOStaticMeth) or \
            isinstance(hop.args_s[0], annmodel.SomeOOClass) or \
            isinstance(hop.args_s[0].ootype, ootype.OOType)
@@ -68,12 +78,14 @@
     return hop.genop('cast_to_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_from_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.OOType)
     assert isinstance(hop.args_s[1], annmodel.SomeOOObject)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('cast_from_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(ootype.Bool, hop.s_result.const)
 
@@ -99,6 +111,7 @@
     return ootype.subclassof(c1, class_)
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     if hop.args_s[0].is_constant():
 ##        INSTANCE = hop.s_result.rtyper_makerepr(hop.rtyper).lowleveltype
 ##        v_instance = hop.inputconst(ootype.Void, INSTANCE)
diff --git a/pypy/rpython/ootypesystem/rtuple.py b/pypy/rpython/ootypesystem/rtuple.py
--- a/pypy/rpython/ootypesystem/rtuple.py
+++ b/pypy/rpython/ootypesystem/rtuple.py
@@ -39,6 +39,7 @@
         RESULT = hop.r_result.lowleveltype
         c_resulttype = inputconst(ootype.Void, RESULT)
         c_length = inputconst(ootype.Signed, len(self.items_r))
+        hop.exception_is_here()
         if isinstance(RESULT, ootype.Array):
             v_list = hop.genop('oonewarray', [c_resulttype, c_length], resulttype=RESULT)
         else:
diff --git a/pypy/rpython/rbool.py b/pypy/rpython/rbool.py
--- a/pypy/rpython/rbool.py
+++ b/pypy/rpython/rbool.py
@@ -34,6 +34,7 @@
 
     def rtype_float(_, hop):
         vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
         return vlist[0]
 
 #
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -111,25 +111,32 @@
         raise TyperError("don't know about built-in function %r" % (
             self.builtinfunc,))
 
+    def _call(self, hop2, **kwds_i):
+        bltintyper = self.findbltintyper(hop2.rtyper)
+        hop2.llops._called_exception_is_here_or_cannot_occur = False
+        v_result = bltintyper(hop2, **kwds_i)
+        if not hop2.llops._called_exception_is_here_or_cannot_occur:
+            raise TyperError("missing hop.exception_cannot_occur() or "
+                             "hop.exception_is_here() in %s" % bltintyper)
+        return v_result
+
     def rtype_simple_call(self, hop):
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
-        return bltintyper(hop2)
+        return self._call(hop2)
 
     def rtype_call_args(self, hop):
         # calling a built-in function with keyword arguments:
         # mostly for rpython.objectmodel.hint()
         hop, kwds_i = call_args_expand(hop)
 
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
         hop2.r_s_popfirstarg()
         # the RPython-level keyword args are passed with an 'i_' prefix and
         # the corresponding value is an *index* in the hop2 arguments,
         # to be used with hop.inputarg(arg=..)
-        return bltintyper(hop2, **kwds_i)
+        return self._call(hop2, **kwds_i)
 
 
 class BuiltinMethodRepr(Repr):
@@ -198,6 +205,7 @@
 # ____________________________________________________________
 
 def rtype_builtin_bool(hop):
+    # not called any more?
     assert hop.nb_args == 1
     return hop.args_r[0].rtype_is_true(hop)
 
@@ -241,6 +249,7 @@
 
 def rtype_builtin_min(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_min, v1, v2)
 
 def ll_min(i1, i2):
@@ -250,6 +259,7 @@
 
 def rtype_builtin_max(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_max, v1, v2)
 
 def ll_max(i1, i2):
@@ -264,6 +274,7 @@
     pass
 
 def rtype_OSError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("OSError() should not be called with "
                          "a single argument")
@@ -274,6 +285,7 @@
         r_self.setfield(v_self, 'errno', v_errno, hop.llops)
 
 def rtype_WindowsError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("WindowsError() should not be called with "
                          "a single argument")
@@ -442,6 +454,7 @@
     assert hop.args_s[0].is_constant()
     TGT = hop.args_s[0].const
     v_type, v_value = hop.inputargs(lltype.Void, hop.args_r[1])
+    hop.exception_cannot_occur()
     return gen_cast(hop.llops, TGT, v_value)
 
 _cast_to_Signed = {
@@ -523,11 +536,13 @@
 
 def rtype_identity_hash(hop):
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('gc_identityhash', vlist, resulttype=lltype.Signed)
 
 def rtype_runtime_type_info(hop):
     assert isinstance(hop.args_r[0], rptr.PtrRepr)
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('runtime_type_info', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
@@ -558,6 +573,7 @@
 
 def rtype_raw_malloc(hop):
     v_size, = hop.inputargs(lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_malloc', [v_size], resulttype=llmemory.Address)
 
 def rtype_raw_malloc_usage(hop):
@@ -586,6 +602,7 @@
     if s_addr.is_null_address():
         raise TyperError("raw_memclear(x, n) where x is the constant NULL")
     v_list = hop.inputargs(llmemory.Address, lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_memclear', v_list)
 
 BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc
@@ -596,6 +613,7 @@
 
 def rtype_offsetof(hop):
     TYPE, field = hop.inputargs(lltype.Void, lltype.Void)
+    hop.exception_cannot_occur()
     return hop.inputconst(lltype.Signed,
                           llmemory.offsetof(TYPE.value, field.value))
 
@@ -605,6 +623,7 @@
 # non-gc objects
 
 def rtype_free_non_gc_object(hop):
+    hop.exception_cannot_occur()
     vinst, = hop.inputargs(hop.args_r[0])
     flavor = hop.args_r[0].gcflavor
     assert flavor != 'gc'
@@ -617,6 +636,7 @@
 # keepalive_until_here
 
 def rtype_keepalive_until_here(hop):
+    hop.exception_cannot_occur()
     for v in hop.args_v:
         hop.genop('keepalive', [v], resulttype=lltype.Void)
     return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/rfloat.py b/pypy/rpython/rfloat.py
--- a/pypy/rpython/rfloat.py
+++ b/pypy/rpython/rfloat.py
@@ -136,7 +136,10 @@
         hop.exception_cannot_occur()
         return hop.genop('cast_float_to_int', vlist, resulttype=Signed)
 
-    rtype_float = rtype_pos
+    def rtype_float(_, hop):
+        vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
+        return vlist[0]
 
     # version picked by specialisation based on which
     # type system rtyping is using, from <type_system>.ll_str module
diff --git a/pypy/rpython/rint.py b/pypy/rpython/rint.py
--- a/pypy/rpython/rint.py
+++ b/pypy/rpython/rint.py
@@ -310,6 +310,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_chr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_char', vlist, resulttype=Char)
 
     def rtype_unichr(_, hop):
@@ -317,6 +319,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_unichr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_unichar', vlist, resulttype=UniChar)
 
     def rtype_is_true(self, hop):
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -115,6 +115,7 @@
     def rtype_bltn_list(self, hop):
         v_lst = hop.inputarg(self, 0)
         cRESLIST = hop.inputconst(Void, hop.r_result.LIST)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_copy, cRESLIST, v_lst)
 
     def rtype_len(self, hop):
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -107,8 +107,10 @@
     if isinstance(hop.r_result, AbstractRangeRepr):
         if hop.r_result.step != 0:
             c_rng = hop.inputconst(Void, hop.r_result.RANGE)
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrange, c_rng, vstart, vstop)
         else:
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrangest, vstart, vstop, vstep)
     else:
         # cannot build a RANGE object, needs a real list
@@ -117,6 +119,7 @@
         if isinstance(ITEMTYPE, Ptr):
             ITEMTYPE = ITEMTYPE.TO
         cLIST = hop.inputconst(Void, ITEMTYPE)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_range2list, cLIST, vstart, vstop, vstep)
 
 rtype_builtin_xrange = rtype_builtin_range
@@ -212,4 +215,5 @@
                                      [v_index, v_item])
 
 def rtype_builtin_enumerate(hop):
+    hop.exception_cannot_occur()
     return hop.r_result.r_baseiter.newiter(hop)
diff --git a/pypy/rpython/rstr.py b/pypy/rpython/rstr.py
--- a/pypy/rpython/rstr.py
+++ b/pypy/rpython/rstr.py
@@ -288,6 +288,8 @@
 
     def rtype_unicode(self, hop):
         if hop.args_s[0].is_constant():
+            # convertion errors occur during annotation, so cannot any more:
+            hop.exception_cannot_occur()
             return hop.inputconst(hop.r_result, hop.s_result.const)
         repr = hop.args_r[0].repr
         v_str = hop.inputarg(repr, 0)
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -846,6 +846,7 @@
         return result
 
     def exception_is_here(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
@@ -861,6 +862,7 @@
         self.llops.llop_raising_exceptions = len(self.llops)
 
     def exception_cannot_occur(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
diff --git a/pypy/rpython/test/test_extregistry.py b/pypy/rpython/test/test_extregistry.py
--- a/pypy/rpython/test/test_extregistry.py
+++ b/pypy/rpython/test/test_extregistry.py
@@ -114,6 +114,7 @@
         _about_ = dummy_func
         s_result_annotation = annmodel.SomeInteger()
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             return hop.inputconst(lltype.Signed, 42)
 
     def func():
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1085,6 +1085,7 @@
                 return annmodel.SomeInteger()
             def specialize_call(self, hop):
                 [v_instance] = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.gendirectcall(ll_my_gethash, v_instance)
 
         def f(n):
diff --git a/pypy/translator/c/test/test_extfunc.py b/pypy/translator/c/test/test_extfunc.py
--- a/pypy/translator/c/test/test_extfunc.py
+++ b/pypy/translator/c/test/test_extfunc.py
@@ -919,4 +919,5 @@
             t, cbuilder = self.compile(does_stuff)
             data = cbuilder.cmdexec('')
             res = os.nice(0) + 3
+            if res > 19: res = 19    # xxx Linux specific, probably
             assert data.startswith('os.nice returned %d\n' % res)
diff --git a/pypy/translator/cli/dotnet.py b/pypy/translator/cli/dotnet.py
--- a/pypy/translator/cli/dotnet.py
+++ b/pypy/translator/cli/dotnet.py
@@ -459,6 +459,7 @@
             
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         assert hop.args_s[1].is_constant()
         TYPE = hop.args_s[1].const
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
@@ -507,6 +508,7 @@
 
     def specialize_call(self, hop):
         v_obj, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('same_as', [v_obj], hop.r_result.lowleveltype)
 
 def new_array(type, length):
@@ -608,6 +610,7 @@
 
     def specialize_call(self, hop):
         v_type, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('cli_typeof', [v_type], hop.r_result.lowleveltype)
 
 
@@ -626,6 +629,7 @@
         v_obj, = hop.inputargs(*hop.args_r)
         methodname = hop.args_r[0].methodname
         c_methodname = hop.inputconst(ootype.Void, methodname)
+        hop.exception_cannot_occur()
         return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype)
 
 
@@ -647,6 +651,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -668,6 +673,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -701,6 +707,7 @@
 
     def specialize_call(self, hop):
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype)
 
 

From noreply at buildbot.pypy.org  Tue Apr  3 01:43:49 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue,  3 Apr 2012 01:43:49 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: merge default into branch;
 translation now works again
Message-ID: <20120402234349.BEA7E46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54168:374968f4e0af
Date: 2012-04-02 16:41 -0700
http://bitbucket.org/pypy/pypy/changeset/374968f4e0af/

Log:	merge default into branch; translation now works again

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -108,7 +108,8 @@
 
 class DictProxyIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_type = strategy.unerase(dictimplementation.dstorage)
         self.iterator = w_type.dict_w.iteritems()
 
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -703,7 +703,8 @@
 
 class MapDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_obj = strategy.unerase(dictimplementation.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()

From notifications-noreply at bitbucket.org  Tue Apr  3 03:32:45 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 03 Apr 2012 01:32:45 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120403013245.5585.88664@bitbucket13.managed.contegix.com>

You have received a notification from Preston Timmons.

Hi, I forked pypy. My fork is at https://bitbucket.org/prestontimmons/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From notifications-noreply at bitbucket.org  Tue Apr  3 09:13:36 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 03 Apr 2012 07:13:36 -0000
Subject: [pypy-commit] Notification: jitviewer
Message-ID: <20120403071336.3153.20260@bitbucket02.managed.contegix.com>

You have received a notification from michielbaird.

Hi, I forked jitviewer. My fork is at https://bitbucket.org/michielbaird/jitviewer.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Tue Apr  3 13:08:27 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue,  3 Apr 2012 13:08:27 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: I think I disabled translate by
	chance
Message-ID: <20120403110827.4EFAD820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r177:54315ab5faa1
Date: 2012-04-03 13:07 +0200
http://bitbucket.org/pypy/benchmarks/changeset/54315ab5faa1/

Log:	I think I disabled translate by chance

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -118,7 +118,7 @@
         ('database', 0.4)
         ]
 
-def XBM_translate(base_python, changed_python, options):
+def BM_translate(base_python, changed_python, options):
     """
     Run translate.py and returns a benchmark result for each of the phases.
     Note that we run it only with ``base_python`` (which corresponds to
@@ -148,7 +148,7 @@
         data = RawResult([time], None)
         result.append((name, data))
     return result
-XBM_translate.benchmark_name = 'trans'
+BM_translate.benchmark_name = 'trans'
 
 def BM_cpython_doc(base_python, changed_python, options):
     from unladen_swallow.perf import RawResult

From noreply at buildbot.pypy.org  Tue Apr  3 18:38:30 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  3 Apr 2012 18:38:30 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Starting to rewrite stmgc according to
 the latest version of
Message-ID: <20120403163830.BD98E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54169:4614cb079670
Date: 2012-04-03 18:09 +0200
http://bitbucket.org/pypy/pypy/changeset/4614cb079670/

Log:	Starting to rewrite stmgc according to the latest version of
	extradoc/planning/stm.txt. Moved stuff around in new files.

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -47,7 +47,7 @@
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
 from pypy.rpython.memory.gc.base import GCBase, MovingGCBase
-from pypy.rpython.memory.gc import minimarkpage, env
+from pypy.rpython.memory.gc import env
 from pypy.rpython.memory.support import mangle_hash
 from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint
 from pypy.rlib.rarithmetic import LONG_BIT_SHIFT
@@ -248,6 +248,7 @@
         #
         # The ArenaCollection() handles the nonmovable objects allocation.
         if ArenaCollectionClass is None:
+            from pypy.rpython.memory.gc import minimarkpage
             ArenaCollectionClass = minimarkpage.ArenaCollection
         self.ac = ArenaCollectionClass(arena_size, page_size,
                                        small_request_threshold)
@@ -2026,43 +2027,3 @@
                 (obj + offset).address[0] = llmemory.NULL
         self.old_objects_with_weakrefs.delete()
         self.old_objects_with_weakrefs = new_with_weakref
-
-
-# ____________________________________________________________
-
-# For testing, a simple implementation of ArenaCollection.
-# This version could be used together with obmalloc.c, but
-# it requires an extra word per object in the 'all_objects'
-# list.
-
-class SimpleArenaCollection(object):
-
-    def __init__(self, arena_size, page_size, small_request_threshold):
-        self.arena_size = arena_size   # ignored
-        self.page_size = page_size
-        self.small_request_threshold = small_request_threshold
-        self.all_objects = []
-        self.total_memory_used = 0
-
-    def malloc(self, size):
-        nsize = raw_malloc_usage(size)
-        ll_assert(nsize > 0, "malloc: size is null or negative")
-        ll_assert(nsize <= self.small_request_threshold,"malloc: size too big")
-        ll_assert((nsize & (WORD-1)) == 0, "malloc: size is not aligned")
-        #
-        result = llarena.arena_malloc(nsize, False)
-        llarena.arena_reserve(result, size)
-        self.all_objects.append((result, nsize))
-        self.total_memory_used += nsize
-        return result
-
-    def mass_free(self, ok_to_free_func):
-        objs = self.all_objects
-        self.all_objects = []
-        self.total_memory_used = 0
-        for rawobj, nsize in objs:
-            if ok_to_free_func(rawobj):
-                llarena.arena_free(rawobj)
-            else:
-                self.all_objects.append((rawobj, nsize))
-                self.total_memory_used += nsize
diff --git a/pypy/rpython/memory/gc/minimarktest.py b/pypy/rpython/memory/gc/minimarktest.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/memory/gc/minimarktest.py
@@ -0,0 +1,44 @@
+from pypy.rpython.lltypesystem import llarena
+from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.rarithmetic import LONG_BIT
+
+# For testing, a simple implementation of ArenaCollection.
+# This version could be used together with obmalloc.c, but
+# it requires an extra word per object in the 'all_objects'
+# list.
+
+WORD = LONG_BIT // 8
+
+
+class SimpleArenaCollection(object):
+
+    def __init__(self, arena_size, page_size, small_request_threshold):
+        self.arena_size = arena_size   # ignored
+        self.page_size = page_size
+        self.small_request_threshold = small_request_threshold
+        self.all_objects = []
+        self.total_memory_used = 0
+
+    def malloc(self, size):
+        nsize = raw_malloc_usage(size)
+        ll_assert(nsize > 0, "malloc: size is null or negative")
+        ll_assert(nsize <= self.small_request_threshold,"malloc: size too big")
+        ll_assert((nsize & (WORD-1)) == 0, "malloc: size is not aligned")
+        #
+        result = llarena.arena_malloc(nsize, False)
+        llarena.arena_reserve(result, size)
+        self.all_objects.append((result, nsize))
+        self.total_memory_used += nsize
+        return result
+
+    def mass_free(self, ok_to_free_func):
+        objs = self.all_objects
+        self.all_objects = []
+        self.total_memory_used = 0
+        for rawobj, nsize in objs:
+            if ok_to_free_func(rawobj):
+                llarena.arena_free(rawobj)
+            else:
+                self.all_objects.append((rawobj, nsize))
+                self.total_memory_used += nsize
diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -1,7 +1,7 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
-from pypy.rpython.memory.gc.base import GCBase
+from pypy.rpython.memory.gc.base import GCBase, MovingGCBase
 from pypy.rpython.memory.support import mangle_hash
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -15,6 +15,26 @@
 
 first_gcflag = 1 << (LONG_BIT//2)
 
+# Terminology:
+#
+#   - Objects can be LOCAL or GLOBAL.  This is what STM is based on.
+#     Local objects are not visible to any other transaction, whereas
+#     global objects are, and need care.
+#
+#   - Each object lives either in the shared area, or in a thread-local
+#     nursery.  The shared area contains:
+#       - the prebuilt objects
+#       - the small objects allocated via minimarkpage.py
+#       - the non-small raw-malloced objects
+#
+#   - The GLOBAL objects are all located in the shared area.
+#     All global objects are non-movable.
+#
+#   - The LOCAL objects might be YOUNG or OLD depending on whether they
+#     already survived a collection.  YOUNG LOCAL objects are either in
+#     the nursery or, if they are big, raw-malloced.  OLD LOCAL objects
+#     are in the shared area.
+#
 GCFLAG_GLOBAL     = first_gcflag << 0     # keep in sync with et.c
 GCFLAG_WAS_COPIED = first_gcflag << 1     # keep in sync with et.c
 GCFLAG_HAS_SHADOW = first_gcflag << 2
@@ -30,7 +50,7 @@
     return fn
 
 
-class StmGC(GCBase):
+class StmGC(MovingGCBase):
     _alloc_flavor_ = "raw"
     inline_simple_malloc = True
     inline_simple_malloc_varsize = True
@@ -43,27 +63,24 @@
     typeid_is_in_field = 'tid'
     withhash_flag_is_in_field = 'tid', GCFLAG_FIXED_HASH
 
-    GCTLS = lltype.Struct('GCTLS', ('nursery_free', llmemory.Address),
-                                   ('nursery_top', llmemory.Address),
-                                   ('nursery_start', llmemory.Address),
-                                   ('nursery_size', lltype.Signed),
-                                   ('malloc_flags', lltype.Signed),
-                                   ('pending_list', llmemory.Address),
-                                   ('surviving_weakrefs', llmemory.Address),
-                                   ('global_free', llmemory.Address),
-                                   ('global_stop', llmemory.Address),
-                          )
-
     TRANSLATION_PARAMS = {
         'stm_operations': 'use_real_one',
-        'max_nursery_size': 400*1024*1024,      # XXX 400MB
-        'tls_page_size': 64*1024,               # 64KB
+        'nursery_size': 4*1024*1024,            # 4 MB
+
+        "page_size": 1024*WORD,                 # copied from minimark.py
+        "arena_size": 65536*WORD,               # copied from minimark.py
+        "small_request_threshold": 35*WORD,     # copied from minimark.py
     }
 
-    def __init__(self, config, stm_operations='use_emulator',
-                 max_nursery_size=1024, tls_page_size=64,
+    def __init__(self, config,
+                 stm_operations='use_emulator',
+                 nursery_size=1024,
+                 page_size=16*WORD,
+                 arena_size=64*WORD,
+                 small_request_threshold=5*WORD,
+                 ArenaCollectionClass=None,
                  **kwds):
-        GCBase.__init__(self, config, **kwds)
+        MovingGCBase.__init__(self, config, **kwds)
         #
         if isinstance(stm_operations, str):
             assert stm_operations == 'use_real_one', (
@@ -72,10 +89,12 @@
             from pypy.translator.stm.stmgcintf import StmOperations
             stm_operations = StmOperations()
         #
+        from pypy.rpython.memory.gc import stmshared
         self.stm_operations = stm_operations
-        self.collector = Collector(self)
-        self.max_nursery_size = max_nursery_size
-        self.tls_page_size = tls_page_size
+        self.nursery_size = nursery_size
+        self.sharedarea = stmshared.StmGCSharedArea(self,
+                                                    page_size, arena_size,
+                                                    small_request_threshold)
         #
         def _get_size(obj):     # indirection to hide 'self'
             return self.get_size(obj)
@@ -87,64 +106,30 @@
 
     def setup(self):
         """Called at run-time to initialize the GC."""
+        #
+        # Hack: MovingGCBase.setup() sets up stuff related to id(), which
+        # we implement differently anyway.  So directly call GCBase.setup().
         GCBase.setup(self)
+        #
         self.stm_operations.setup_size_getter(
                 llhelper(self.stm_operations.GETSIZE, self._getsize_fn))
-        self.main_thread_tls = self.setup_thread(True)
-        self.mutex_lock = ll_thread.allocate_ll_lock()
+        #
+        self.sharedarea.setup()
+        #
+        from pypy.rpython.memory.gc.stmtls import StmGCTLS
+        self.main_thread_tls = StmGCTLS(self, in_main_thread=True)
+        self.main_thread_tls.enter()
 
-    def _alloc_nursery(self):
-        nursery = llarena.arena_malloc(self.max_nursery_size, 1)
-        if not nursery:
-            raise MemoryError("cannot allocate nursery")
-        return nursery
-
-    def _free_nursery(self, nursery):
-        llarena.arena_free(nursery)
-
-    def setup_thread(self, in_main_thread=False):
-        """Setup a thread.  Allocates the thread-local data structures.
-        Must be called only once per OS-level thread."""
-        tls = lltype.malloc(self.GCTLS, zero=True, flavor='raw')
-        self.stm_operations.set_tls(llmemory.cast_ptr_to_adr(tls),
-                                    int(in_main_thread))
-        tls.nursery_start = self._alloc_nursery()
-        tls.nursery_size  = self.max_nursery_size
-        tls.nursery_free  = tls.nursery_start
-        tls.nursery_top   = tls.nursery_start + tls.nursery_size
-        #
-        # XXX for now, we use as the "global area" the nursery of the
-        # main thread.  So allocation in the main thread is the same as
-        # allocation in another thread, except that the new objects
-        # should be immediately marked as GCFLAG_GLOBAL.
-        if in_main_thread:
-            tls.malloc_flags = GCFLAG_GLOBAL
-        else:
-            tls.malloc_flags = -1   # don't malloc outside a transaction!
-        return tls
-
-    @staticmethod
-    def reset_nursery(tls):
-        """Clear and forget all locally allocated objects."""
-        size = tls.nursery_free - tls.nursery_start
-        llarena.arena_reset(tls.nursery_start, size, 2)
-        tls.nursery_free = tls.nursery_start
-
-    def teardown_thread(self):
-        """Teardown a thread.  Call this just before the OS-level thread
-        disappears."""
-        tls = self.collector.get_tls()
-        self.stm_operations.del_tls()
-        self._free_nursery(tls.nursery_start)
-        lltype.free(tls, flavor='raw')
+    def get_tls(self):
+        from pypy.rpython.memory.gc.stmtls import StmGCTLS
+        tls = self.stm_operations.get_tls()
+        return StmGCTLS.cast_address_to_tls_object(tls)
 
     # ----------
 
+    @always_inline
     def allocate_bump_pointer(self, size):
-        return self._allocate_bump_pointer(self.collector.get_tls(), size)
-
-    @always_inline
-    def _allocate_bump_pointer(self, tls, size):
+        tls = self.collector.get_tls()
         free = tls.nursery_free
         top  = tls.nursery_top
         if (top - free) < llmemory.raw_malloc_usage(size):
@@ -154,7 +139,11 @@
 
     @dont_inline
     def local_collection(self, size):
-        raise MemoryError("nursery exhausted")   # XXX for now
+        tls = self.collector.get_tls()
+        if not tls.nursery_free:
+            fatalerror("malloc in a non-main thread but outside a transaction")
+        #...
+        xxxxxxxxx
 
 
     def malloc_fixedsize_clear(self, typeid, size,
@@ -166,19 +155,16 @@
         # Check the mode: either in a transactional thread, or in
         # the main thread.  For now we do the same thing in both
         # modes, but set different flags.
-        tls = self.collector.get_tls()
-        flags = tls.malloc_flags
-        ll_assert(flags != -1, "malloc() in a transactional thread but "
-                               "outside a transaction")
         #
         # Get the memory from the nursery.
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
-        result = self._allocate_bump_pointer(tls, totalsize)
+        result = self.allocate_bump_pointer(totalsize)
         #
         # Build the object.
         llarena.arena_reserve(result, totalsize)
         obj = result + size_gc_header
+        flags = 0
         if contains_weakptr:   # check constant-folded
             flags |= GCFLAG_WEAKREF
         self.init_gc_object(result, typeid, flags=flags)
@@ -190,18 +176,14 @@
                              offset_to_length):
         # XXX blindly copied from malloc_fixedsize_clear() for now.
         # XXX Be more subtle, e.g. detecting overflows, at least
-        tls = self.collector.get_tls()
-        flags = tls.malloc_flags
-        ll_assert(flags != -1, "malloc() in a transactional thread but "
-                               "outside a transaction")
         size_gc_header = self.gcheaderbuilder.size_gc_header
         nonvarsize = size_gc_header + size
         totalsize = nonvarsize + itemsize * length
         totalsize = llarena.round_up_for_allocation(totalsize)
-        result = self._allocate_bump_pointer(tls, totalsize)
+        result = self.allocate_bump_pointer(totalsize)
         llarena.arena_reserve(result, totalsize)
         obj = result + size_gc_header
-        self.init_gc_object(result, typeid, flags=flags)
+        self.init_gc_object(result, typeid, flags=0)
         (obj + offset_to_length).signed[0] = length
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -209,6 +191,7 @@
     def _malloc_local_raw(self, tls, size):
         # for _stm_write_barrier_global(): a version of malloc that does
         # no initialization of the malloc'ed object
+        xxxxxxxx
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self._allocate_bump_pointer(tls, totalsize)
@@ -235,6 +218,7 @@
 
     @dont_inline
     def _malloc_global_more(self, tls, totalsize):
+        xxxxxxxxxxxxxxxxx
         if totalsize < self.tls_page_size:
             totalsize = self.tls_page_size
         main_tls = self.main_thread_tls
@@ -480,10 +464,6 @@
     def _freeze_(self):
         return True
 
-    def get_tls(self):
-        tls = self.stm_operations.get_tls()
-        return llmemory.cast_adr_to_ptr(tls, lltype.Ptr(StmGC.GCTLS))
-
     def is_in_nursery(self, tls, addr):
         ll_assert(llmemory.cast_adr_to_int(addr) & 1 == 0,
                   "odd-valued (i.e. tagged) pointer unexpected here")
@@ -497,7 +477,6 @@
         """Start a transaction, by clearing and resetting the tls nursery."""
         tls = self.get_tls()
         self.gc.reset_nursery(tls)
-        tls.malloc_flags = 0
 
 
     def commit_transaction(self):
@@ -508,7 +487,6 @@
         debug_start("gc-collect-commit")
         #
         tls = self.get_tls()
-        tls.malloc_flags = -1
         #
         # Do a mark-and-move minor collection out of the tls' nursery
         # into the main thread's global area (which is right now also
diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -0,0 +1,14 @@
+
+
+class StmGCSharedArea(object):
+
+    def __init__(self, gc, page_size, arena_size, small_request_threshold):
+        ...
+        # The ArenaCollection() handles the nonmovable objects allocation.
+        # It contains all small GCFLAG_GLOBAL objects.  The non-small ones
+        # are directly malloc'ed.
+        if ArenaCollectionClass is None:
+            from pypy.rpython.memory.gc import minimarkpage
+            ArenaCollectionClass = minimarkpage.ArenaCollection
+        self.ac = ArenaCollectionClass(arena_size, page_size,
+                                       small_request_threshold)
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -0,0 +1,108 @@
+from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
+from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, base_ptr_lltype
+from pypy.rlib import objectmodel
+from pypy.rlib.rarithmetic import r_uint
+from pypy.rlib.debug import ll_assert
+
+from pypy.rpython.memory.gc.stmgc import WORD, NULL
+
+
+class StmGCTLS(object):
+    """The thread-local structure: we have one instance of these per thread,
+    including one for the main thread."""
+
+    _alloc_flavor_ = 'raw'
+
+    def __init__(self, gc, in_main_thread):
+        self.in_main_thread = in_main_thread
+        self.stm_operations = self.gc.stm_operations
+        self.null_address_dict = self.gc.null_address_dict
+        self.AddressStack = self.gc.AddressStack
+        self.AddressDict = self.gc.AddressDict
+        #
+        # --- current position and end of nursery, or NULL when
+        #     mallocs are forbidden
+        self.nursery_free = NULL
+        self.nursery_top  = NULL
+        # --- the start and size of the nursery belonging to this thread.
+        #     never changes.
+        self.nursery_size  = self.gc.nursery_size
+        self.nursery_start = self._alloc_nursery(self.nursery_size)
+        #
+        # --- the local raw-malloced objects, young and old
+        self.rawmalloced_young_objects = self.null_address_dict()
+        self.rawmalloced_old_objects = self.AddressStack()
+        self.rawmalloced_total_size = r_uint(0)
+        # --- the local objects with weakrefs, young and old
+        self.young_objects_with_weakrefs = self.AddressStack()
+        self.old_objects_with_weakrefs = self.AddressStack()
+        # --- support for id and identityhash: maps nursery objects with
+        #     GCFLAG_HAS_SHADOW to their future location at the next
+        #     local collection
+        self.nursery_objects_shadows = self.AddressDict()
+        #
+        self._register_with_C_code()
+
+    def teardown_thread(self):
+        self._unregister_with_C_code()
+        self._free_nursery(self.nursery_start)
+        objectmodel.free_non_gc_object(self)
+
+    def _alloc_nursery(self, nursery_size):
+        nursery = llarena.arena_malloc(nursery_size, 1)
+        if not nursery:
+            raise MemoryError("cannot allocate nursery")
+        return nursery
+
+    def _free_nursery(self, nursery):
+        llarena.arena_free(nursery)
+
+    def _register_with_C_code(self):
+        tls = cast_instance_to_base_ptr(self)
+        self.stm_operations.set_tls(llmemory.cast_ptr_to_adr(tls),
+                                    int(self.in_main_thread))
+
+    def _unregister_with_C_code(self):
+        ll_assert(self.gc.get_tls() is self,
+                  "unregister_with_C_code: wrong thread")
+        self.stm_operations.del_tls()
+
+    @staticmethod
+    def cast_address_to_tls_object(self, tlsaddr):
+        tls = llmemory.cast_adr_to_ptr(tlsaddr, base_ptr_lltype())
+        return cast_base_ptr_to_instance(tls)
+
+    # ------------------------------------------------------------
+
+    def enter(self):
+        """Enter a thread: performs any pending cleanups, and set
+        up a fresh state for allocating.  Called at the start of
+        transactions, and at the start of the main thread."""
+        # Note that the calls to enter() and leave() are not balanced:
+        # if a transaction is aborted, leave() might never be called.
+        # Be ready here to clean up any state.
+        xxx
+
+    def leave(self):
+        """Leave a thread: no more allocations are allowed.
+        Called when a transaction finishes."""
+        xxx
+
+    # ------------------------------------------------------------
+
+    def end_of_transaction_collection(self):
+        """Do an end-of-transaction collection.  Finds all surviving
+        young objects and make them old.  This guarantees that the
+        nursery is empty afterwards.  (Even if there are finalizers, XXX)
+        Assumes that there are no roots from the stack.
+        """
+        xxx
+
+    def local_collection(self):
+        """Do a local collection.  Finds all surviving young objects
+        and make them old.  Also looks for roots from the stack.
+        """
+        xxx
+
+    # ------------------------------------------------------------
diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -497,7 +497,7 @@
 
 class TestMiniMarkGCSimple(DirectGCTest):
     from pypy.rpython.memory.gc.minimark import MiniMarkGC as GCClass
-    from pypy.rpython.memory.gc.minimark import SimpleArenaCollection
+    from pypy.rpython.memory.gc.minimarktest import SimpleArenaCollection
     # test the GC itself, providing a simple class for ArenaCollection
     GC_PARAMS = {'ArenaCollectionClass': SimpleArenaCollection}
 
diff --git a/pypy/rpython/memory/gc/test/test_inspector.py b/pypy/rpython/memory/gc/test/test_inspector.py
--- a/pypy/rpython/memory/gc/test/test_inspector.py
+++ b/pypy/rpython/memory/gc/test/test_inspector.py
@@ -43,6 +43,6 @@
 
 class TestMiniMarkGCSimple(InspectorTest):
     from pypy.rpython.memory.gc.minimark import MiniMarkGC as GCClass
-    from pypy.rpython.memory.gc.minimark import SimpleArenaCollection
+    from pypy.rpython.memory.gc.minimarktest import SimpleArenaCollection
     GC_PARAMS = {'ArenaCollectionClass': SimpleArenaCollection,
                  "card_page_indices": 4}

From noreply at buildbot.pypy.org  Tue Apr  3 21:43:31 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  3 Apr 2012 21:43:31 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: adding tests where os.write
 should fail, add impl of validate_fd
Message-ID: <20120403194331.2AF4D820C7@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54170:f89544de8192
Date: 2012-04-03 22:37 +0300
http://bitbucket.org/pypy/pypy/changeset/f89544de8192/

Log:	adding tests where os.write should fail, add impl of validate_fd

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -81,7 +81,7 @@
 FILEP = rffi.COpaquePtr('FILE')
 
 fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
-#fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
+_fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
     try:
         fd = fileno(fp)
diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -1,5 +1,6 @@
 import os
-from pypy.rpython.lltypesystem.rffi import CConstant, CExternVariable, INT
+from pypy.rpython.lltypesystem.rffi import (CConstant, CExternVariable, 
+        INT, CCHARPP)
 from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.rarithmetic import intmask
@@ -18,12 +19,68 @@
     def __setitem__(self, index, value):
         assert index == 0
         ll2ctypes.TLS.errno = value
+if os.name == 'nt':
+    post_include_bits =['''
+        /* Lifted completely from CPython 3.3 Modules/posix_module.c */
+        typedef struct {
+            intptr_t osfhnd;
+            char osfile;
+        } my_ioinfo;
+        extern __declspec(dllimport) char * __pioinfo[];
+        #define IOINFO_L2E 5
+        #define IOINFO_ARRAY_ELTS   (1 << IOINFO_L2E)
+        #define IOINFO_ARRAYS 64
+        #define _NHANDLE_           (IOINFO_ARRAYS * IOINFO_ARRAY_ELTS)
+        #define FOPEN 0x01
+        #define _NO_CONSOLE_FILENO (intptr_t)-2
 
-errno_eci = ExternalCompilationInfo(
-    includes=['errno.h']
+        /* This function emulates what the windows CRT
+            does to validate file handles */
+        int
+        _PyVerify_fd(int fd)
+        {
+            const int i1 = fd >> IOINFO_L2E;
+            const int i2 = fd & ((1 << IOINFO_L2E) - 1);
+
+            static size_t sizeof_ioinfo = 0;
+
+            /* Determine the actual size of the ioinfo structure,
+             * as used by the CRT loaded in memory
+             */
+            if (sizeof_ioinfo == 0 && __pioinfo[0] != NULL) {
+                sizeof_ioinfo = _msize(__pioinfo[0]) / IOINFO_ARRAY_ELTS;
+            }
+            if (sizeof_ioinfo == 0) {
+                /* This should not happen... */
+                goto fail;
+            }
+
+            /* See that it isn't a special CLEAR fileno */
+                if (fd != _NO_CONSOLE_FILENO) {
+                /* Microsoft CRT would check that 0<=fd<_nhandle but we can't do that.  Instead
+                 * we check pointer validity and other info
+                 */
+                if (0 <= i1 && i1 < IOINFO_ARRAYS && __pioinfo[i1] != NULL) {
+                    /* finally, check that the file is open */
+                    my_ioinfo* info = (my_ioinfo*)(__pioinfo[i1] + i2 * sizeof_ioinfo);
+                    if (info->osfile & FOPEN) {
+                        return 1;
+                    }
+                }
+            }
+          fail:
+            errno = EBADF;
+            return 0;
+        }
+    ''']
+else:
+    post_include_bits = []
+eci = ExternalCompilationInfo(
+    includes=['errno.h','stdio.h'],
+    post_include_bits = post_include_bits,
 )
 
-_get_errno, _set_errno = CExternVariable(INT, 'errno', errno_eci,
+_get_errno, _set_errno = CExternVariable(INT, 'errno', eci,
                                          CConstantErrno, sandboxsafe=True,
                                          _nowrapper=True, c_type='int')
 # the default wrapper for set_errno is not suitable for use in critical places
@@ -35,12 +92,29 @@
 def set_errno(errno):
     _set_errno(rffi.cast(INT, errno))
 
+if os.name == 'nt':
+    def validate_fd_emulator(fd):
+        try:
+            os.fstat(fd)
+            return 1
+        except:
+            return 0
 
+    validate_fd = rffi.llexternal(
+        "_PyVerify_fd", [rffi.INT], rffi.INT,
+        _callable=validate_fd_emulator, compilation_info=eci,
+        _nowrapper=True, elidable_function=True, sandboxsafe=True,
+        )
+else:
+    def validate_fd(fd):
+        return 1
+    
 def closerange(fd_low, fd_high):
     # this behaves like os.closerange() from Python 2.6.
     for fd in xrange(fd_low, fd_high):
         try:
-            os.close(fd)
+            if validate_fd(fd):
+                os.close(fd)
         except OSError:
             pass
 
diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -131,3 +131,13 @@
                 os.rmdir(self.ufilename)
             except Exception:
                 pass
+
+    def test_validate_fd(self):
+        assert rposix.validate_fd(0) == 1
+        fid = open(str(udir.join('validate_test.txt')), 'w')
+        fd = fid.fileno()
+        assert rposix.validate_fd(fd) == 1
+        fid.close()
+        assert rposix.validate_fd(fd) == 0
+
+
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -916,6 +916,8 @@
 
         def os_write_llimpl(fd, data):
             count = len(data)
+            if not rposix.validate_fd(fd):
+                raise OSError(rposix.get_errno(), 'Bad file descriptor')
             buf = rffi.get_nonmovingbuffer(data)
             try:
                 written = rffi.cast(lltype.Signed, os_write(
diff --git a/pypy/rpython/test/test_llinterp.py b/pypy/rpython/test/test_llinterp.py
--- a/pypy/rpython/test/test_llinterp.py
+++ b/pypy/rpython/test/test_llinterp.py
@@ -139,7 +139,7 @@
         got = interp.find_exception(info.value)
     except ValueError:
         got = None
-    assert got is exc, "wrong exception type"
+    assert got is exc, "wrong exception type, expected %r got %r"%(exc, got)
 
 #__________________________________________________________________
 # tests
diff --git a/pypy/rpython/test/test_rbuiltin.py b/pypy/rpython/test/test_rbuiltin.py
--- a/pypy/rpython/test/test_rbuiltin.py
+++ b/pypy/rpython/test/test_rbuiltin.py
@@ -201,6 +201,14 @@
         os.close(res)
         hello = open(tmpdir).read()
         assert hello == "hello world"
+        def throws(fname):
+            fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 777)
+            os.close(fd)
+            os.write(fd, "hello world")
+            return fd
+        def g():
+            return throws(tmpdir)
+        self.interpret_raises(OSError, g, [])
 
     def test_os_write_single_char(self):
         tmpdir = str(udir.udir.join("os_write_test_char"))
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -20,7 +20,7 @@
 #endif
 
 #ifdef MS_WINDOWS
-#include "src/winstuff.c"
+/*#include "src/winstuff.c"*/
 #endif
 
 #ifdef __GNUC__

From noreply at buildbot.pypy.org  Tue Apr  3 21:43:32 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue,  3 Apr 2012 21:43:32 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: finish cleanup of fclose
Message-ID: <20120403194333.0005D820C7@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54171:21b9e0657e6f
Date: 2012-04-03 22:42 +0300
http://bitbucket.org/pypy/pypy/changeset/21b9e0657e6f/

Log:	finish cleanup of fclose

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -26,6 +26,7 @@
 from pypy.module.__builtin__.interp_classobj import W_ClassObject
 from pypy.module.__builtin__.interp_memoryview import W_MemoryView
 from pypy.rlib.entrypoint import entrypoint
+from pypy.rlib.rposix import validate_fd
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import specialize
 from pypy.rlib.exports import export_struct
@@ -80,14 +81,18 @@
 # FILE* interface
 FILEP = rffi.COpaquePtr('FILE')
 
+if sys.platform == 'win32':
+    fileno = rffi.llexternal('_fileno', [FILEP], rffi.INT)
+else:
+    fileno = rffi.llexternal('fileno', [FILEP], rffi.INT)
+
 fopen = rffi.llexternal('fopen', [CONST_STRING, CONST_STRING], FILEP)
+
 _fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    try:
-        fd = fileno(fp)
-        return os.close(fd)
-    except:
+    if not validate_fd(fileno(fp)):
         return -1
+    return _fclose(fp)
 
 fwrite = rffi.llexternal('fwrite',
                          [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
@@ -96,10 +101,6 @@
                         [rffi.VOIDP, rffi.SIZE_T, rffi.SIZE_T, FILEP],
                         rffi.SIZE_T)
 feof = rffi.llexternal('feof', [FILEP], rffi.INT)
-if sys.platform == 'win32':
-    fileno = rffi.llexternal('_fileno', [FILEP], rffi.INT)
-else:
-    fileno = rffi.llexternal('fileno', [FILEP], rffi.INT)
 
 
 constant_names = """

From noreply at buildbot.pypy.org  Wed Apr  4 00:16:18 2012
From: noreply at buildbot.pypy.org (taavi_burns)
Date: Wed,  4 Apr 2012 00:16:18 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-ufuncs3: Add trunc,
	and refactor tests for floor and ceil.
Message-ID: <20120403221618.F1D53820C7@wyvern.cs.uni-duesseldorf.de>

Author: Taavi Burns <taavi.burns at gmail.com>
Branch: numpy-ufuncs3
Changeset: r54172:68288dd9b201
Date: 2012-04-03 18:15 -0400
http://bitbucket.org/pypy/pypy/changeset/68288dd9b201/

Log:	Add trunc, and refactor tests for floor and ceil.

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -104,6 +104,7 @@
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -546,6 +546,7 @@
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
+            ("trunc", "trunc", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
             ("exp2", "exp2", 1, {"promote_to_float": True}),
             ("expm1", "expm1", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -253,24 +253,17 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floorceil(self):
-        from _numpypy import array, floor, ceil
+    def test_floorceiltrunc(self):
+        from _numpypy import array, floor, ceil, trunc
         import math
-        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = floor(a)
-        for i in range(5):
-            assert b[i] == reference[i]
-        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = ceil(a)
-        assert (reference == b).all()
-        inf = float("inf")
-        data = [1.5, 2.9999, -1.999, inf]
-        results = [math.floor(x) for x in data]
-        assert (floor(data) == results).all()
-        results = [math.ceil(x) for x in data]
-        assert (ceil(data) == results).all()
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == floor(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
+        assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -668,6 +668,15 @@
         return math.ceil(v)
 
     @simple_unary_op
+    def trunc(self, v):
+        try:
+            return int(v)
+        except OverflowError:
+            return rfloat.copysign(rfloat.INFINITY, v)
+        except ValueError:
+            return v
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)

From noreply at buildbot.pypy.org  Wed Apr  4 10:44:35 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  4 Apr 2012 10:44:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: an obscure case - hopefully fix the
	windows annotation
Message-ID: <20120404084435.59C61820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54173:add873f63889
Date: 2012-04-04 10:44 +0200
http://bitbucket.org/pypy/pypy/changeset/add873f63889/

Log:	an obscure case - hopefully fix the windows annotation

diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py
--- a/pypy/rpython/annlowlevel.py
+++ b/pypy/rpython/annlowlevel.py
@@ -543,11 +543,11 @@
         else:
             assert False
 
+        hop.exception_cannot_occur()
         if isinstance(hop.args_r[1], rpbc.NoneFrozenPBCRepr):
             return hop.inputconst(PTR, null)
         v_arg = hop.inputarg(hop.args_r[1], arg=1)
         assert isinstance(v_arg.concretetype, T)
-        hop.exception_cannot_occur()
         return hop.genop(opname, [v_arg], resulttype = PTR)
 
 

From noreply at buildbot.pypy.org  Wed Apr  4 10:58:52 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed,  4 Apr 2012 10:58:52 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fixes
Message-ID: <20120404085852.0B176820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54174:cae9d28240ae
Date: 2012-04-03 22:16 +0200
http://bitbucket.org/pypy/pypy/changeset/cae9d28240ae/

Log:	Fixes

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -92,7 +92,7 @@
         from pypy.rpython.memory.gc import stmshared
         self.stm_operations = stm_operations
         self.nursery_size = nursery_size
-        self.sharedarea = stmshared.StmGCSharedArea(self,
+        self.sharedarea = stmshared.StmGCSharedArea(self, ArenaCollectionClass,
                                                     page_size, arena_size,
                                                     small_request_threshold)
         #
@@ -118,7 +118,7 @@
         #
         from pypy.rpython.memory.gc.stmtls import StmGCTLS
         self.main_thread_tls = StmGCTLS(self, in_main_thread=True)
-        self.main_thread_tls.enter()
+        self.main_thread_tls.start_transaction()
 
     def get_tls(self):
         from pypy.rpython.memory.gc.stmtls import StmGCTLS
@@ -234,7 +234,7 @@
         debug_print("XXX collect() ignored")
 
     def start_transaction(self):
-        self.collector.start_transaction()
+        self.get_tls().start_transaction()
 
     def commit_transaction(self):
         self.collector.commit_transaction()
diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -2,8 +2,9 @@
 
 class StmGCSharedArea(object):
 
-    def __init__(self, gc, page_size, arena_size, small_request_threshold):
-        ...
+    def __init__(self, gc, ArenaCollectionClass,
+                 page_size, arena_size, small_request_threshold):
+        self.gc = gc
         # The ArenaCollection() handles the nonmovable objects allocation.
         # It contains all small GCFLAG_GLOBAL objects.  The non-small ones
         # are directly malloc'ed.
@@ -12,3 +13,6 @@
             ArenaCollectionClass = minimarkpage.ArenaCollection
         self.ac = ArenaCollectionClass(arena_size, page_size,
                                        small_request_threshold)
+
+    def setup(self):
+        pass
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -1,7 +1,7 @@
-from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
 from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, base_ptr_lltype
-from pypy.rlib import objectmodel
+from pypy.rlib.objectmodel import we_are_translated, free_non_gc_object
 from pypy.rlib.rarithmetic import r_uint
 from pypy.rlib.debug import ll_assert
 
@@ -14,7 +14,10 @@
 
     _alloc_flavor_ = 'raw'
 
+    nontranslated_dict = {}
+
     def __init__(self, gc, in_main_thread):
+        self.gc = gc
         self.in_main_thread = in_main_thread
         self.stm_operations = self.gc.stm_operations
         self.null_address_dict = self.gc.null_address_dict
@@ -25,6 +28,7 @@
         #     mallocs are forbidden
         self.nursery_free = NULL
         self.nursery_top  = NULL
+        self.nursery_pending_clear = 0
         # --- the start and size of the nursery belonging to this thread.
         #     never changes.
         self.nursery_size  = self.gc.nursery_size
@@ -32,7 +36,7 @@
         #
         # --- the local raw-malloced objects, young and old
         self.rawmalloced_young_objects = self.null_address_dict()
-        self.rawmalloced_old_objects = self.AddressStack()
+        self.rawmalloced_old_objects = None
         self.rawmalloced_total_size = r_uint(0)
         # --- the local objects with weakrefs, young and old
         self.young_objects_with_weakrefs = self.AddressStack()
@@ -47,7 +51,7 @@
     def teardown_thread(self):
         self._unregister_with_C_code()
         self._free_nursery(self.nursery_start)
-        objectmodel.free_non_gc_object(self)
+        free_non_gc_object(self)
 
     def _alloc_nursery(self, nursery_size):
         nursery = llarena.arena_malloc(nursery_size, 1)
@@ -59,9 +63,14 @@
         llarena.arena_free(nursery)
 
     def _register_with_C_code(self):
-        tls = cast_instance_to_base_ptr(self)
-        self.stm_operations.set_tls(llmemory.cast_ptr_to_adr(tls),
-                                    int(self.in_main_thread))
+        if we_are_translated():
+            tls = cast_instance_to_base_ptr(self)
+            tlsaddr = llmemory.cast_ptr_to_adr(tls)
+        else:
+            n = 10000 + len(self.nontranslated_dict)
+            tlsaddr = rffi.cast(llmemory.Address, n)
+            self.nontranslated_dict[n] = self
+        self.stm_operations.set_tls(tlsaddr, int(self.in_main_thread))
 
     def _unregister_with_C_code(self):
         ll_assert(self.gc.get_tls() is self,
@@ -70,38 +79,56 @@
 
     @staticmethod
     def cast_address_to_tls_object(self, tlsaddr):
-        tls = llmemory.cast_adr_to_ptr(tlsaddr, base_ptr_lltype())
-        return cast_base_ptr_to_instance(tls)
+        if we_are_translated():
+            tls = llmemory.cast_adr_to_ptr(tlsaddr, base_ptr_lltype())
+            return cast_base_ptr_to_instance(tls)
+        else:
+            n = rffi.cast(lltype.Signed, tlsaddr)
+            return self.nontranslated_dict[n]
 
     # ------------------------------------------------------------
 
-    def enter(self):
+    def start_transaction(self):
         """Enter a thread: performs any pending cleanups, and set
         up a fresh state for allocating.  Called at the start of
-        transactions, and at the start of the main thread."""
-        # Note that the calls to enter() and leave() are not balanced:
-        # if a transaction is aborted, leave() might never be called.
+        each transaction, and at the start of the main thread."""
+        # Note that the calls to enter() and
+        # end_of_transaction_collection() are not balanced: if a
+        # transaction is aborted, the latter might never be called.
         # Be ready here to clean up any state.
-        xxx
-
-    def leave(self):
-        """Leave a thread: no more allocations are allowed.
-        Called when a transaction finishes."""
-        xxx
+        if self.nursery_free:
+            clear_size = self.nursery_free - self.nursery_start
+        else:
+            clear_size = self.nursery_pending_clear
+        if clear_size > 0:
+            llarena.arena_reset(self.nursery_start, clear_size, 2)
+            self.nursery_pending_clear = 0
+        if self.rawmalloced_young_objects:
+            xxx
+        if self.rawmalloced_old_objects:
+            xxx
+        self.nursery_free = self.nursery_start
+        self.nursery_top  = self.nursery_start + self.nursery_size
 
     # ------------------------------------------------------------
 
-    def end_of_transaction_collection(self):
-        """Do an end-of-transaction collection.  Finds all surviving
-        young objects and make them old.  This guarantees that the
-        nursery is empty afterwards.  (Even if there are finalizers, XXX)
-        Assumes that there are no roots from the stack.
-        """
-        xxx
-
     def local_collection(self):
         """Do a local collection.  Finds all surviving young objects
         and make them old.  Also looks for roots from the stack.
+        The flag GCFLAG_WAS_COPIED is kept and the C tree is updated
+        if the local young object moves.
+        """
+        xxx
+
+    def end_of_transaction_collection(self):
+        """Do an end-of-transaction collection.  Finds all surviving
+        non-GCFLAG_WAS_COPIED young objects and make them old.  Assumes
+        that there are no roots from the stack.  This guarantees that the
+        nursery will end up empty, apart from GCFLAG_WAS_COPIED objects.
+        To finish the commit, the C code will need to copy them over the
+        global objects (or abort in case of conflict, which is still ok).
+
+        No more mallocs are allowed after this is called.
         """
         xxx
 

From pullrequests-noreply at bitbucket.org  Wed Apr  4 17:48:00 2012
From: pullrequests-noreply at bitbucket.org (unbit)
Date: Wed, 04 Apr 2012 15:48:00 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404154800.26572.80674@bitbucket05.managed.contegix.com>

Pull request #67 has been updated by unbit to include new changes.

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize

Title: added implementation of Py_Initialize,Py_Finalize,Py_SetPythonHome,Py_SetProgramName as cpyext

Creator: unbit

This commit add support for a series o cpython-compatible api functions required for embedding libpypy in c apps. They are all written in rpython (as cpyext) except for Py_Initialize requiring a call to RPython_StartupCode.
Tested with uWSGI tip.

Updated list of changes:

7eb44aa590a2 by rob... at mrspurr: "added Py_Initialize,Py_Finalize,Py_SetPythonHome,Py_SetProgramName as cpyext"
4c1e64afbe9a by rob... at goyle: "added implementation of Py_Initialize"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From pullrequests-noreply at bitbucket.org  Wed Apr  4 18:38:15 2012
From: pullrequests-noreply at bitbucket.org (Amaury Forgeot d'Arc)
Date: Wed, 04 Apr 2012 16:38:15 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404163815.5671.16282@bitbucket03.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize#comment-4791

Amaury Forgeot d'Arc (amauryfa) said:

Very nice! Especially for the RPython implementation of PyPy_Initialize().

Do the cpyext unit tests still pass?
I suspect that RPython_StartupCode is not defined in the C stub library we compile for testing cpyext with an interpreted pypy.


--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From pullrequests-noreply at bitbucket.org  Wed Apr  4 19:41:20 2012
From: pullrequests-noreply at bitbucket.org (unbit)
Date: Wed, 04 Apr 2012 17:41:20 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404174120.24185.54759@bitbucket05.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize#comment-4794

unbit said:

yes, it should be defined. I am about to commit PyFile_FromFile support, i will add the fix for the unit test too

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From pullrequests-noreply at bitbucket.org  Wed Apr  4 19:56:23 2012
From: pullrequests-noreply at bitbucket.org (Amaury Forgeot d'Arc)
Date: Wed, 04 Apr 2012 17:56:23 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404175623.19485.35891@bitbucket02.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize#comment-4795

Amaury Forgeot d'Arc (amauryfa) said:

{{{
#!python

sys.pypy_initial_path(srcdir)
}}}
It seems you are trying to reimplement setup_initial_paths() from pypy/translator/goal/app_main.py. Is there a way to reuse this function?


--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Wed Apr  4 20:30:51 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  4 Apr 2012 20:30:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: disable some features to make group
 RPython, a fair bit controversial, maybe I should make rgroup iface or so
Message-ID: <20120404183051.8700C820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54175:52f2110eff80
Date: 2012-04-04 20:30 +0200
http://bitbucket.org/pypy/pypy/changeset/52f2110eff80/

Log:	disable some features to make group RPython, a fair bit
	controversial, maybe I should make rgroup iface or so

diff --git a/pypy/rlib/rsre/rsre_re.py b/pypy/rlib/rsre/rsre_re.py
--- a/pypy/rlib/rsre/rsre_re.py
+++ b/pypy/rlib/rsre/rsre_re.py
@@ -172,8 +172,9 @@
         self._ctx = ctx
 
     def span(self, groupnum=0):
-        if not isinstance(groupnum, (int, long)):
-            groupnum = self.re.groupindex[groupnum]
+#        if not isinstance(groupnum, (int, long)):
+#            groupnum = self.re.groupindex[groupnum]
+ 
         return self._ctx.span(groupnum)
 
     def start(self, groupnum=0):
@@ -182,19 +183,25 @@
     def end(self, groupnum=0):
         return self.span(groupnum)[1]
 
-    def group(self, *groups):
-        groups = groups or (0,)
-        result = []
-        for group in groups:
-            frm, to = self.span(group)
-            if 0 <= frm <= to:
-                result.append(self._ctx._string[frm:to])
-            else:
-                result.append(None)
-        if len(result) > 1:
-            return tuple(result)
+    def group(self, group=0):
+        frm, to = self.span(group)
+        if 0 <= frm <= to:
+            return self._ctx._string[frm:to]
         else:
-            return result[0]
+            return None
+
+#    def group(self, *groups):
+#        groups = groups or (0,)
+#        result = []
+#        for group in groups:
+#            frm, to = self.span(group)
+#            if 0 <= frm <= to:
+#                result.append(self._ctx._string[frm:to])
+#            else:
+#                result.append(None)
+#        if len(result) > 1:
+#            return tuple(result)
+
 
     def groups(self, default=None):
         fmarks = self._ctx.flatten_marks()
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -204,7 +204,7 @@
         assert re.match('(a)', 'a').groups() == ('a',)
         assert re.match(r'(a)', 'a').group(0) == 'a'
         assert re.match(r'(a)', 'a').group(1) == 'a'
-        assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
+        #assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
 
         pat = re.compile('((a)|(b))(c)?')
         assert pat.match('a').groups() == ('a', 'a', None, None)
@@ -218,13 +218,13 @@
         assert m.group(0) == 'a'
         assert m.group(0) == 'a'
         assert m.group(1) == 'a'
-        assert m.group(1, 1) == ('a', 'a')
+        #assert m.group(1, 1) == ('a', 'a')
 
         pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?')
-        assert pat.match('a').group(1, 2, 3) == ('a', None, None)
-        assert pat.match('b').group('a1', 'b2', 'c3') == (
-                         (None, 'b', None))
-        assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
+        #assert pat.match('a').group(1, 2, 3) == ('a', None, None)
+        #assert pat.match('b').group('a1', 'b2', 'c3') == (
+        #                 (None, 'b', None))
+        #assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
     def test_bug_923(self):
         # Issue923: grouping inside optional lookahead problem
diff --git a/pypy/rlib/rsre/test/test_zinterp.py b/pypy/rlib/rsre/test/test_zinterp.py
--- a/pypy/rlib/rsre/test/test_zinterp.py
+++ b/pypy/rlib/rsre/test/test_zinterp.py
@@ -1,7 +1,8 @@
 # minimal test: just checks that (parts of) rsre can be translated
 
-from pypy.rpython.test.test_llinterp import gengraph
+from pypy.rpython.test.test_llinterp import gengraph, interpret
 from pypy.rlib.rsre import rsre_core
+from pypy.rlib.rsre.rsre_re import compile
 
 def main(n):
     assert n >= 0
@@ -19,3 +20,18 @@
 
 def test_gengraph():
     t, typer, graph = gengraph(main, [int])
+
+m = compile("(a|b)aaaaa")
+
+def test_match():
+    def f(i):
+        if i:
+            s = "aaaaaa"
+        else:
+            s = "caaaaa"
+        g = m.match(s)
+        if g is None:
+            return 3
+        return int("aaaaaa" == g.group(0))
+    assert interpret(f, [3]) == 1
+    assert interpret(f, [0]) == 3

From noreply at buildbot.pypy.org  Wed Apr  4 20:34:17 2012
From: noreply at buildbot.pypy.org (taavi_burns)
Date: Wed,  4 Apr 2012 20:34:17 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-ufuncs3: Update trunc to use the same
 implementation as numpy. Works around some "won't work in pypy" issues.
Message-ID: <20120404183417.ED7AC820C7@wyvern.cs.uni-duesseldorf.de>

Author: Taavi Burns <taavi.burns at gmail.com>
Branch: numpy-ufuncs3
Changeset: r54176:382d1c98cd25
Date: 2012-04-04 14:08 -0400
http://bitbucket.org/pypy/pypy/changeset/382d1c98cd25/

Log:	Update trunc to use the same implementation as numpy. Works around
	some "won't work in pypy" issues.

diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -669,12 +669,10 @@
 
     @simple_unary_op
     def trunc(self, v):
-        try:
-            return int(v)
-        except OverflowError:
-            return rfloat.copysign(rfloat.INFINITY, v)
-        except ValueError:
-            return v
+        if v < 0:
+            return math.ceil(v)
+        else:
+            return math.floor(v)
 
     @simple_unary_op
     def exp(self, v):

From noreply at buildbot.pypy.org  Wed Apr  4 20:34:19 2012
From: noreply at buildbot.pypy.org (taavi_burns)
Date: Wed,  4 Apr 2012 20:34:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge in more ufuncs
Message-ID: <20120404183419.C5B31820C7@wyvern.cs.uni-duesseldorf.de>

Author: Taavi Burns <taavi.burns at gmail.com>
Branch: 
Changeset: r54177:7244bbc777dc
Date: 2012-04-04 14:33 -0400
http://bitbucket.org/pypy/pypy/changeset/7244bbc777dc/

Log:	Merge in more ufuncs

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -105,6 +105,7 @@
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -132,6 +133,8 @@
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
         ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -546,6 +546,7 @@
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
+            ("trunc", "trunc", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
             ("exp2", "exp2", 1, {"promote_to_float": True}),
             ("expm1", "expm1", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -253,24 +253,17 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floorceil(self):
-        from _numpypy import array, floor, ceil
+    def test_floorceiltrunc(self):
+        from _numpypy import array, floor, ceil, trunc
         import math
-        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = floor(a)
-        for i in range(5):
-            assert b[i] == reference[i]
-        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = ceil(a)
-        assert (reference == b).all()
-        inf = float("inf")
-        data = [1.5, 2.9999, -1.999, inf]
-        results = [math.floor(x) for x in data]
-        assert (floor(data) == results).all()
-        results = [math.ceil(x) for x in data]
-        assert (ceil(data) == results).all()
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == floor(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
+        assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -597,6 +590,13 @@
         assert (bitwise_not(a) == ~a).all()
         assert (invert(a) == ~a).all()
 
+    def test_shift(self):
+        from _numpypy import left_shift, right_shift
+        import sys
+
+        assert (left_shift([5, 1], [2, 31]) == [20, 2**31]).all()
+        assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -668,6 +668,13 @@
         return math.ceil(v)
 
     @simple_unary_op
+    def trunc(self, v):
+        if v < 0:
+            return math.ceil(v)
+        else:
+            return math.floor(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)

From pullrequests-noreply at bitbucket.org  Wed Apr  4 20:45:13 2012
From: pullrequests-noreply at bitbucket.org (unbit)
Date: Wed, 04 Apr 2012 18:45:13 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404184513.11913.53324@bitbucket15.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize#comment-4796

unbit said:

would be great to resue that code, but i think both will require refactoring to support PYTHONHOME env var and custom sys.prefix (required by Py_SetPythonHome)

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From pullrequests-noreply at bitbucket.org  Wed Apr  4 20:55:15 2012
From: pullrequests-noreply at bitbucket.org (unbit)
Date: Wed, 04 Apr 2012 18:55:15 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404185515.2342.73686@bitbucket15.managed.contegix.com>

Updated comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize#comment-4796

unbit said:

would be great to reuse that code, but i think both will require refactoring to support PYTHONHOME env var and custom sys.prefix (required by Py_SetPythonHome)

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From pullrequests-noreply at bitbucket.org  Wed Apr  4 20:57:21 2012
From: pullrequests-noreply at bitbucket.org (Amaury Forgeot d'Arc)
Date: Wed, 04 Apr 2012 18:57:21 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404185721.23121.11178@bitbucket12.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize#comment-4799

Amaury Forgeot d'Arc (amauryfa) said:

I'm OK to leave it simple to begin with; but pypy.__file__ will hardcode the path of the pypy source tree, at translation time.
Is sys.path correctly set if one donwloads the nightly builds and unpack it anywhere?  Should the code use sys.executable somehow?

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From pullrequests-noreply at bitbucket.org  Wed Apr  4 21:05:23 2012
From: pullrequests-noreply at bitbucket.org (unbit)
Date: Wed, 04 Apr 2012 19:05:23 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404190523.9309.34742@bitbucket02.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize#comment-4800

unbit said:

honestly it is not clear to me how to make 'srcdir' independent from pypy.__file__ Apps embeding python normally set sys.executable to their binary path, so following that value could lead to problems. Maybe we could take the absolute path of libpypy-c with dladdr() trick, but this is glibc specific.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From pullrequests-noreply at bitbucket.org  Wed Apr  4 23:37:50 2012
From: pullrequests-noreply at bitbucket.org (Amaury Forgeot d'Arc)
Date: Wed, 04 Apr 2012 21:37:50 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404213750.18322.73739@bitbucket12.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize#comment-4804

Amaury Forgeot d'Arc (amauryfa) said:

You are right of course. I did embed a Python interpreter several times, but I forgot that PythonHome and Path should almost always be overridden.  Since pypy does not use sys.prefix, defaulting to the original source path is the right thing to do.


--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Thu Apr  5 00:04:06 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu,  5 Apr 2012 00:04:06 +0200 (CEST)
Subject: [pypy-commit] pypy embedded: A branch to define Py_Initialize() and
 allow embedding pypy
Message-ID: <20120404220406.1CAA3820C7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: embedded
Changeset: r54180:8e2e02b0c754
Date: 2012-04-04 23:52 +0200
http://bitbucket.org/pypy/pypy/changeset/8e2e02b0c754/

Log:	A branch to define Py_Initialize() and allow embedding pypy


From noreply at buildbot.pypy.org  Thu Apr  5 00:42:41 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu,  5 Apr 2012 00:42:41 +0200 (CEST)
Subject: [pypy-commit] pypy embedded-pypy: Redo the branch for an embedded
	pypy interpreter
Message-ID: <20120404224241.0B961820C7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: embedded-pypy
Changeset: r54183:01c28572ad1b
Date: 2012-04-05 00:34 +0200
http://bitbucket.org/pypy/pypy/changeset/01c28572ad1b/

Log:	Redo the branch for an embedded pypy interpreter


From noreply at buildbot.pypy.org  Thu Apr  5 00:42:45 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu,  5 Apr 2012 00:42:45 +0200 (CEST)
Subject: [pypy-commit] pypy embedded-pypy: Fix cpyext tests,
 I hope I did not break Py_Initialize.
Message-ID: <20120404224245.2D1AC820C7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: embedded-pypy
Changeset: r54186:e4568fc96f21
Date: 2012-04-05 00:33 +0200
http://bitbucket.org/pypy/pypy/changeset/e4568fc96f21/

Log:	Fix cpyext tests, I hope I did not break Py_Initialize.

diff --git a/pypy/module/cpyext/pythonrun.py b/pypy/module/cpyext/pythonrun.py
--- a/pypy/module/cpyext/pythonrun.py
+++ b/pypy/module/cpyext/pythonrun.py
@@ -13,8 +13,8 @@
 def Py_GetProgramName(space):
     """
     Return the program name set with Py_SetProgramName(), or the default.
-    The returned string points into static storage; the caller should not modify its
-    value."""
+    The returned string points into static storage; the caller should
+    not modify its value."""
     return space.fromcache(State).get_programname()
 
 @cpython_api([rffi.CCHARP], lltype.Void, error=CANNOT_FAIL)
@@ -27,7 +27,8 @@
 @cpython_api([rffi.CCHARP], lltype.Void, error=CANNOT_FAIL)
 def Py_SetPythonHome(space, home):
     """
-    Set the default &#8220;home&#8221; directory, that is, the location of the standard Python libraries.
+    Set the default "home" directory, that is, the location of the
+    standard Python libraries.
     """
     space.fromcache(State).set_pythonhome(home)
 
@@ -79,7 +80,7 @@
 ''').interphook('pypy_init')
 
 @cpython_api([], lltype.Void, error=CANNOT_FAIL)
-def PyPy_Initialize(space):
+def _PyPy_Initialize(space):
     srcdir = pypy.__file__
     # set pythonhome/virtualenv
     pyhome = None
diff --git a/pypy/module/cpyext/src/pythonrun.c b/pypy/module/cpyext/src/pythonrun.c
--- a/pypy/module/cpyext/src/pythonrun.c
+++ b/pypy/module/cpyext/src/pythonrun.c
@@ -36,15 +36,15 @@
 }
 
 void Py_Initialize(void) {
-
+#ifdef PYPY_STANDALONE   /* 1 or 0, but defined */
     char *errmsg = RPython_StartupCode();
     if (errmsg) {
         fprintf(stderr, "unable to initialize PyPy: %s\n", errmsg);
         abort();
         return;
     }
+#endif
 
-    PyPy_Initialize();
-
+    _PyPy_Initialize();
 }
 

From noreply at buildbot.pypy.org  Thu Apr  5 00:45:10 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu,  5 Apr 2012 00:45:10 +0200 (CEST)
Subject: [pypy-commit] pypy embedded: Close branch that was not correctly
	created :(
Message-ID: <20120404224510.3F5ED820C7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: embedded
Changeset: r54187:97498245b867
Date: 2012-04-05 00:44 +0200
http://bitbucket.org/pypy/pypy/changeset/97498245b867/

Log:	Close branch that was not correctly created :(


From pullrequests-noreply at bitbucket.org  Thu Apr  5 00:47:46 2012
From: pullrequests-noreply at bitbucket.org (Amaury Forgeot d'Arc)
Date: Wed, 04 Apr 2012 22:47:46 -0000
Subject: [pypy-commit] [pypy/pypy] added implementation of Py_Initialize,
 Py_Finalize, Py_SetPythonHome,
 Py_SetProgramName as cpyext (pull request #67)
In-Reply-To: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
References: <e8ff2d3a2fe368994d6d8f077d023f88@bitbucket.org>
Message-ID: <20120404224746.29770.67184@bitbucket13.managed.contegix.com>

Pull request #67 has been rejected by Amaury Forgeot d'Arc.

unbit/pypy had changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/67/added-implementation-of-py_initialize

I actually pulled the changes in a branch: embedded-pypy
to continue the developments from here.
This is a very good start!

Rejected changes:

7eb44aa590a2 by rob... at mrspurr: "added Py_Initialize,Py_Finalize,Py_SetPythonHome,Py_SetProgramName as cpyext"
4c1e64afbe9a by rob... at goyle: "added implementation of Py_Initialize"


The pull request has been closed.

--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Thu Apr  5 01:18:42 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  5 Apr 2012 01:18:42 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: add a test in the proper place,
	fix failing test
Message-ID: <20120404231842.05543820C7@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54188:2eeb44e7e81f
Date: 2012-04-05 00:47 +0300
http://bitbucket.org/pypy/pypy/changeset/2eeb44e7e81f/

Log:	add a test in the proper place, fix failing test

diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -84,8 +84,10 @@
         if (len == 0) and "WINGDB_PYTHON" in os.environ:
             # the ctypes call seems not to work in the Wing debugger
             return
-        assert str(buf.value).lower() == pwd
-        # ctypes returns the drive letter in uppercase, os.getcwd does not
+        assert str(buf.value).lower() == pwd.lower()
+        # ctypes returns the drive letter in uppercase, 
+        # os.getcwd does not, 
+        # but there may be uppercase in os.getcwd path
 
     pwd = os.getcwd()
     try:
@@ -187,7 +189,18 @@
         OSError, ll_execve, "/etc/passwd", [], {})
     assert info.value.errno == errno.EACCES
 
-
+def test_os_write():
+    #Same as test in rpython/test/test_rbuiltin
+    fname = str(udir.join('os_write_test.txt'))
+    fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
+    assert fd >= 0
+    os.write(fd, 'Hello world')
+    os.close(fd)
+    hello = open(fname).read()
+    assert hello == "Hello world"
+    fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
+    os.close(fd)
+    raises(OSError, os.write, fd, 'Hello world')
 
 class ExpectTestOs:
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Apr  5 01:18:43 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  5 Apr 2012 01:18:43 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: make os.utime()
	subsecond-accurate
Message-ID: <20120404231843.56CE8822B2@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54189:571fd39e0a35
Date: 2012-04-05 02:18 +0300
http://bitbucket.org/pypy/pypy/changeset/571fd39e0a35/

Log:	make os.utime() subsecond-accurate

diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -455,6 +455,6 @@
     return intmask(time), intmask(nsec)
 
 def time_t_to_FILE_TIME(time, filetime):
-    ft = (rffi.r_longlong(time) + secs_between_epochs) * 10000000
+    ft = rffi.r_longlong(time * 10000000) + secs_between_epochs * 10000000
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
     filetime.c_dwLowDateTime = rffi.r_uint(ft)    # masking off high bits

From noreply at buildbot.pypy.org  Thu Apr  5 03:37:16 2012
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu,  5 Apr 2012 03:37:16 +0200 (CEST)
Subject: [pypy-commit] pypy length-hint: length_hint basics and some initial
	iterator support
Message-ID: <20120405013716.0F2B3820C7@wyvern.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: length-hint
Changeset: r54190:856f01d18746
Date: 2012-04-04 18:31 -0700
http://bitbucket.org/pypy/pypy/changeset/856f01d18746/

Log:	length_hint basics and some initial iterator support

diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -4,6 +4,29 @@
 from pypy.objspace.std.register_all import register_all
 
 
+def length_hint(space, w_obj, default):
+    """Return the length of an object, consulting its __length_hint__
+    method if necessary.
+    """
+    try:
+        return space.len_w(w_obj)
+    except OperationError, e:
+        if not (e.match(space, space.w_TypeError) or
+                e.match(space, space.w_AttributeError)):
+            raise
+
+    try:
+        w_hint = space.call_method(w_obj, '__length_hint__')
+    except OperationError, e:
+        if not (e.match(space, space.w_TypeError) or
+                e.match(space, space.w_AttributeError)):
+            raise
+        return default
+
+    hint = space.int_w(w_hint)
+    return default if hint < 0 else hint
+
+
 class W_AbstractIterObject(W_Object):
     __slots__ = ()
 
@@ -71,10 +94,6 @@
     w_seqiter.index += 1
     return w_item
 
-# XXX __length_hint__()
-##def len__SeqIter(space,  w_seqiter):
-##    return w_seqiter.getlength(space)
-
 
 def iter__FastTupleIter(space, w_seqiter):
     return w_seqiter
@@ -92,10 +111,6 @@
     w_seqiter.index = index + 1
     return w_item
 
-# XXX __length_hint__()
-##def len__FastTupleIter(space, w_seqiter):
-##    return w_seqiter.getlength(space)
-
 
 def iter__FastListIter(space, w_seqiter):
     return w_seqiter
@@ -115,10 +130,6 @@
     w_seqiter.index = index + 1
     return w_item
 
-# XXX __length_hint__()
-##def len__FastListIter(space, w_seqiter):
-##    return w_seqiter.getlength(space)
-
 
 def iter__ReverseSeqIter(space, w_seqiter):
     return w_seqiter
@@ -136,20 +147,4 @@
         raise OperationError(space.w_StopIteration, space.w_None)
     return w_item
 
-# XXX __length_hint__()
-##def len__ReverseSeqIter(space, w_seqiter):
-##    if w_seqiter.w_seq is None:
-##        return space.wrap(0)
-##    index = w_seqiter.index+1
-##    w_length = space.len(w_seqiter.w_seq)
-##    # if length of sequence is less than index :exhaust iterator
-##    if space.is_true(space.gt(space.wrap(w_seqiter.index), w_length)):
-##        w_len = space.wrap(0)
-##        w_seqiter.w_seq = None
-##    else:
-##        w_len =space.wrap(index)
-##    if space.is_true(space.lt(w_len,space.wrap(0))):
-##        w_len = space.wrap(0)
-##    return w_len
-
 register_all(vars())
diff --git a/pypy/objspace/std/itertype.py b/pypy/objspace/std/itertype.py
--- a/pypy/objspace/std/itertype.py
+++ b/pypy/objspace/std/itertype.py
@@ -23,6 +23,12 @@
     tup      = [w_self.w_seq, space.wrap(w_self.index)]
     return space.newtuple([new_inst, space.newtuple(tup)])
 
+
+def descr_seqiter__length_hint__(space, w_self):
+    from pypy.objspace.std.iterobject import W_AbstractSeqIterObject
+    assert isinstance(w_self, W_AbstractSeqIterObject)
+    return w_self.getlength(space)
+
 # ____________________________________________________________
 
 def descr_reverseseqiter__reduce__(w_self, space):
@@ -39,6 +45,24 @@
     tup      = [w_self.w_seq, space.wrap(w_self.index)]
     return space.newtuple([new_inst, space.newtuple(tup)])
 
+
+def descr_reverseseqiter__length_hint__(space, w_self):
+    from pypy.objspace.std.iterobject import W_ReverseSeqIterObject
+    assert isinstance(w_self, W_ReverseSeqIterObject)
+    if w_self.w_seq is None:
+        return space.wrap(0)
+    index = w_self.index + 1
+    w_length = space.len(w_self.w_seq)
+    # if length of sequence is less than index :exhaust iterator
+    if space.is_true(space.gt(space.wrap(w_self.index), w_length)):
+        w_len = space.wrap(0)
+        w_self.w_seq = None
+    else:
+        w_len = space.wrap(index)
+    if space.is_true(space.lt(w_len, space.wrap(0))):
+        w_len = space.wrap(0)
+    return w_len
+
 # ____________________________________________________________
 iter_typedef = StdTypeDef("sequenceiterator",
     __doc__ = '''iter(collection) -> iterator
@@ -49,11 +73,13 @@
 In the second form, the callable is called until it returns the sentinel.''',
 
     __reduce__ = gateway.interp2app(descr_seqiter__reduce__),
+    __length_hint__ = gateway.interp2app(descr_seqiter__length_hint__),
     )
 iter_typedef.acceptable_as_base_class = False
 
 reverse_iter_typedef = StdTypeDef("reversesequenceiterator",
 
     __reduce__ = gateway.interp2app(descr_reverseseqiter__reduce__),
+    __length_hint__ = gateway.interp2app(descr_reverseseqiter__length_hint__),
     )
 reverse_iter_typedef.acceptable_as_base_class = False
diff --git a/pypy/objspace/std/test/test_lengthhint.py b/pypy/objspace/std/test/test_lengthhint.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_lengthhint.py
@@ -0,0 +1,53 @@
+from pypy.objspace.std.iterobject import length_hint
+
+
+class TestLengthHint:
+
+    SIZE = 4
+    ITEMS = range(SIZE)
+
+    def _test_length_hint(self, w_obj):
+        space = self.space
+        assert length_hint(space, w_obj, 8) == self.SIZE
+
+        w_iter = space.iter(w_obj)
+        assert space.int_w(
+            space.call_method(w_iter, '__length_hint__')) == self.SIZE
+        assert length_hint(space, w_iter, 8) == self.SIZE
+
+        space.next(w_iter)
+        assert length_hint(space, w_iter, 8) == self.SIZE - 1
+
+    def test_list(self):
+        self._test_length_hint(self.space.newlist(self.ITEMS))
+
+    def test_tuple(self):
+        self._test_length_hint(self.space.newtuple(self.ITEMS))
+
+    def test_reversed(self):
+        space = self.space
+        w_reversed = space.call_method(space.builtin, 'reversed',
+                                       space.newlist(self.ITEMS))
+        assert space.int_w(
+            space.call_method(w_reversed, '__length_hint__')) == self.SIZE
+        self._test_length_hint(w_reversed)
+
+    def test_default(self):
+        space = self.space
+        assert length_hint(space, space.w_False, 3) == 3
+
+    def test_exc(self):
+        from pypy.interpreter.error import OperationError
+        space = self.space
+        w_foo = space.appexec([], """():
+            class Foo:
+                def __length_hint__(self):
+                    1 / 0
+            return Foo()
+        """)
+        try:
+            assert length_hint(space, w_foo, 3)
+        except OperationError, e:
+            assert e.match(space, space.w_ZeroDivisionError)
+        else:
+            assert False, 'ZeroDivisionError expected'

From noreply at buildbot.pypy.org  Thu Apr  5 04:25:58 2012
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu,  5 Apr 2012 04:25:58 +0200 (CEST)
Subject: [pypy-commit] pypy length-hint: consult length_hint for unknown
	length iterables
Message-ID: <20120405022559.013EB820C7@wyvern.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: length-hint
Changeset: r54191:68bdcad461d4
Date: 2012-04-04 19:08 -0700
http://bitbucket.org/pypy/pypy/changeset/68bdcad461d4/

Log:	consult length_hint for unknown length iterables

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -826,19 +826,12 @@
         # contains a loop (made explicit with the decorator above).
         #
         # If we can guess the expected length we can preallocate.
+        from pypy.objspace.std.iterobject import length_hint
         try:
-            lgt_estimate = self.len_w(w_iterable)
-        except OperationError, o:
-            if (not o.match(self, self.w_AttributeError) and
-                not o.match(self, self.w_TypeError)):
-                raise
-            items = []
-        else:
-            try:
-                items = newlist_hint(lgt_estimate)
-            except MemoryError:
-                items = [] # it might have lied
-        #
+            items = newlist_hint(length_hint(self, w_iterable, 0))
+        except MemoryError:
+            items = [] # it might have lied
+
         while True:
             try:
                 w_item = self.next(w_iterator)

From pullrequests-noreply at bitbucket.org  Thu Apr  5 05:57:20 2012
From: pullrequests-noreply at bitbucket.org (unbit)
Date: Thu, 05 Apr 2012 03:57:20 -0000
Subject: [pypy-commit] [pypy/pypy] PyFile_FromFile implementation (pull
	request #68)
Message-ID: <819aec1e6e3998acaa7b35b136186f8a@bitbucket.org>

A new pull request has been opened by unbit.

unbit/pypy has changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/68/pyfile_fromfile-implementation

Title: PyFile_FromFile implementation

PyFile_FromFile() is used by c apps to map a libc FILE object to a Python file object. The implementation make use of the pypy-specific file.fdopen function. Tested with latest uWSGI tip

Changes to be pulled:

2afeef0e350b by unbit: "implemed PyFile_FromFile as cpyext"
7eb44aa590a2 by rob... at mrspurr: "added Py_Initialize,Py_Finalize,Py_SetPythonHome,Py_SetProgramName as cpyext"
4c1e64afbe9a by rob... at goyle: "added implementation of Py_Initialize"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Thu Apr  5 07:27:59 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Thu,  5 Apr 2012 07:27:59 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: add return type to function
	signatures
Message-ID: <20120405052759.173638208A@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54192:db42c62fffbc
Date: 2012-04-04 11:56 -0700
http://bitbucket.org/pypy/pypy/changeset/db42c62fffbc/

Log:	add return type to function signatures

diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx
--- a/pypy/module/cppyy/src/cintcwrapper.cxx
+++ b/pypy/module/cppyy/src/cintcwrapper.cxx
@@ -546,6 +546,9 @@
     TFunction* f = type_get_method(handle, method_index);
     TClassRef cr = type_from_handle(handle);
     std::ostringstream sig;
+    if (cr.GetClass() && cr->GetClassInfo()
+        && strcmp(f->GetName(), ((G__ClassInfo*)cr->GetClassInfo())->Name()) != 0)
+        sig << f->GetReturnTypeName() << " ";
     sig << cr.GetClassName() << "::" << f->GetName() << "(";
     int nArgs = f->GetNargs();
     for (int iarg = 0; iarg < nArgs; ++iarg) {
diff --git a/pypy/module/cppyy/src/reflexcwrapper.cxx b/pypy/module/cppyy/src/reflexcwrapper.cxx
--- a/pypy/module/cppyy/src/reflexcwrapper.cxx
+++ b/pypy/module/cppyy/src/reflexcwrapper.cxx
@@ -361,6 +361,8 @@
     Reflex::Member m = s.FunctionMemberAt(method_index);
     Reflex::Type mt = m.TypeOf();
     std::ostringstream sig;
+    if (!m.IsConstructor())
+        sig << mt.ReturnType().Name() << " ";
     sig << s.Name(Reflex::SCOPED) << "::" << m.Name() << "(";
     int nArgs = m.FunctionParameterSize();
     for (int iarg = 0; iarg < nArgs; ++iarg) {

From noreply at buildbot.pypy.org  Thu Apr  5 07:28:00 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Thu,  5 Apr 2012 07:28:00 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: code cleanup
Message-ID: <20120405052800.AB5468208A@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54193:b7f583abfe01
Date: 2012-04-04 22:28 -0700
http://bitbucket.org/pypy/pypy/changeset/b7f583abfe01/

Log:	code cleanup

diff --git a/pypy/module/cppyy/__init__.py b/pypy/module/cppyy/__init__.py
--- a/pypy/module/cppyy/__init__.py
+++ b/pypy/module/cppyy/__init__.py
@@ -6,7 +6,7 @@
     interpleveldefs = {
         '_load_dictionary'       : 'interp_cppyy.load_dictionary',
         '_resolve_name'          : 'interp_cppyy.resolve_name',
-        '_type_byname'           : 'interp_cppyy.type_byname',
+        '_scope_byname'          : 'interp_cppyy.scope_byname',
         '_template_byname'       : 'interp_cppyy.template_byname',
         '_set_class_generator'   : 'interp_cppyy.set_class_generator',
         '_register_class'        : 'interp_cppyy.register_class',
diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py
--- a/pypy/module/cppyy/capi/__init__.py
+++ b/pypy/module/cppyy/capi/__init__.py
@@ -39,9 +39,9 @@
     [rffi.CCHARP], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-def c_resolve_name(cppitem_name):
-    return charp2str_free(_c_resolve_name(cppitem_name))
-c_get_scope = rffi.llexternal(
+def c_resolve_name(name):
+    return charp2str_free(_c_resolve_name(name))
+c_get_scope_opaque = rffi.llexternal(
     "cppyy_get_scope",
     [rffi.CCHARP], C_SCOPE,
     threadsafe=threadsafe,
@@ -51,28 +51,36 @@
     [rffi.CCHARP], C_TYPE,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-c_get_object_type = rffi.llexternal(
-    "cppyy_get_object_type",
+_c_actual_class = rffi.llexternal(
+    "cppyy_actual_class",
     [C_TYPE, C_OBJECT], C_TYPE,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_actual_class(cppclass, cppobj):
+    return _c_actual_class(cppclass.handle, cppobj)
 
 # memory management ----------------------------------------------------------
-c_allocate = rffi.llexternal(
+_c_allocate = rffi.llexternal(
     "cppyy_allocate",
     [C_TYPE], C_OBJECT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-c_deallocate = rffi.llexternal(
+def c_allocate(cppclass):
+    return _c_allocate(cppclass.handle)
+_c_deallocate = rffi.llexternal(
     "cppyy_deallocate",
     [C_TYPE, C_OBJECT], lltype.Void,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-c_destruct = rffi.llexternal(
+def c_deallocate(cppclass, cppobject):
+    _c_deallocate(cppclass.handle, cppobject)
+_c_destruct = rffi.llexternal(
     "cppyy_destruct",
     [C_TYPE, C_OBJECT], lltype.Void,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_destruct(cppclass, cppobject):
+    _c_destruct(cppclass.handle, cppobject)
 
 # method/function dispatching ------------------------------------------------
 c_call_v = rffi.llexternal(
@@ -133,18 +141,22 @@
     threadsafe=threadsafe,
     compilation_info=backend.eci)
 
-c_call_o = rffi.llexternal(
+_c_call_o = rffi.llexternal(
     "cppyy_call_o",
     [C_METHOD, C_OBJECT, rffi.INT, rffi.VOIDP, C_TYPE], rffi.LONG,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_call_o(method_index, cppobj, nargs, args, cppclass):
+    return _c_call_o(method_index, cppobj, nargs, args, cppclass.handle)
 
-c_get_methptr_getter = rffi.llexternal(
+_c_get_methptr_getter = rffi.llexternal(
     "cppyy_get_methptr_getter",
     [C_SCOPE, rffi.INT], C_METHPTRGETTER_PTR,
     threadsafe=threadsafe,
     compilation_info=backend.eci,
     elidable_function=True)
+def c_get_methptr_getter(cppscope, method_index):
+    return _c_get_methptr_getter(cppscope.handle, method_index)
 
 # handling of function argument buffer ---------------------------------------
 c_allocate_function_args = rffi.llexternal(
@@ -202,18 +214,20 @@
     [C_TYPE], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-c_num_bases = rffi.llexternal(
+_c_num_bases = rffi.llexternal(
     "cppyy_num_bases",
     [C_TYPE], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_num_bases(cppclass):
+    return _c_num_bases(cppclass.handle)
 _c_base_name = rffi.llexternal(
     "cppyy_base_name",
     [C_TYPE, rffi.INT], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-def c_base_name(cpptype, base_index):
-    return charp2str_free(_c_base_name(cpptype, base_index))
+def c_base_name(cppclass, base_index):
+    return charp2str_free(_c_base_name(cppclass.handle, base_index))
 
 _c_is_subtype = rffi.llexternal(
     "cppyy_is_subtype",
@@ -221,12 +235,11 @@
     threadsafe=threadsafe,
     compilation_info=backend.eci,
     elidable_function=True)
-
 @jit.elidable_promote()
 def c_is_subtype(derived, base):
     if derived == base:
         return 1
-    return _c_is_subtype(derived, base)
+    return _c_is_subtype(derived.handle, base.handle)
 
 _c_base_offset = rffi.llexternal(
     "cppyy_base_offset",
@@ -234,132 +247,155 @@
     threadsafe=threadsafe,
     compilation_info=backend.eci,
     elidable_function=True)
-
 @jit.elidable_promote()
 def c_base_offset(derived, base, address):
     if derived == base:
         return 0
-    return _c_base_offset(derived, base, address)
+    return _c_base_offset(derived.handle, base.handle, address)
 
 # method/function reflection information -------------------------------------
-c_num_methods = rffi.llexternal(
+_c_num_methods = rffi.llexternal(
     "cppyy_num_methods",
     [C_SCOPE], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_num_methods(cppscope):
+    return _c_num_methods(cppscope.handle)
 _c_method_name = rffi.llexternal(
     "cppyy_method_name",
     [C_SCOPE, rffi.INT], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
 def c_method_name(cppscope, method_index):
-    return charp2str_free(_c_method_name(cppscope, method_index))
+    return charp2str_free(_c_method_name(cppscope.handle, method_index))
 _c_method_result_type = rffi.llexternal(
     "cppyy_method_result_type",
     [C_SCOPE, rffi.INT], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
 def c_method_result_type(cppscope, method_index):
-    return charp2str_free(_c_method_result_type(cppscope, method_index))
-c_method_num_args = rffi.llexternal(
+    return charp2str_free(_c_method_result_type(cppscope.handle, method_index))
+_c_method_num_args = rffi.llexternal(
     "cppyy_method_num_args",
     [C_SCOPE, rffi.INT], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-c_method_req_args = rffi.llexternal(
+def c_method_num_args(cppscope, method_index):
+    return _c_method_num_args(cppscope.handle, method_index)
+_c_method_req_args = rffi.llexternal(
     "cppyy_method_req_args",
     [C_SCOPE, rffi.INT], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_method_req_args(cppscope, method_index):
+    return _c_method_req_args(cppscope.handle, method_index)
 _c_method_arg_type = rffi.llexternal(
     "cppyy_method_arg_type",
     [C_SCOPE, rffi.INT, rffi.INT], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
 def c_method_arg_type(cppscope, method_index, arg_index):
-    return charp2str_free(_c_method_arg_type(cppscope, method_index, arg_index))
+    return charp2str_free(_c_method_arg_type(cppscope.handle, method_index, arg_index))
 _c_method_arg_default = rffi.llexternal(
     "cppyy_method_arg_default",
     [C_SCOPE, rffi.INT, rffi.INT], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
 def c_method_arg_default(cppscope, method_index, arg_index):
-    return charp2str_free(_c_method_arg_default(cppscope, method_index, arg_index))
+    return charp2str_free(_c_method_arg_default(cppscope.handle, method_index, arg_index))
 _c_method_signature = rffi.llexternal(
     "cppyy_method_signature",
     [C_SCOPE, rffi.INT], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
 def c_method_signature(cppscope, method_index):
-    return charp2str_free(_c_method_signature(cppscope, method_index))
+    return charp2str_free(_c_method_signature(cppscope.handle, method_index))
 
-c_method_index = rffi.llexternal(
+_c_method_index = rffi.llexternal(
     "cppyy_method_index",
     [C_SCOPE, rffi.CCHARP], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_method_index(cppscope, name):
+    return _c_method_index(cppscope.handle, name)
 
-c_get_method = rffi.llexternal(
+_c_get_method = rffi.llexternal(
     "cppyy_get_method",
     [C_SCOPE, rffi.INT], C_METHOD,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_get_method(cppscope, method_index):
+    return _c_get_method(cppscope.handle, method_index)
 
 # method properties ----------------------------------------------------------
-c_is_constructor = rffi.llexternal(
+_c_is_constructor = rffi.llexternal(
     "cppyy_is_constructor",
     [C_TYPE, rffi.INT], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-c_is_staticmethod = rffi.llexternal(
+def c_is_constructor(cppclass, method_index):
+    return _c_is_constructor(cppclass.handle, method_index)
+_c_is_staticmethod = rffi.llexternal(
     "cppyy_is_staticmethod",
     [C_TYPE, rffi.INT], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_is_staticmethod(cppclass, method_index):
+    return _c_is_staticmethod(cppclass.handle, method_index)
 
 # data member reflection information -----------------------------------------
-c_num_data_members = rffi.llexternal(
-    "cppyy_num_data_members",
+_c_num_datamembers = rffi.llexternal(
+    "cppyy_num_datamembers",
     [C_SCOPE], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-_c_data_member_name = rffi.llexternal(
-    "cppyy_data_member_name",
+def c_num_datamembers(cppscope):
+    return _c_num_datamembers(cppscope.handle)
+_c_datamember_name = rffi.llexternal(
+    "cppyy_datamember_name",
     [C_SCOPE, rffi.INT], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-def c_data_member_name(cppscope, data_member_index):
-    return charp2str_free(_c_data_member_name(cppscope, data_member_index))
-_c_data_member_type = rffi.llexternal(
-    "cppyy_data_member_type",
+def c_datamember_name(cppscope, datamember_index):
+    return charp2str_free(_c_datamember_name(cppscope.handle, datamember_index))
+_c_datamember_type = rffi.llexternal(
+    "cppyy_datamember_type",
     [C_SCOPE, rffi.INT], rffi.CCHARP,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-def c_data_member_type(cppscope, data_member_index):
-    return charp2str_free(_c_data_member_type(cppscope, data_member_index))
-c_data_member_offset = rffi.llexternal(
-    "cppyy_data_member_offset",
+def c_datamember_type(cppscope, datamember_index):
+    return charp2str_free(_c_datamember_type(cppscope.handle, datamember_index))
+_c_datamember_offset = rffi.llexternal(
+    "cppyy_datamember_offset",
     [C_SCOPE, rffi.INT], rffi.SIZE_T,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_datamember_offset(cppscope, datamember_index):
+    return _c_datamember_offset(cppscope.handle, datamember_index)
 
-c_data_member_index = rffi.llexternal(
-    "cppyy_data_member_index",
+_c_datamember_index = rffi.llexternal(
+    "cppyy_datamember_index",
     [C_SCOPE, rffi.CCHARP], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_datamember_index(cppscope, name):
+    return _c_datamember_index(cppscope.handle, name)
 
 # data member properties -----------------------------------------------------
-c_is_publicdata = rffi.llexternal(
+_c_is_publicdata = rffi.llexternal(
     "cppyy_is_publicdata",
     [C_SCOPE, rffi.INT], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
-c_is_staticdata = rffi.llexternal(
+def c_is_publicdata(cppscope, datamember_index):
+    return _c_is_publicdata(cppscope.handle, datamember_index)
+_c_is_staticdata = rffi.llexternal(
     "cppyy_is_staticdata",
     [C_SCOPE, rffi.INT], rffi.INT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+def c_is_staticdata(cppscope, datamember_index):
+    return _c_is_staticdata(cppscope.handle, datamember_index)
 
 # misc helpers ---------------------------------------------------------------
 c_strtoll = rffi.llexternal(
diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -64,7 +64,7 @@
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
         raise FastCallNotPossible
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         self._is_abstract(space)
 
     def to_memory(self, space, w_obj, w_value, offset):
@@ -98,7 +98,7 @@
         else:
             self.size = array_size
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         if hasattr(space, "fake"):
             raise NotImplementedError
         # read access, so no copy needed
@@ -124,7 +124,7 @@
     def __init__(self, space, array_size):
         self.size = sys.maxint
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         # read access, so no copy needed
         address_value = self._get_raw_address(space, w_obj, offset)
         address = rffi.cast(rffi.ULONGP, address_value)
@@ -154,7 +154,7 @@
     def default_argument_libffi(self, space, argchain):
         argchain.arg(self.default)
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.rffiptype, address)
         return space.wrap(rffiptr[0])
@@ -213,7 +213,7 @@
     def convert_argument_libffi(self, space, w_obj, argchain):
         argchain.arg(self._unwrap_object(space, w_obj))
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         if address[0] == '\x01':
             return space.w_True
@@ -255,7 +255,7 @@
     def convert_argument_libffi(self, space, w_obj, argchain): 
         argchain.arg(self._unwrap_object(space, w_obj))
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
         return space.wrap(address[0])
 
@@ -347,7 +347,7 @@
     def _unwrap_object(self, space, w_obj):
         return r_singlefloat(space.float_w(w_obj))
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         rffiptr = rffi.cast(self.rffiptype, address)
         return space.wrap(float(rffiptr[0]))
@@ -378,7 +378,7 @@
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset()] = 'o'
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
         charpptr = rffi.cast(rffi.CCHARPP, address)
         return space.wrap(rffi.charp2str(charpptr[0]))
@@ -505,25 +505,23 @@
 class InstancePtrConverter(TypeConverter):
     _immutable_ = True
 
-    def __init__(self, space, cpptype, name):
-        from pypy.module.cppyy.interp_cppyy import W_CPPType
-        assert isinstance(cpptype, W_CPPType)
-        self.cpptype = cpptype
-        self.name = name
+    def __init__(self, space, cppclass):
+        from pypy.module.cppyy.interp_cppyy import W_CPPClass
+        assert isinstance(cppclass, W_CPPClass)
+        self.cppclass = cppclass
 
     def _unwrap_object(self, space, w_obj):
         from pypy.module.cppyy.interp_cppyy import W_CPPInstance
         obj = space.interpclass_w(w_obj)
         if isinstance(obj, W_CPPInstance):
-            if capi.c_is_subtype(obj.cppclass.handle, self.cpptype.handle):
+            if capi.c_is_subtype(obj.cppclass, self.cppclass):
                 rawobject = obj.get_rawobject()
-                offset = capi.c_base_offset(
-                    obj.cppclass.handle, self.cpptype.handle, rawobject)
+                offset = capi.c_base_offset(obj.cppclass, self.cppclass, rawobject)
                 obj_address = capi.direct_ptradd(rawobject, offset)
                 return rffi.cast(capi.C_OBJECT, obj_address)
         raise OperationError(space.w_TypeError,
                              space.wrap("cannot pass %s as %s" %
-                             (space.type(w_obj).getname(space, "?"), self.cpptype.name)))
+                             (space.type(w_obj).getname(space, "?"), self.cppclass.name)))
 
     def convert_argument(self, space, w_obj, address):
         x = rffi.cast(rffi.VOIDPP, address)
@@ -535,10 +533,11 @@
     def convert_argument_libffi(self, space, w_obj, argchain):
         argchain.arg(self._unwrap_object(space, w_obj))
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module.cppyy import interp_cppyy
-        return interp_cppyy.wrap_cppobject_nocast(space, w_type, self.cpptype, address, True, False)
+        return interp_cppyy.wrap_cppobject_nocast(
+            space, w_pycppclass, self.cppclass, address, isref=True, python_owns=False)
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset))
@@ -547,10 +546,11 @@
 class InstanceConverter(InstancePtrConverter):
     _immutable_ = True
 
-    def from_memory(self, space, w_obj, w_type, offset):
+    def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
         from pypy.module.cppyy import interp_cppyy
-        return interp_cppyy.wrap_cppobject_nocast(space, w_type, self.cpptype, address, False, False)
+        return interp_cppyy.wrap_cppobject_nocast(
+            space, w_pycppclass, self.cppclass, address, isref=False, python_owns=False)
 
     def to_memory(self, space, w_obj, w_value, offset):
         self._is_abstract(space)
@@ -561,8 +561,8 @@
 
     def __init__(self, space, extra):
         from pypy.module.cppyy import interp_cppyy
-        cpptype = interp_cppyy.type_byname(space, "std::string")
-        InstanceConverter.__init__(self, space, cpptype, "std::string")
+        cppclass = interp_cppyy.scope_byname(space, "std::string")
+        InstanceConverter.__init__(self, space, cppclass)
 
     def _unwrap_object(self, space, w_obj):
         try:
@@ -582,8 +582,8 @@
 
     def __init__(self, space, extra):
         from pypy.module.cppyy import interp_cppyy
-        cpptype = interp_cppyy.type_byname(space, "std::string")
-        InstancePtrConverter.__init__(self, space, cpptype, "std::string")
+        cppclass = interp_cppyy.scope_byname(space, "std::string")
+        InstancePtrConverter.__init__(self, space, cppclass)
 
 
 _converters = {}         # builtin and custom types
@@ -631,15 +631,15 @@
 
     #   5) generalized cases (covers basically all user classes)
     from pypy.module.cppyy import interp_cppyy
-    cpptype = interp_cppyy.type_byname(space, clean_name)
-    if cpptype:
+    cppclass = interp_cppyy.scope_byname(space, clean_name)
+    if cppclass:
         # type check for the benefit of the annotator
-        from pypy.module.cppyy.interp_cppyy import W_CPPType
-        cpptype = space.interp_w(W_CPPType, cpptype, can_be_None=False)
+        from pypy.module.cppyy.interp_cppyy import W_CPPClass
+        cppclass = space.interp_w(W_CPPClass, cppclass, can_be_None=False)
         if compound == "*" or compound == "&":
-            return InstancePtrConverter(space, cpptype, clean_name)
+            return InstancePtrConverter(space, cppclass)
         elif compound == "":
-            return InstanceConverter(space, cpptype, clean_name)
+            return InstanceConverter(space, cppclass)
     elif capi.c_is_enum(clean_name):
         return UnsignedIntConverter(space, default)
     
diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py
--- a/pypy/module/cppyy/executor.py
+++ b/pypy/module/cppyy/executor.py
@@ -17,7 +17,7 @@
     _immutable_ = True
     libffitype = NULL
 
-    def __init__(self, space, cpptype):
+    def __init__(self, space, extra):
         pass
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
@@ -251,20 +251,22 @@
     _immutable_ = True
     libffitype = libffi.types.pointer
 
-    def __init__(self, space, cpptype):
-        FunctionExecutor.__init__(self, space, cpptype)
-        self.cpptype = cpptype
+    def __init__(self, space, cppclass):
+        FunctionExecutor.__init__(self, space, cppclass)
+        self.cppclass = cppclass
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         from pypy.module.cppyy import interp_cppyy
         long_result = capi.c_call_l(cppmethod, cppthis, num_args, args)
         ptr_result = rffi.cast(capi.C_OBJECT, long_result)
-        return interp_cppyy.wrap_cppobject(space, None, self.cpptype, ptr_result, False, False)
+        return interp_cppyy.wrap_cppobject(
+            space, space.w_None, self.cppclass, ptr_result, isref=False, python_owns=False)
 
     def execute_libffi(self, space, libffifunc, argchain):
         from pypy.module.cppyy import interp_cppyy
         ptr_result = rffi.cast(capi.C_OBJECT, libffifunc.call(argchain, rffi.VOIDP))
-        return interp_cppyy.wrap_cppobject(space, None, self.cpptype, ptr_result, False, False)
+        return interp_cppyy.wrap_cppobject(
+            space, space.w_None, self.cppclass, ptr_result, isref=False, python_owns=False)
 
 class InstancePtrPtrExecutor(InstancePtrExecutor):
     _immutable_ = True
@@ -274,7 +276,8 @@
         voidp_result = capi.c_call_r(cppmethod, cppthis, num_args, args)
         ref_address = rffi.cast(rffi.VOIDPP, voidp_result)
         ptr_result = rffi.cast(capi.C_OBJECT, ref_address[0])
-        return interp_cppyy.wrap_cppobject(space, None, self.cpptype, ptr_result, False, False)
+        return interp_cppyy.wrap_cppobject(
+            space, space.w_None, self.cppclass, ptr_result, isref=False, python_owns=False)
 
     def execute_libffi(self, space, libffifunc, argchain):
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
@@ -285,9 +288,10 @@
 
     def execute(self, space, cppmethod, cppthis, num_args, args):
         from pypy.module.cppyy import interp_cppyy
-        long_result = capi.c_call_o(cppmethod, cppthis, num_args, args, self.cpptype.handle)
+        long_result = capi.c_call_o(cppmethod, cppthis, num_args, args, self.cppclass)
         ptr_result = rffi.cast(capi.C_OBJECT, long_result)
-        return interp_cppyy.wrap_cppobject(space, None, self.cpptype, ptr_result, False, True)
+        return interp_cppyy.wrap_cppobject(
+            space, space.w_None, self.cppclass, ptr_result, isref=False, python_owns=True)
 
     def execute_libffi(self, space, libffifunc, argchain):
         from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
@@ -343,17 +347,17 @@
 
     #   3) types/classes, either by ref/ptr or by value
     from pypy.module.cppyy import interp_cppyy
-    cpptype = interp_cppyy.type_byname(space, clean_name)
-    if cpptype:
+    cppclass = interp_cppyy.scope_byname(space, clean_name)
+    if cppclass:
         # type check for the benefit of the annotator
-        from pypy.module.cppyy.interp_cppyy import W_CPPType
-        cpptype = space.interp_w(W_CPPType, cpptype, can_be_None=False)
+        from pypy.module.cppyy.interp_cppyy import W_CPPClass
+        cppclass = space.interp_w(W_CPPClass, cppclass, can_be_None=False)
         if compound == "":
-            return InstanceExecutor(space, cpptype)
+            return InstanceExecutor(space, cppclass)
         elif compound == "*" or compound == "&":
-            return InstancePtrExecutor(space, cpptype)
+            return InstancePtrExecutor(space, cppclass)
         elif compound == "**" or compound == "*&":
-            return InstancePtrPtrExecutor(space, cpptype)
+            return InstancePtrPtrExecutor(space, cppclass)
     elif capi.c_is_enum(clean_name):
         return UnsignedIntExecutor(space, None)
 
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -17,7 +17,7 @@
     char* cppyy_resolve_name(const char* cppitem_name);
     cppyy_scope_t cppyy_get_scope(const char* scope_name);
     cppyy_type_t cppyy_get_template(const char* template_name);
-    cppyy_type_t cppyy_get_object_type(cppyy_type_t klass, cppyy_object_t obj);
+    cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj);
 
     /* memory management ------------------------------------------------------ */
     cppyy_object_t cppyy_allocate(cppyy_type_t type);
@@ -80,16 +80,16 @@
     int cppyy_is_staticmethod(cppyy_type_t type, int method_index);
 
     /* data member reflection information ------------------------------------  */
-    int cppyy_num_data_members(cppyy_scope_t scope);
-    char* cppyy_data_member_name(cppyy_scope_t scope, int data_member_index);
-    char* cppyy_data_member_type(cppyy_scope_t scope, int data_member_index);
-    size_t cppyy_data_member_offset(cppyy_scope_t scope, int data_member_index);
+    int cppyy_num_datamembers(cppyy_scope_t scope);
+    char* cppyy_datamember_name(cppyy_scope_t scope, int datamember_index);
+    char* cppyy_datamember_type(cppyy_scope_t scope, int datamember_index);
+    size_t cppyy_datamember_offset(cppyy_scope_t scope, int datamember_index);
 
-    int cppyy_data_member_index(cppyy_scope_t scope, const char* name);
+    int cppyy_datamember_index(cppyy_scope_t scope, const char* name);
 
     /* data member properties ------------------------------------------------  */
-    int cppyy_is_publicdata(cppyy_type_t type, int data_member_index);
-    int cppyy_is_staticdata(cppyy_type_t type, int data_member_index);
+    int cppyy_is_publicdata(cppyy_type_t type, int datamember_index);
+    int cppyy_is_staticdata(cppyy_type_t type, int datamember_index);
 
     /* misc helpers ----------------------------------------------------------- */
     void cppyy_free(void* ptr);
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -27,10 +27,10 @@
 
 class State(object):
     def __init__(self, space):
-        self.r_cppscope_cache = {
-            "void" : W_CPPType(space, "void", capi.C_NULL_TYPE) }
-        self.r_cpptemplate_cache = {}
-        self.type_registry = {}
+        self.cppscope_cache = {
+            "void" : W_CPPClass(space, "void", capi.C_NULL_TYPE) }
+        self.cpptemplate_cache = {}
+        self.cppclass_registry = {}
         self.w_clgen_callback = None
 
 @unwrap_spec(name=str)
@@ -38,30 +38,30 @@
     return space.wrap(capi.c_resolve_name(name))
 
 @unwrap_spec(name=str)
-def type_byname(space, name):
+def scope_byname(space, name):
     true_name = capi.c_resolve_name(name)
 
     state = space.fromcache(State)
     try:
-        return state.r_cppscope_cache[true_name]
+        return state.cppscope_cache[true_name]
     except KeyError:
         pass
 
-    cppscope = capi.c_get_scope(true_name)
-    assert lltype.typeOf(cppscope) == capi.C_SCOPE
-    if cppscope:
-        final_name = capi.c_final_name(cppscope)
-        if capi.c_is_namespace(cppscope):
-            r_cppscope = W_CPPNamespace(space, final_name, cppscope)
-        elif capi.c_has_complex_hierarchy(cppscope):
-            r_cppscope = W_ComplexCPPType(space, final_name, cppscope)
+    opaque_handle = capi.c_get_scope_opaque(true_name)
+    assert lltype.typeOf(opaque_handle) == capi.C_SCOPE
+    if opaque_handle:
+        final_name = capi.c_final_name(opaque_handle)
+        if capi.c_is_namespace(opaque_handle):
+            cppscope = W_CPPNamespace(space, final_name, opaque_handle)
+        elif capi.c_has_complex_hierarchy(opaque_handle):
+            cppscope = W_ComplexCPPClass(space, final_name, opaque_handle)
         else:
-            r_cppscope = W_CPPType(space, final_name, cppscope)
-        state.r_cppscope_cache[name] = r_cppscope
+            cppscope = W_CPPClass(space, final_name, opaque_handle)
+        state.cppscope_cache[name] = cppscope
 
-        r_cppscope._find_methods()
-        r_cppscope._find_data_members()
-        return r_cppscope
+        cppscope._find_methods()
+        cppscope._find_datamembers()
+        return cppscope
 
     return None
 
@@ -69,16 +69,16 @@
 def template_byname(space, name):
     state = space.fromcache(State)
     try:
-        return state.r_cpptemplate_cache[name]
+        return state.cpptemplate_cache[name]
     except KeyError:
         pass
 
-    cpptemplate = capi.c_get_template(name)
-    assert lltype.typeOf(cpptemplate) == capi.C_TYPE
-    if cpptemplate:
-        r_cpptemplate = W_CPPTemplateType(space, name, cpptemplate)
-        state.r_cpptemplate_cache[name] = r_cpptemplate
-        return r_cpptemplate
+    opaque_handle = capi.c_get_template(name)
+    assert lltype.typeOf(opaque_handle) == capi.C_TYPE
+    if opaque_handle:
+        cpptemplate = W_CPPTemplateType(space, name, opaque_handle)
+        state.cpptemplate_cache[name] = cpptemplate
+        return cpptemplate
 
     return None
 
@@ -87,12 +87,12 @@
     state = space.fromcache(State)
     state.w_clgen_callback = w_callback
 
- at unwrap_spec(w_type=W_Root)
-def register_class(space, w_type):
-    w_cpptype = space.findattr(w_type, space.wrap("_cpp_proxy"))
-    cpptype = space.interp_w(W_CPPType, w_cpptype, can_be_None=False)
+ at unwrap_spec(w_pycppclass=W_Root)
+def register_class(space, w_pycppclass):
+    w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+    cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
     state = space.fromcache(State)
-    state.type_registry[cpptype.handle] = w_type
+    state.cppclass_registry[cppclass.handle] = w_pycppclass
 
 
 class W_CPPLibrary(Wrappable):
@@ -112,18 +112,19 @@
     """ A concrete function after overloading has been resolved """
     _immutable_ = True
     
-    def __init__(self, cpptype, method_index, result_type, arg_defs, args_required):
-        self.space = cpptype.space
-        self.cpptype = cpptype
-        self.method_index = method_index
-        self.cppmethod = capi.c_get_method(self.cpptype.handle, method_index)
+    def __init__(self, space, containing_scope, method_index, result_type, arg_defs, args_required):
+        self.space = space
+        self.scope = containing_scope
+        self.index = method_index
+        self.cppmethod = capi.c_get_method(self.scope, method_index)
         self.arg_defs = arg_defs
         self.args_required = args_required
-        self.executor = executor.get_executor(self.space, result_type)
+        self.result_type = result_type
 
         # Setup of the method dispatch's innards is done lazily, i.e. only when
-        # the method is actually used. TODO: executor should be lazy as well.
+        # the method is actually used.
         self.arg_converters = None
+        self.executor = None
         self._libffifunc = None
 
     @jit.unroll_safe
@@ -169,11 +170,12 @@
     def _setup(self, cppthis):
         self.arg_converters = [converter.get_converter(self.space, arg_type, arg_dflt)
                                    for arg_type, arg_dflt in self.arg_defs]
+        self.executor = executor.get_executor(self.space, self.result_type)
 
         # Each CPPMethod corresponds one-to-one to a C++ equivalent and cppthis
         # has been offset to the matching class. Hence, the libffi pointer is
         # uniquely defined and needs to be setup only once.
-        methgetter = capi.c_get_methptr_getter(self.cpptype.handle, self.method_index)
+        methgetter = capi.c_get_methptr_getter(self.scope, self.index)
         if methgetter and cppthis:      # methods only for now
             funcptr = methgetter(rffi.cast(capi.C_OBJECT, cppthis))
             argtypes_libffi = [conv.libffitype for conv in self.arg_converters
@@ -217,11 +219,10 @@
         capi.c_deallocate_function_args(args)
 
     def signature(self):
-        return capi.c_method_signature(self.cpptype.handle, self.method_index)
+        return capi.c_method_signature(self.scope, self.index)
 
     def __repr__(self):
-        return "CPPFunction(%s, %s, %r, %s)" % (
-            self.cpptype, self.method_index, self.executor, self.arg_defs)
+        return "CPPMethod: %s" % self.signature()
 
     def _freeze_(self):
         assert 0, "you should never have a pre-built instance of this!"
@@ -230,29 +231,34 @@
 class CPPFunction(CPPMethod):
     _immutable_ = True
 
+    def __repr__(self):
+        return "CPPFunction: %s" % self.signature()
+
 
 class CPPConstructor(CPPMethod):
     _immutable_ = True
 
     def call(self, cppthis, args_w):
-        newthis = capi.c_allocate(self.cpptype.handle)
+        newthis = capi.c_allocate(self.scope)
         assert lltype.typeOf(newthis) == capi.C_OBJECT
         try:
             CPPMethod.call(self, newthis, args_w)
         except:
-            capi.c_deallocate(self.cpptype.handle, newthis)
+            capi.c_deallocate(self.scope, newthis)
             raise
-        return wrap_new_cppobject_nocast(self.space, None, self.cpptype, newthis, False, True)
+        return wrap_new_cppobject_nocast(
+            self.space, self.space.w_None, self.scope, newthis, isref=False, python_owns=True)
+
+    def __repr__(self):
+        return "CPPConstructor: %s" % self.signature()
 
 
 class W_CPPOverload(Wrappable):
     _immutable_ = True
 
-    def __init__(self, space, scope_handle, func_name, functions):
+    def __init__(self, space, containing_scope, functions):
         self.space = space
-        assert lltype.typeOf(scope_handle) == capi.C_SCOPE
-        self.scope_handle = scope_handle
-        self.func_name = func_name
+        self.scope = containing_scope
         self.functions = debug.make_sure_not_resized(functions)
 
     def is_static(self):
@@ -263,8 +269,7 @@
         cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True)
         if cppinstance is not None:
             cppinstance._nullcheck()
-            assert isinstance(cppinstance.cppclass, W_CPPType)
-            cppthis = cppinstance.cppclass.get_cppthis(cppinstance, self.scope_handle)
+            cppthis = cppinstance.get_cppthis(self.scope)
         else:
             cppthis = capi.C_NULL_OBJECT
         assert lltype.typeOf(cppthis) == capi.C_OBJECT
@@ -300,7 +305,7 @@
                 errmsg += '    '+e.errorstr(self.space)
             except Exception, e:
                 errmsg += '\n  '+cppyyfunc.signature()+' =>\n'
-                errmsg += '    Exception:'+str(e)
+                errmsg += '    Exception: '+str(e)
 
         raise OperationError(self.space.w_TypeError, self.space.wrap(errmsg))
 
@@ -311,7 +316,7 @@
         return self.space.wrap(sig)
 
     def __repr__(self):
-        return "W_CPPOverload(%s, %s)" % (self.func_name, self.functions)
+        return "W_CPPOverload(%s)" % [f.signature() for f in self.functions]
 
 W_CPPOverload.typedef = TypeDef(
     'CPPOverload',
@@ -324,10 +329,9 @@
 class W_CPPDataMember(Wrappable):
     _immutable_ = True
 
-    def __init__(self, space, scope_handle, type_name, offset, is_static):
+    def __init__(self, space, containing_scope, type_name, offset, is_static):
         self.space = space
-        assert lltype.typeOf(scope_handle) == capi.C_SCOPE
-        self.scope_handle = scope_handle
+        self.scope = containing_scope
         self.converter = converter.get_converter(self.space, type_name, '')
         self.offset = offset
         self._is_static = is_static
@@ -341,17 +345,17 @@
     @jit.elidable_promote()
     def _get_offset(self, cppinstance):
         if cppinstance:
-            assert lltype.typeOf(cppinstance.cppclass.handle) == lltype.typeOf(self.scope_handle)
+            assert lltype.typeOf(cppinstance.cppclass.handle) == lltype.typeOf(self.scope.handle)
             offset = self.offset + capi.c_base_offset(
-                cppinstance.cppclass.handle, self.scope_handle, cppinstance.get_rawobject())
+                cppinstance.cppclass, self.scope, cppinstance.get_rawobject())
         else:
             offset = self.offset
         return offset
 
-    def get(self, w_cppinstance, w_type):
+    def get(self, w_cppinstance, w_pycppclass):
         cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True)
         offset = self._get_offset(cppinstance)
-        return self.converter.from_memory(self.space, w_cppinstance, w_type, offset)
+        return self.converter.from_memory(self.space, w_cppinstance, w_pycppclass, offset)
 
     def set(self, w_cppinstance, w_value):
         cppinstance = self.space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=True)
@@ -371,38 +375,38 @@
 
 class W_CPPScope(Wrappable):
     _immutable_ = True
-    _immutable_fields_ = ["methods[*]", "data_members[*]"]
+    _immutable_fields_ = ["methods[*]", "datamembers[*]"]
 
     kind = "scope"
 
-    def __init__(self, space, name, handle):
+    def __init__(self, space, name, opaque_handle):
         self.space = space
         self.name = name
-        assert lltype.typeOf(handle) == capi.C_SCOPE
-        self.handle = handle
+        assert lltype.typeOf(opaque_handle) == capi.C_SCOPE
+        self.handle = opaque_handle
         self.methods = {}
         # Do not call "self._find_methods()" here, so that a distinction can
         #  be made between testing for existence (i.e. existence in the cache
         #  of classes) and actual use. Point being that a class can use itself,
         #  e.g. as a return type or an argument to one of its methods.
 
-        self.data_members = {}
+        self.datamembers = {}
         # Idem self.methods: a type could hold itself by pointer.
 
     def _find_methods(self):
-        num_methods = capi.c_num_methods(self.handle)
+        num_methods = capi.c_num_methods(self)
         args_temp = {}
         for i in range(num_methods):
-            method_name = capi.c_method_name(self.handle, i)
+            method_name = capi.c_method_name(self, i)
             pymethod_name = helper.map_operator_name(
-                    method_name, capi.c_method_num_args(self.handle, i),
-                    capi.c_method_result_type(self.handle, i))
+                    method_name, capi.c_method_num_args(self, i),
+                    capi.c_method_result_type(self, i))
             if not pymethod_name in self.methods:
                 cppfunction = self._make_cppfunction(i)
                 overload = args_temp.setdefault(pymethod_name, [])
                 overload.append(cppfunction)
         for name, functions in args_temp.iteritems():
-            overload = W_CPPOverload(self.space, self.handle, name, functions[:])
+            overload = W_CPPOverload(self.space, self, functions[:])
             self.methods[name] = overload
 
     def get_method_names(self):
@@ -414,24 +418,31 @@
             return self.methods[name]
         except KeyError:
             pass
-        return self.find_overload(name)
+        new_method = self.find_overload(name)
+        self.methods[name] = new_method
+        return new_method
 
-    def get_data_member_names(self):
-        return self.space.newlist([self.space.wrap(name) for name in self.data_members])
+    def get_datamember_names(self):
+        return self.space.newlist([self.space.wrap(name) for name in self.datamembers])
 
     @jit.elidable_promote('0')
-    def get_data_member(self, name):
+    def get_datamember(self, name):
         try:
-            return self.data_members[name]
+            return self.datamembers[name]
         except KeyError:
             pass
-        return self.find_data_member(name)
+        new_dm = self.find_datamember(name)
+        self.datamembers[name] = new_dm
+        return new_dm
 
     def missing_attribute_error(self, name):
         return OperationError(
             self.space.w_AttributeError,
             self.space.wrap("%s '%s' has no attribute %s" % (self.kind, self.name, name)))
 
+    def __eq__(self, other):
+        return self.handle == other.handle
+
 
 # For now, keep namespaces and classes separate as namespaces are extensible
 # with info from multiple dictionaries and do not need to bother with meta
@@ -439,56 +450,54 @@
 # may be in order at some point.
 class W_CPPNamespace(W_CPPScope):
     _immutable_ = True
-
     kind = "namespace"
 
     def _make_cppfunction(self, method_index):
-        result_type = capi.c_method_result_type(self.handle, method_index)
-        num_args = capi.c_method_num_args(self.handle, method_index)
-        args_required = capi.c_method_req_args(self.handle, method_index)
+        result_type = capi.c_method_result_type(self, method_index)
+        num_args = capi.c_method_num_args(self, method_index)
+        args_required = capi.c_method_req_args(self, method_index)
         arg_defs = []
         for i in range(num_args):
-            arg_type = capi.c_method_arg_type(self.handle, method_index, i)
-            arg_dflt = capi.c_method_arg_default(self.handle, method_index, i)
+            arg_type = capi.c_method_arg_type(self, method_index, i)
+            arg_dflt = capi.c_method_arg_default(self, method_index, i)
             arg_defs.append((arg_type, arg_dflt))
-        return CPPFunction(self, method_index, result_type, arg_defs, args_required)
+        return CPPFunction(self.space, self, method_index, result_type, arg_defs, args_required)
 
-    def _make_data_member(self, dm_name, dm_idx):
-        type_name = capi.c_data_member_type(self.handle, dm_idx)
-        offset = capi.c_data_member_offset(self.handle, dm_idx)
-        data_member = W_CPPDataMember(self.space, self.handle, type_name, offset, True)
-        self.data_members[dm_name] = data_member
-        return data_member
+    def _make_datamember(self, dm_name, dm_idx):
+        type_name = capi.c_datamember_type(self, dm_idx)
+        offset = capi.c_datamember_offset(self, dm_idx)
+        datamember = W_CPPDataMember(self.space, self, type_name, offset, True)
+        self.datamembers[dm_name] = datamember
+        return datamember
 
-    def _find_data_members(self):
-        num_data_members = capi.c_num_data_members(self.handle)
-        for i in range(num_data_members):
-            if not capi.c_is_publicdata(self.handle, i):
+    def _find_datamembers(self):
+        num_datamembers = capi.c_num_datamembers(self)
+        for i in range(num_datamembers):
+            if not capi.c_is_publicdata(self, i):
                 continue
-            data_member_name = capi.c_data_member_name(self.handle, i)
-            if not data_member_name in self.data_members:
-                self._make_data_member(data_member_name, i)
+            datamember_name = capi.c_datamember_name(self, i)
+            if not datamember_name in self.datamembers:
+                self._make_datamember(datamember_name, i)
 
     def find_overload(self, meth_name):
         # TODO: collect all overloads, not just the non-overloaded version
-        meth_idx = capi.c_method_index(self.handle, meth_name)
+        meth_idx = capi.c_method_index(self, meth_name)
         if meth_idx < 0:
             raise self.missing_attribute_error(meth_name)
         cppfunction = self._make_cppfunction(meth_idx)
-        overload = W_CPPOverload(self.space, self.handle, meth_name, [cppfunction])
-        self.methods[meth_name] = overload
+        overload = W_CPPOverload(self.space, self, [cppfunction])
         return overload
 
-    def find_data_member(self, dm_name):
-        dm_idx = capi.c_data_member_index(self.handle, dm_name)
+    def find_datamember(self, dm_name):
+        dm_idx = capi.c_datamember_index(self, dm_name)
         if dm_idx < 0:
             raise self.missing_attribute_error(dm_name)
-        data_member = self._make_data_member(dm_name, dm_idx)
-        return data_member
+        datamember = self._make_datamember(dm_name, dm_idx)
+        return datamember
 
     def update(self):
         self._find_methods()
-        self._find_data_members()
+        self._find_datamembers()
 
     def is_namespace(self):
         return self.space.w_True
@@ -498,56 +507,55 @@
     update = interp2app(W_CPPNamespace.update, unwrap_spec=['self']),
     get_method_names = interp2app(W_CPPNamespace.get_method_names, unwrap_spec=['self']),
     get_overload = interp2app(W_CPPNamespace.get_overload, unwrap_spec=['self', str]),
-    get_data_member_names = interp2app(W_CPPNamespace.get_data_member_names, unwrap_spec=['self']),
-    get_data_member = interp2app(W_CPPNamespace.get_data_member, unwrap_spec=['self', str]),
+    get_datamember_names = interp2app(W_CPPNamespace.get_datamember_names, unwrap_spec=['self']),
+    get_datamember = interp2app(W_CPPNamespace.get_datamember, unwrap_spec=['self', str]),
     is_namespace = interp2app(W_CPPNamespace.is_namespace, unwrap_spec=['self']),
 )
 W_CPPNamespace.typedef.acceptable_as_base_class = False
 
 
-class W_CPPType(W_CPPScope):
+class W_CPPClass(W_CPPScope):
     _immutable_ = True
-
     kind = "class"
 
     def _make_cppfunction(self, method_index):
-        result_type = capi.c_method_result_type(self.handle, method_index)
-        num_args = capi.c_method_num_args(self.handle, method_index)
-        args_required = capi.c_method_req_args(self.handle, method_index)
+        result_type = capi.c_method_result_type(self, method_index)
+        num_args = capi.c_method_num_args(self, method_index)
+        args_required = capi.c_method_req_args(self, method_index)
         arg_defs = []
         for i in range(num_args):
-            arg_type = capi.c_method_arg_type(self.handle, method_index, i)
-            arg_dflt = capi.c_method_arg_default(self.handle, method_index, i)
+            arg_type = capi.c_method_arg_type(self, method_index, i)
+            arg_dflt = capi.c_method_arg_default(self, method_index, i)
             arg_defs.append((arg_type, arg_dflt))
-        if capi.c_is_constructor(self.handle, method_index):
+        if capi.c_is_constructor(self, method_index):
             result_type = "constructor"
             cls = CPPConstructor
-        elif capi.c_is_staticmethod(self.handle, method_index):
+        elif capi.c_is_staticmethod(self, method_index):
             cls = CPPFunction
         else:
             cls = CPPMethod
-        return cls(self, method_index, result_type, arg_defs, args_required)
+        return cls(self.space, self, method_index, result_type, arg_defs, args_required)
 
-    def _find_data_members(self):
-        num_data_members = capi.c_num_data_members(self.handle)
-        for i in range(num_data_members):
-            if not capi.c_is_publicdata(self.handle, i):
+    def _find_datamembers(self):
+        num_datamembers = capi.c_num_datamembers(self)
+        for i in range(num_datamembers):
+            if not capi.c_is_publicdata(self, i):
                 continue
-            data_member_name = capi.c_data_member_name(self.handle, i)
-            type_name = capi.c_data_member_type(self.handle, i)
-            offset = capi.c_data_member_offset(self.handle, i)
-            is_static = bool(capi.c_is_staticdata(self.handle, i))
-            data_member = W_CPPDataMember(self.space, self.handle, type_name, offset, is_static)
-            self.data_members[data_member_name] = data_member
+            datamember_name = capi.c_datamember_name(self, i)
+            type_name = capi.c_datamember_type(self, i)
+            offset = capi.c_datamember_offset(self, i)
+            is_static = bool(capi.c_is_staticdata(self, i))
+            datamember = W_CPPDataMember(self.space, self, type_name, offset, is_static)
+            self.datamembers[datamember_name] = datamember
 
     def find_overload(self, name):
         raise self.missing_attribute_error(name)
 
-    def find_data_member(self, name):
+    def find_datamember(self, name):
         raise self.missing_attribute_error(name)
 
-    def get_cppthis(self, cppinstance, scope_handle):
-        assert self.handle == cppinstance.cppclass.handle
+    def get_cppthis(self, cppinstance, calling_scope):
+        assert self == cppinstance.cppclass
         return cppinstance.get_rawobject()
 
     def is_namespace(self):
@@ -555,59 +563,59 @@
 
     def get_base_names(self):
         bases = []
-        num_bases = capi.c_num_bases(self.handle)
+        num_bases = capi.c_num_bases(self)
         for i in range(num_bases):
-            base_name = capi.c_base_name(self.handle, i)
+            base_name = capi.c_base_name(self, i)
             bases.append(self.space.wrap(base_name))
         return self.space.newlist(bases)
 
-W_CPPType.typedef = TypeDef(
-    'CPPType',
-    type_name = interp_attrproperty('name', W_CPPType),
-    get_base_names = interp2app(W_CPPType.get_base_names, unwrap_spec=['self']),
-    get_method_names = interp2app(W_CPPType.get_method_names, unwrap_spec=['self']),
-    get_overload = interp2app(W_CPPType.get_overload, unwrap_spec=['self', str]),
-    get_data_member_names = interp2app(W_CPPType.get_data_member_names, unwrap_spec=['self']),
-    get_data_member = interp2app(W_CPPType.get_data_member, unwrap_spec=['self', str]),
-    is_namespace = interp2app(W_CPPType.is_namespace, unwrap_spec=['self']),
+W_CPPClass.typedef = TypeDef(
+    'CPPClass',
+    type_name = interp_attrproperty('name', W_CPPClass),
+    get_base_names = interp2app(W_CPPClass.get_base_names, unwrap_spec=['self']),
+    get_method_names = interp2app(W_CPPClass.get_method_names, unwrap_spec=['self']),
+    get_overload = interp2app(W_CPPClass.get_overload, unwrap_spec=['self', str]),
+    get_datamember_names = interp2app(W_CPPClass.get_datamember_names, unwrap_spec=['self']),
+    get_datamember = interp2app(W_CPPClass.get_datamember, unwrap_spec=['self', str]),
+    is_namespace = interp2app(W_CPPClass.is_namespace, unwrap_spec=['self']),
 )
-W_CPPType.typedef.acceptable_as_base_class = False
+W_CPPClass.typedef.acceptable_as_base_class = False
 
 
-class W_ComplexCPPType(W_CPPType):
+class W_ComplexCPPClass(W_CPPClass):
     _immutable_ = True
 
-    def get_cppthis(self, cppinstance, scope_handle):
-        assert self.handle == cppinstance.cppclass.handle
-        offset = capi.c_base_offset(self.handle, scope_handle, cppinstance.get_rawobject())
+    def get_cppthis(self, cppinstance, calling_scope):
+        assert self == cppinstance.cppclass
+        offset = capi.c_base_offset(self, calling_scope, cppinstance.get_rawobject())
         return capi.direct_ptradd(cppinstance.get_rawobject(), offset)
 
-W_ComplexCPPType.typedef = TypeDef(
-    'ComplexCPPType',
-    type_name = interp_attrproperty('name', W_CPPType),
-    get_base_names = interp2app(W_ComplexCPPType.get_base_names, unwrap_spec=['self']),
-    get_method_names = interp2app(W_ComplexCPPType.get_method_names, unwrap_spec=['self']),
-    get_overload = interp2app(W_ComplexCPPType.get_overload, unwrap_spec=['self', str]),
-    get_data_member_names = interp2app(W_ComplexCPPType.get_data_member_names, unwrap_spec=['self']),
-    get_data_member = interp2app(W_ComplexCPPType.get_data_member, unwrap_spec=['self', str]),
-    is_namespace = interp2app(W_ComplexCPPType.is_namespace, unwrap_spec=['self']),
+W_ComplexCPPClass.typedef = TypeDef(
+    'ComplexCPPClass',
+    type_name = interp_attrproperty('name', W_CPPClass),
+    get_base_names = interp2app(W_ComplexCPPClass.get_base_names, unwrap_spec=['self']),
+    get_method_names = interp2app(W_ComplexCPPClass.get_method_names, unwrap_spec=['self']),
+    get_overload = interp2app(W_ComplexCPPClass.get_overload, unwrap_spec=['self', str]),
+    get_datamember_names = interp2app(W_ComplexCPPClass.get_datamember_names, unwrap_spec=['self']),
+    get_datamember = interp2app(W_ComplexCPPClass.get_datamember, unwrap_spec=['self', str]),
+    is_namespace = interp2app(W_ComplexCPPClass.is_namespace, unwrap_spec=['self']),
 )
-W_ComplexCPPType.typedef.acceptable_as_base_class = False
+W_ComplexCPPClass.typedef.acceptable_as_base_class = False
 
 
 class W_CPPTemplateType(Wrappable):
     _immutable_ = True
 
-    def __init__(self, space, name, handle):
+    def __init__(self, space, name, opaque_handle):
         self.space = space
         self.name = name
-        assert lltype.typeOf(handle) == capi.C_TYPE
-        self.handle = handle
+        assert lltype.typeOf(opaque_handle) == capi.C_TYPE
+        self.handle = opaque_handle
 
     def __call__(self, args_w):
         # TODO: this is broken but unused (see pythonify.py)
         fullname = "".join([self.name, '<', self.space.str_w(args_w[0]), '>'])
-        return type_byname(self.space, fullname)
+        return scope_byname(self.space, fullname)
 
 W_CPPTemplateType.typedef = TypeDef(
     'CPPTemplateType',
@@ -621,7 +629,6 @@
 
     def __init__(self, space, cppclass, rawobject, isref, python_owns):
         self.space = space
-        assert isinstance(cppclass, W_CPPType)
         self.cppclass = cppclass
         assert lltype.typeOf(rawobject) == capi.C_OBJECT
         assert not isref or rawobject
@@ -635,6 +642,9 @@
             raise OperationError(self.space.w_ReferenceError,
                                  self.space.wrap("trying to access a NULL pointer"))
 
+    def get_cppthis(self, calling_scope):
+        return self.cppclass.get_cppthis(self, calling_scope)
+
     def get_rawobject(self):
         if not self.isref:
             return self._rawobject
@@ -659,7 +669,7 @@
         assert isinstance(self, W_CPPInstance)
         if self._rawobject and not self.isref:
             memory_regulator.unregister(self)
-            capi.c_destruct(self.cppclass.handle, self._rawobject)
+            capi.c_destruct(self.cppclass, self._rawobject)
             self._rawobject = capi.C_NULL_OBJECT
 
     def __del__(self):
@@ -704,40 +714,40 @@
 memory_regulator = MemoryRegulator()
 
 
-def get_wrapped_type(space, handle):
+def get_pythonized_cppclass(space, handle):
     state = space.fromcache(State)
     try:
-        w_type = state.type_registry[handle]
+        w_pycppclass = state.cppclass_registry[handle]
     except KeyError:
         final_name = capi.c_scoped_final_name(handle)
-        w_type = space.call_function(state.w_clgen_callback, space.wrap(final_name))
-    return w_type
+        w_pycppclass = space.call_function(state.w_clgen_callback, space.wrap(final_name))
+    return w_pycppclass
 
-def wrap_new_cppobject_nocast(space, w_type, cpptype, rawobject, isref, python_owns):
-    if w_type is None:
-        w_type = get_wrapped_type(space, cpptype.handle)
-    w_cppinstance = space.allocate_instance(W_CPPInstance, w_type)
+def wrap_new_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns):
+    if space.is_w(w_pycppclass, space.w_None):
+        w_pycppclass = get_pythonized_cppclass(space, cppclass.handle)
+    w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass)
     cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False)
-    W_CPPInstance.__init__(cppinstance, space, cpptype, rawobject, isref, python_owns)
+    W_CPPInstance.__init__(cppinstance, space, cppclass, rawobject, isref, python_owns)
     memory_regulator.register(cppinstance)
     return w_cppinstance
 
-def wrap_cppobject_nocast(space, w_type, cpptype, rawobject, isref, python_owns):
+def wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns):
     obj = memory_regulator.retrieve(rawobject)
-    if obj and obj.cppclass == cpptype:
-         return obj
-    return wrap_new_cppobject_nocast(space, w_type, cpptype, rawobject, isref, python_owns)
+    if obj and obj.cppclass == cppclass:
+        return obj
+    return wrap_new_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns)
 
-def wrap_cppobject(space, w_type, cpptype, rawobject, isref, python_owns):
+def wrap_cppobject(space, w_pycppclass, cppclass, rawobject, isref, python_owns):
     if rawobject:
-        actual = capi.c_get_object_type(cpptype.handle, rawobject)
-        if actual != cpptype.handle:
-            offset = capi.c_base_offset(actual, cpptype.handle, rawobject)
+        actual = capi.c_actual_class(cppclass, rawobject)
+        if actual != cppclass.handle:
+            offset = capi._c_base_offset(actual, cppclass.handle, rawobject)
             rawobject = capi.direct_ptradd(rawobject, offset)
-            w_type = get_wrapped_type(space, actual)
-            w_cpptype = space.findattr(w_type, space.wrap("_cpp_proxy"))
-            cpptype = space.interp_w(W_CPPType, w_cpptype, can_be_None=False)
-    return wrap_cppobject_nocast(space, w_type, cpptype, rawobject, isref, python_owns)
+            w_pycppclass = get_pythonized_cppclass(space, actual)
+            w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+            cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
+    return wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns)
 
 @unwrap_spec(cppinstance=W_CPPInstance)
 def addressof(space, cppinstance):
@@ -745,8 +755,8 @@
      return space.wrap(address)
 
 @unwrap_spec(address=int, owns=bool)
-def bind_object(space, address, w_type, owns=False):
+def bind_object(space, address, w_pycppclass, owns=False):
     rawobject = rffi.cast(capi.C_OBJECT, address)
-    w_cpptype = space.findattr(w_type, space.wrap("_cpp_proxy"))
-    cpptype = space.interp_w(W_CPPType, w_cpptype, can_be_None=False)
-    return wrap_cppobject_nocast(space, w_type, cpptype, rawobject, False, owns)
+    w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
+    cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False)
+    return wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, False, owns)
diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -9,8 +9,8 @@
 class CppyyScopeMeta(type):
     def __getattr__(self, name):
         try:
-            return get_cppitem(self, name)  # will cache on self
-        except TypeError:
+            return get_pycppitem(self, name)  # will cache on self
+        except TypeError, t:
             raise AttributeError("%s object has no attribute '%s'" % (self, name))
 
 class CppyyNamespaceMeta(CppyyScopeMeta):
@@ -44,10 +44,10 @@
 
 
 def clgen_callback(name):
-    return get_cppclass(name)
+    return get_pycppclass(name)
 cppyy._set_class_generator(clgen_callback)
 
-def make_static_function(cpptype, func_name, cppol):
+def make_static_function(func_name, cppol):
     def function(*args):
         return cppol.call(None, *args)
     function.__name__ = func_name
@@ -61,13 +61,13 @@
     return method
 
 
-def make_data_member(cppdm):
+def make_datamember(cppdm):
     rettype = cppdm.get_returntype()
     if not rettype:                              # return builtin type
         cppclass = None
     else:                                        # return instance
         try:
-            cppclass = get_cppclass(rettype)
+            cppclass = get_pycppclass(rettype)
         except AttributeError:
             import warnings
             warnings.warn("class %s unknown: no data member access" % rettype,
@@ -96,7 +96,7 @@
     else:
         d = dict()
         def cpp_proxy_loader(cls):
-            cpp_proxy = cppyy._type_byname(cls.__name__ != '::' and cls.__name__ or '')
+            cpp_proxy = cppyy._scope_byname(cls.__name__ != '::' and cls.__name__ or '')
             del cls.__class__._cpp_proxy
             cls._cpp_proxy = cpp_proxy
             return cpp_proxy
@@ -111,14 +111,14 @@
         # insert static methods into the "namespace" dictionary
         for func_name in cppns.get_method_names():
             cppol = cppns.get_overload(func_name)
-            pyfunc = make_static_function(cppns, func_name, cppol)
+            pyfunc = make_static_function(func_name, cppol)
             setattr(pycppns, func_name, pyfunc)
 
         # add all data members to the dictionary of the class to be created, and
         # static ones also to the meta class (needed for property setters)
-        for dm in cppns.get_data_member_names():
-            cppdm = cppns.get_data_member(dm)
-            pydm = make_data_member(cppdm)
+        for dm in cppns.get_datamember_names():
+            cppdm = cppns.get_datamember(dm)
+            pydm = make_datamember(cppdm)
             setattr(pycppns, dm, pydm)
             setattr(metans, dm, pydm)
 
@@ -133,9 +133,9 @@
                 break
     return tuple(bases)
 
-def make_new(class_name, cpptype):
+def make_new(class_name, cppclass):
     try:
-        constructor_overload = cpptype.get_overload(cpptype.type_name)
+        constructor_overload = cppclass.get_overload(cppclass.type_name)
     except AttributeError:
         msg = "cannot instantiate abstract class '%s'" % class_name
         def __new__(cls, *args):
@@ -145,10 +145,10 @@
             return constructor_overload.call(None, *args)
     return __new__
 
-def make_cppclass(scope, class_name, final_class_name, cpptype):
+def make_pycppclass(scope, class_name, final_class_name, cppclass):
 
     # get a list of base classes for class creation
-    bases = [get_cppclass(base) for base in cpptype.get_base_names()]
+    bases = [get_pycppclass(base) for base in cppclass.get_base_names()]
     if not bases:
         bases = [CPPObject,]
     else:
@@ -164,57 +164,57 @@
     metacpp = type(CppyyClass)(class_name+'_meta', _drop_cycles(metabases), {})
 
     # create the python-side C++ class representation
-    d = {"_cpp_proxy" : cpptype,
-         "__new__"    : make_new(class_name, cpptype),
+    d = {"_cpp_proxy" : cppclass,
+         "__new__"    : make_new(class_name, cppclass),
          }
-    pycpptype = metacpp(class_name, _drop_cycles(bases), d)
+    pycppclass = metacpp(class_name, _drop_cycles(bases), d)
  
     # cache result early so that the class methods can find the class itself
-    setattr(scope, final_class_name, pycpptype)
+    setattr(scope, final_class_name, pycppclass)
 
     # insert (static) methods into the class dictionary
-    for meth_name in cpptype.get_method_names():
-        cppol = cpptype.get_overload(meth_name)
+    for meth_name in cppclass.get_method_names():
+        cppol = cppclass.get_overload(meth_name)
         if cppol.is_static():
-            setattr(pycpptype, meth_name, make_static_function(cpptype, meth_name, cppol))
+            setattr(pycppclass, meth_name, make_static_function(meth_name, cppol))
         else:
-            setattr(pycpptype, meth_name, make_method(meth_name, cppol))
+            setattr(pycppclass, meth_name, make_method(meth_name, cppol))
 
     # add all data members to the dictionary of the class to be created, and
     # static ones also to the meta class (needed for property setters)
-    for dm_name in cpptype.get_data_member_names():
-        cppdm = cpptype.get_data_member(dm_name)
-        pydm = make_data_member(cppdm)
+    for dm_name in cppclass.get_datamember_names():
+        cppdm = cppclass.get_datamember(dm_name)
+        pydm = make_datamember(cppdm)
 
-        setattr(pycpptype, dm_name, pydm)
+        setattr(pycppclass, dm_name, pydm)
         if cppdm.is_static():
             setattr(metacpp, dm_name, pydm)
 
-    _pythonize(pycpptype)
-    cppyy._register_class(pycpptype)
-    return pycpptype
+    _pythonize(pycppclass)
+    cppyy._register_class(pycppclass)
+    return pycppclass
 
 def make_cpptemplatetype(scope, template_name):
     return CppyyTemplateType(scope, template_name)
 
 
-def get_cppitem(scope, name):
+def get_pycppitem(scope, name):
     # resolve typedefs/aliases
     full_name = (scope == gbl) and name or (scope.__name__+'::'+name)
     true_name = cppyy._resolve_name(full_name)
     if true_name != full_name:
-        return get_cppclass(true_name)
+        return get_pycppclass(true_name)
 
     pycppitem = None
 
     # classes
-    cppitem = cppyy._type_byname(true_name)
+    cppitem = cppyy._scope_byname(true_name)
     if cppitem:
         if cppitem.is_namespace():
             pycppitem = make_cppnamespace(scope, true_name, cppitem)
             setattr(scope, name, pycppitem)
         else:
-            pycppitem = make_cppclass(scope, true_name, name, cppitem)
+            pycppitem = make_pycppclass(scope, true_name, name, cppitem)
 
     # templates
     if not cppitem:
@@ -227,7 +227,7 @@
     if not cppitem:
         try:
             cppitem = scope._cpp_proxy.get_overload(name)
-            pycppitem = make_static_function(scope._cpp_proxy, name, cppitem)
+            pycppitem = make_static_function(name, cppitem)
             setattr(scope.__class__, name, pycppitem)
             pycppitem = getattr(scope, name)      # binds function as needed
         except AttributeError:
@@ -236,8 +236,8 @@
     # data
     if not cppitem:
         try:
-            cppitem = scope._cpp_proxy.get_data_member(name)
-            pycppitem = make_data_member(cppitem)
+            cppitem = scope._cpp_proxy.get_datamember(name)
+            pycppitem = make_datamember(cppitem)
             setattr(scope, name, pycppitem)
             if cppitem.is_static():
                 setattr(scope.__class__, name, pycppitem)
@@ -266,7 +266,7 @@
         scope += c
     yield scope
 
-def get_cppclass(name):
+def get_pycppclass(name):
     # break up the name, to walk the scopes and get the class recursively
     scope = gbl
     for part in scope_splitter(name):
@@ -367,7 +367,7 @@
         return dct
     
 
-# user interface objects (note the two-step of not calling type_byname here:
+# user interface objects (note the two-step of not calling scope_byname here:
 # creation of global functions may cause the creation of classes in the global
 # namespace, so gbl must exist at that point to cache them)
 gbl = make_cppnamespace(None, "::", None, False)   # global C++ namespace
diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx
--- a/pypy/module/cppyy/src/cintcwrapper.cxx
+++ b/pypy/module/cppyy/src/cintcwrapper.cxx
@@ -231,7 +231,7 @@
     return (cppyy_type_t)sz;
 }
 
-cppyy_type_t cppyy_get_object_type(cppyy_type_t klass, cppyy_object_t obj) {
+cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj) {
     TClassRef cr = type_from_handle(klass);
     TClass* clActual = cr->GetActualClass( (void*)obj );
     if (clActual && clActual != cr.GetClass()) {
@@ -605,7 +605,7 @@
 
 
 /* data member reflection information ------------------------------------- */
-int cppyy_num_data_members(cppyy_scope_t handle) {
+int cppyy_num_datamembers(cppyy_scope_t handle) {
     TClassRef cr = type_from_handle(handle);
     if (cr.GetClass() && cr->GetListOfDataMembers())
         return cr->GetListOfDataMembers()->GetSize();
@@ -627,20 +627,20 @@
     return 0;
 }
 
-char* cppyy_data_member_name(cppyy_scope_t handle, int data_member_index) {
+char* cppyy_datamember_name(cppyy_scope_t handle, int datamember_index) {
     TClassRef cr = type_from_handle(handle);
     if (cr.GetClass()) {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(data_member_index);
+        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
         return cppstring_to_cstring(m->GetName());
     }
-    TGlobal& gbl = g_globalvars[data_member_index];
+    TGlobal& gbl = g_globalvars[datamember_index];
     return cppstring_to_cstring(gbl.GetName());
 }
 
-char* cppyy_data_member_type(cppyy_scope_t handle, int data_member_index) {
+char* cppyy_datamember_type(cppyy_scope_t handle, int datamember_index) {
     TClassRef cr = type_from_handle(handle);
     if (cr.GetClass())  {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(data_member_index);
+        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
         std::string fullType = m->GetFullTypeName();
         if ((int)m->GetArrayDim() > 1 || (!m->IsBasic() && m->IsaPointer()))
             fullType.append("*");
@@ -651,21 +651,21 @@
         }
         return cppstring_to_cstring(fullType);
     }
-    TGlobal& gbl = g_globalvars[data_member_index];
+    TGlobal& gbl = g_globalvars[datamember_index];
     return cppstring_to_cstring(gbl.GetFullTypeName());
 }
 
-size_t cppyy_data_member_offset(cppyy_scope_t handle, int data_member_index) {
+size_t cppyy_datamember_offset(cppyy_scope_t handle, int datamember_index) {
     TClassRef cr = type_from_handle(handle);
     if (cr.GetClass()) {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(data_member_index);
+        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
         return (size_t)m->GetOffsetCint();
     }
-    TGlobal& gbl = g_globalvars[data_member_index];
+    TGlobal& gbl = g_globalvars[datamember_index];
     return (size_t)gbl.GetAddress();
 }
 
-int cppyy_data_member_index(cppyy_scope_t handle, const char* name) {
+int cppyy_datamember_index(cppyy_scope_t handle, const char* name) {
     TClassRef cr = type_from_handle(handle);
     if (cr.GetClass()) {
         // called from updates; add a hard reset as the code itself caches in
@@ -700,19 +700,19 @@
 
 
 /* data member properties ------------------------------------------------  */
-int cppyy_is_publicdata(cppyy_scope_t handle, int data_member_index) {
+int cppyy_is_publicdata(cppyy_scope_t handle, int datamember_index) {
     TClassRef cr = type_from_handle(handle);
     if (cr.GetClass()) {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(data_member_index);
+        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
         return m->Property() & G__BIT_ISPUBLIC;
     }
     return 1;  // global data is always public
 }
 
-int cppyy_is_staticdata(cppyy_scope_t handle, int data_member_index) {
+int cppyy_is_staticdata(cppyy_scope_t handle, int datamember_index) {
     TClassRef cr = type_from_handle(handle);
     if (cr.GetClass()) {
-        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(data_member_index);
+        TDataMember* m = (TDataMember*)cr->GetListOfDataMembers()->At(datamember_index);
         return m->Property() & G__BIT_ISSTATIC;
     }
     return 1;  // global data is always static
diff --git a/pypy/module/cppyy/src/reflexcwrapper.cxx b/pypy/module/cppyy/src/reflexcwrapper.cxx
--- a/pypy/module/cppyy/src/reflexcwrapper.cxx
+++ b/pypy/module/cppyy/src/reflexcwrapper.cxx
@@ -71,7 +71,7 @@
    return (cppyy_type_t)tt.Id();
 }
 
-cppyy_type_t cppyy_get_object_type(cppyy_type_t klass, cppyy_object_t obj) {
+cppyy_type_t cppyy_actual_class(cppyy_type_t klass, cppyy_object_t obj) {
     Reflex::Type t = type_from_handle(klass);
     Reflex::Type tActual = t.DynamicType(Reflex::Object(t, (void*)obj));
     if (tActual && tActual != t) {
@@ -414,7 +414,7 @@
 
 
 /* data member reflection information ------------------------------------- */
-int cppyy_num_data_members(cppyy_scope_t handle) {
+int cppyy_num_datamembers(cppyy_scope_t handle) {
     Reflex::Scope s = scope_from_handle(handle);
     // fix enum representation by adding them to the containing scope as per C++
     // TODO: this (relatively harmlessly) dupes data members when updating in the
@@ -433,34 +433,34 @@
     return s.DataMemberSize();
 }
 
-char* cppyy_data_member_name(cppyy_scope_t handle, int data_member_index) {
+char* cppyy_datamember_name(cppyy_scope_t handle, int datamember_index) {
     Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(data_member_index);
+    Reflex::Member m = s.DataMemberAt(datamember_index);
     std::string name = m.Name();
     return cppstring_to_cstring(name);
 }
 
-char* cppyy_data_member_type(cppyy_scope_t handle, int data_member_index) {
+char* cppyy_datamember_type(cppyy_scope_t handle, int datamember_index) {
     Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(data_member_index);
+    Reflex::Member m = s.DataMemberAt(datamember_index);
     std::string name = m.TypeOf().Name(Reflex::FINAL|Reflex::SCOPED|Reflex::QUALIFIED);
     return cppstring_to_cstring(name);
 }
 
-size_t cppyy_data_member_offset(cppyy_scope_t handle, int data_member_index) {
+size_t cppyy_datamember_offset(cppyy_scope_t handle, int datamember_index) {
     Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(data_member_index);
+    Reflex::Member m = s.DataMemberAt(datamember_index);
     if (m.IsArtificial() && m.TypeOf().IsEnum())
         return (size_t)&m.InterpreterOffset();
     return m.Offset();
 }
 
-int cppyy_data_member_index(cppyy_scope_t handle, const char* name) {
+int cppyy_datamember_index(cppyy_scope_t handle, const char* name) {
     Reflex::Scope s = scope_from_handle(handle);
     // the following appears dumb, but the internal storage for Reflex is an
     // unsorted std::vector anyway, so there's no gain to be had in using the
     // Scope::DataMemberByName() function (which returns Member, not an index)
-    int num_dm = cppyy_num_data_members(handle);
+    int num_dm = cppyy_num_datamembers(handle);
     for (int idm = 0; idm < num_dm; ++idm) {
         Reflex::Member m = s.DataMemberAt(idm);
         if (m.Name() == name || m.Name(Reflex::FINAL) == name) {
@@ -474,15 +474,15 @@
 
 
 /* data member properties ------------------------------------------------  */
-int cppyy_is_publicdata(cppyy_scope_t handle, int data_member_index) {
+int cppyy_is_publicdata(cppyy_scope_t handle, int datamember_index) {
     Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(data_member_index);
+    Reflex::Member m = s.DataMemberAt(datamember_index);
     return m.IsPublic();
 }
 
-int cppyy_is_staticdata(cppyy_scope_t handle, int data_member_index) {
+int cppyy_is_staticdata(cppyy_scope_t handle, int datamember_index) {
     Reflex::Scope s = scope_from_handle(handle);
-    Reflex::Member m = s.DataMemberAt(data_member_index);
+    Reflex::Member m = s.DataMemberAt(datamember_index);
     return m.IsStatic();
 }
 
diff --git a/pypy/module/cppyy/test/bench1.py b/pypy/module/cppyy/test/bench1.py
--- a/pypy/module/cppyy/test/bench1.py
+++ b/pypy/module/cppyy/test/bench1.py
@@ -54,7 +54,7 @@
         import cppyy
         self.lib = cppyy.load_reflection_info("./example01Dict.so")
 
-        self.cls  = cppyy._type_byname("example01")
+        self.cls  = cppyy._scope_byname("example01")
         self.inst = self.cls.get_overload(self.cls.type_name).call(None, 0)
 
     def __call__(self):
diff --git a/pypy/module/cppyy/test/test_advancedcpp.py b/pypy/module/cppyy/test/test_advancedcpp.py
--- a/pypy/module/cppyy/test/test_advancedcpp.py
+++ b/pypy/module/cppyy/test/test_advancedcpp.py
@@ -228,7 +228,7 @@
         assert isinstance(c, gbl.some_concrete_class)
         assert isinstance(c, gbl.some_abstract_class)
 
-    def test06_data_members(self):
+    def test06_datamembers(self):
         """Test data member access when using virtual inheritence"""
 
         import cppyy
diff --git a/pypy/module/cppyy/test/test_cppyy.py b/pypy/module/cppyy/test/test_cppyy.py
--- a/pypy/module/cppyy/test/test_cppyy.py
+++ b/pypy/module/cppyy/test/test_cppyy.py
@@ -16,13 +16,15 @@
         raise OSError("'make' failed (see stderr)")
 
 class TestCPPYYImplementation:
-    def test_class_query(self):
+    def test01_class_query(self):
         dct = interp_cppyy.load_dictionary(space, test_dct)
-        w_cppyyclass = interp_cppyy.type_byname(space, "example01")
-        w_cppyyclass2 = interp_cppyy.type_byname(space, "example01")
+        w_cppyyclass = interp_cppyy.scope_byname(space, "example01")
+        w_cppyyclass2 = interp_cppyy.scope_byname(space, "example01")
         assert space.is_w(w_cppyyclass, w_cppyyclass2)
         adddouble = w_cppyyclass.methods["staticAddToDouble"]
         func, = adddouble.functions
+        assert func.executor is None
+        func._setup(None)     # creates executor
         assert isinstance(func.executor, executor.DoubleExecutor)
         assert func.arg_defs == [("double", "")]
 
@@ -34,7 +36,7 @@
         cls.w_example01, cls.w_payload = cls.space.unpackiterable(cls.space.appexec([], """():
             import cppyy
             cppyy.load_reflection_info(%r)
-            return cppyy._type_byname('example01'), cppyy._type_byname('payload')""" % (test_dct, )))
+            return cppyy._scope_byname('example01'), cppyy._scope_byname('payload')""" % (test_dct, )))
 
     def test01_static_int(self):
         """Test passing of an int, returning of an int, and overloading on a
diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py
--- a/pypy/module/cppyy/test/test_fragile.py
+++ b/pypy/module/cppyy/test/test_fragile.py
@@ -137,7 +137,7 @@
 
         g = fragile.G()
 
-    def test08_unhandled_scoped_data_member(self):
+    def test08_unhandled_scoped_datamember(self):
         """Test that an unhandled scoped data member does not cause infinite recursion"""
 
         import cppyy
@@ -190,4 +190,4 @@
 
         j = fragile.J()
         assert fragile.J.method1.__doc__ == j.method1.__doc__
-        assert j.method1.__doc__ == "fragile::J::method1(int, double)"
+        assert j.method1.__doc__ == "int fragile::J::method1(int, double)"
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -160,7 +160,7 @@
         drv = jit.JitDriver(greens=[], reds=["i", "inst", "addDataToInt"])
         def f():
             lib = interp_cppyy.load_dictionary(space, "./example01Dict.so")
-            cls  = interp_cppyy.type_byname(space, "example01")
+            cls  = interp_cppyy.scope_byname(space, "example01")
             inst = cls.get_overload("example01").call(None, [FakeInt(0)])
             addDataToInt = cls.get_overload("addDataToInt")
             assert isinstance(inst, interp_cppyy.W_CPPInstance)
@@ -185,7 +185,7 @@
         drv = jit.JitDriver(greens=[], reds=["i", "inst", "addDataToInt"])
         def f():
             lib = interp_cppyy.load_dictionary(space, "./example01Dict.so")
-            cls  = interp_cppyy.type_byname(space, "example01")
+            cls  = interp_cppyy.scope_byname(space, "example01")
             inst = cls.get_overload("example01").call(None, [FakeInt(0)])
             addDataToInt = cls.get_overload("overloadedAddDataToInt")
             assert isinstance(inst, interp_cppyy.W_CPPInstance)

From noreply at buildbot.pypy.org  Thu Apr  5 10:23:23 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  5 Apr 2012 10:23:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: minimark: Tweaks to reduce by one the
 number of checks in the common
Message-ID: <20120405082323.B71C08208A@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54194:01a73d86d3c7
Date: 2012-04-04 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/01a73d86d3c7/

Log:	minimark: Tweaks to reduce by one the number of checks in the common
	case.

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1426,23 +1426,25 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
-        # If 'obj' was already forwarded, change it to its forwarding address.
-        if self.is_forwarded(obj):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
+            #
+            # Common case: 'obj' was not already forwarded (otherwise
+            # tid == -42, containing all flags), and it doesn't have the
+            # HAS_SHADOW flag either.  We must move it out of the nursery,
+            # into a new nonmovable location.
+            totalsize = size_gc_header + self.get_size(obj)
+            newhdr = self._malloc_out_of_nursery(totalsize)
+            #
+        elif self.is_forwarded(obj):
+            #
+            # 'obj' was already forwarded.  Change the original reference
+            # to point to its forwarding address, and we're done.
             root.address[0] = self.get_forwarding_address(obj)
             return
-        #
-        # First visit to 'obj': we must move it out of the nursery.
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        size = self.get_size(obj)
-        totalsize = size_gc_header + size
-        #
-        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
-            #
-            # Common case: allocate a new nonmovable location for it.
-            newhdr = self._malloc_out_of_nursery(totalsize)
             #
         else:
-            # The object has already a shadow.
+            # First visit to an object that has already a shadow.
             newobj = self.nursery_objects_shadows.get(obj)
             ll_assert(newobj != NULL, "GCFLAG_HAS_SHADOW but no shadow found")
             newhdr = newobj - size_gc_header
@@ -1450,6 +1452,8 @@
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
             # copied to the shadow itself.
             self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            #
+            totalsize = size_gc_header + self.get_size(obj)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.

From noreply at buildbot.pypy.org  Thu Apr  5 10:23:25 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu,  5 Apr 2012 10:23:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120405082325.99FE28208A@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54195:057219807d73
Date: 2012-04-05 10:14 +0200
http://bitbucket.org/pypy/pypy/changeset/057219807d73/

Log:	merge heads

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -105,6 +105,7 @@
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -132,6 +133,8 @@
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
         ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -546,6 +546,7 @@
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
+            ("trunc", "trunc", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
             ("exp2", "exp2", 1, {"promote_to_float": True}),
             ("expm1", "expm1", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -253,24 +253,17 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floorceil(self):
-        from _numpypy import array, floor, ceil
+    def test_floorceiltrunc(self):
+        from _numpypy import array, floor, ceil, trunc
         import math
-        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = floor(a)
-        for i in range(5):
-            assert b[i] == reference[i]
-        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = ceil(a)
-        assert (reference == b).all()
-        inf = float("inf")
-        data = [1.5, 2.9999, -1.999, inf]
-        results = [math.floor(x) for x in data]
-        assert (floor(data) == results).all()
-        results = [math.ceil(x) for x in data]
-        assert (ceil(data) == results).all()
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == floor(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
+        assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -597,6 +590,13 @@
         assert (bitwise_not(a) == ~a).all()
         assert (invert(a) == ~a).all()
 
+    def test_shift(self):
+        from _numpypy import left_shift, right_shift
+        import sys
+
+        assert (left_shift([5, 1], [2, 31]) == [20, 2**31]).all()
+        assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -668,6 +668,13 @@
         return math.ceil(v)
 
     @simple_unary_op
+    def trunc(self, v):
+        if v < 0:
+            return math.ceil(v)
+        else:
+            return math.floor(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
diff --git a/pypy/rlib/rsre/rsre_re.py b/pypy/rlib/rsre/rsre_re.py
--- a/pypy/rlib/rsre/rsre_re.py
+++ b/pypy/rlib/rsre/rsre_re.py
@@ -172,8 +172,9 @@
         self._ctx = ctx
 
     def span(self, groupnum=0):
-        if not isinstance(groupnum, (int, long)):
-            groupnum = self.re.groupindex[groupnum]
+#        if not isinstance(groupnum, (int, long)):
+#            groupnum = self.re.groupindex[groupnum]
+ 
         return self._ctx.span(groupnum)
 
     def start(self, groupnum=0):
@@ -182,19 +183,25 @@
     def end(self, groupnum=0):
         return self.span(groupnum)[1]
 
-    def group(self, *groups):
-        groups = groups or (0,)
-        result = []
-        for group in groups:
-            frm, to = self.span(group)
-            if 0 <= frm <= to:
-                result.append(self._ctx._string[frm:to])
-            else:
-                result.append(None)
-        if len(result) > 1:
-            return tuple(result)
+    def group(self, group=0):
+        frm, to = self.span(group)
+        if 0 <= frm <= to:
+            return self._ctx._string[frm:to]
         else:
-            return result[0]
+            return None
+
+#    def group(self, *groups):
+#        groups = groups or (0,)
+#        result = []
+#        for group in groups:
+#            frm, to = self.span(group)
+#            if 0 <= frm <= to:
+#                result.append(self._ctx._string[frm:to])
+#            else:
+#                result.append(None)
+#        if len(result) > 1:
+#            return tuple(result)
+
 
     def groups(self, default=None):
         fmarks = self._ctx.flatten_marks()
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -204,7 +204,7 @@
         assert re.match('(a)', 'a').groups() == ('a',)
         assert re.match(r'(a)', 'a').group(0) == 'a'
         assert re.match(r'(a)', 'a').group(1) == 'a'
-        assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
+        #assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
 
         pat = re.compile('((a)|(b))(c)?')
         assert pat.match('a').groups() == ('a', 'a', None, None)
@@ -218,13 +218,13 @@
         assert m.group(0) == 'a'
         assert m.group(0) == 'a'
         assert m.group(1) == 'a'
-        assert m.group(1, 1) == ('a', 'a')
+        #assert m.group(1, 1) == ('a', 'a')
 
         pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?')
-        assert pat.match('a').group(1, 2, 3) == ('a', None, None)
-        assert pat.match('b').group('a1', 'b2', 'c3') == (
-                         (None, 'b', None))
-        assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
+        #assert pat.match('a').group(1, 2, 3) == ('a', None, None)
+        #assert pat.match('b').group('a1', 'b2', 'c3') == (
+        #                 (None, 'b', None))
+        #assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
     def test_bug_923(self):
         # Issue923: grouping inside optional lookahead problem
diff --git a/pypy/rlib/rsre/test/test_zinterp.py b/pypy/rlib/rsre/test/test_zinterp.py
--- a/pypy/rlib/rsre/test/test_zinterp.py
+++ b/pypy/rlib/rsre/test/test_zinterp.py
@@ -1,7 +1,8 @@
 # minimal test: just checks that (parts of) rsre can be translated
 
-from pypy.rpython.test.test_llinterp import gengraph
+from pypy.rpython.test.test_llinterp import gengraph, interpret
 from pypy.rlib.rsre import rsre_core
+from pypy.rlib.rsre.rsre_re import compile
 
 def main(n):
     assert n >= 0
@@ -19,3 +20,18 @@
 
 def test_gengraph():
     t, typer, graph = gengraph(main, [int])
+
+m = compile("(a|b)aaaaa")
+
+def test_match():
+    def f(i):
+        if i:
+            s = "aaaaaa"
+        else:
+            s = "caaaaa"
+        g = m.match(s)
+        if g is None:
+            return 3
+        return int("aaaaaa" == g.group(0))
+    assert interpret(f, [3]) == 1
+    assert interpret(f, [0]) == 3

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:02 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:02 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: combine all the mixins for
 operation implementations and change the assembler setup
Message-ID: <20120405121902.B57E08208B@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54197:3f8b478d181b
Date: 2012-03-27 12:21 +0000
http://bitbucket.org/pypy/pypy/changeset/3f8b478d181b/

Log:	combine all the mixins for operation implementations and change the
	assembler setup

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -14,6 +14,7 @@
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
 from pypy.jit.backend.model import CompiledLoopToken
 from pypy.jit.codewriter import longlong
+from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
 from pypy.jit.metainterp.history import BoxInt, ConstInt
 from pypy.jit.metainterp.resoperation import rop, ResOperation
@@ -1264,24 +1265,25 @@
 
 asm_operations = [notimplemented_op] * (rop._LAST + 1)
 asm_operations_with_guard = [notimplemented_op_with_guard] * (rop._LAST + 1)
+asm_math_operations = {}
 
-for key, value in rop.__dict__.items():
-    key = key.lower()
-    if key.startswith('_'):
-        continue
-    methname = 'emit_op_%s' % key
-    if hasattr(AssemblerARM, methname):
-        func = getattr(AssemblerARM, methname).im_func
-        asm_operations[value] = func
-
-for key, value in rop.__dict__.items():
-    key = key.lower()
-    if key.startswith('_'):
-        continue
-    methname = 'emit_guard_%s' % key
-    if hasattr(AssemblerARM, methname):
-        func = getattr(AssemblerARM, methname).im_func
-        asm_operations_with_guard[value] = func
+for name, value in ResOpAssembler.__dict__.iteritems():
+    if name.startswith('emit_guard_'):
+        opname = name[len('emit_guard_'):]
+        num = getattr(rop, opname.upper())
+        asm_operations_with_guard[num] = value
+    elif name.startswith('emit_op_llong_'):
+        opname = name[len('emit_op_llong_'):]
+        num = getattr(EffectInfo, 'OS_LLONG_' + opname.upper())
+        asm_llong_operations[num] = value
+    elif name.startswith('emit_op_math_'):
+        opname = name[len('emit_op_math_'):]
+        num = getattr(EffectInfo, 'OS_MATH_' + opname.upper())
+        asm_math_operations[num] = value
+    elif name.startswith('emit_op_'):
+        opname = name[len('emit_op_'):]
+        num = getattr(rop, opname.upper())
+        asm_operations[num] = value
 
 
 class BridgeAlreadyCompiled(Exception):
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -45,9 +45,7 @@
         self.fcond = fcond
 
 
-class IntOpAsslember(object):
-
-    _mixin_ = True
+class ResOpAssembler(object):
 
     def emit_op_int_add(self, op, arglocs, regalloc, fcond, flags=False):
         l0, l1, res = arglocs
@@ -162,10 +160,6 @@
     emit_op_int_sub_ovf = emit_op_int_sub
 
 
-class UnaryIntOpAssembler(object):
-
-    _mixin_ = True
-
     emit_op_int_is_true = gen_emit_op_unary_cmp('int_is_true', c.NE)
     emit_op_int_is_zero = gen_emit_op_unary_cmp('int_is_zero', c.EQ)
 
@@ -184,10 +178,6 @@
         return fcond
 
 
-class GuardOpAssembler(object):
-
-    _mixin_ = True
-
     def _emit_guard(self, op, arglocs, fcond, save_exc,
                                     is_guard_not_invalidated=False):
         assert isinstance(save_exc, bool)
@@ -291,10 +281,6 @@
                                             is_guard_not_invalidated=True)
 
 
-class OpAssembler(object):
-
-    _mixin_ = True
-
     def emit_op_jump(self, op, arglocs, regalloc, fcond):
         # The backend's logic assumes that the target code is in a piece of
         # assembler that was also called with the same number of arguments,
@@ -616,10 +602,6 @@
     emit_op_cond_call_gc_wb_array = emit_op_cond_call_gc_wb
 
 
-class FieldOpAssembler(object):
-
-    _mixin_ = True
-
     def emit_op_setfield_gc(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs, size = arglocs
         if size.value == 8:
@@ -760,10 +742,6 @@
     emit_op_setinteriorfield_raw = emit_op_setinteriorfield_gc
 
 
-class ArrayOpAssember(object):
-
-    _mixin_ = True
-
     def emit_op_arraylen_gc(self, op, arglocs, regalloc, fcond):
         res, base_loc, ofs = arglocs
         self.mc.LDR_ri(res.value, base_loc.value, ofs.value)
@@ -846,10 +824,6 @@
     emit_op_getarrayitem_gc_pure = emit_op_getarrayitem_gc
 
 
-class StrOpAssembler(object):
-
-    _mixin_ = True
-
     def emit_op_strlen(self, op, arglocs, regalloc, fcond):
         l0, l1, res = arglocs
         if l1.is_imm():
@@ -998,11 +972,7 @@
             raise AssertionError("bad unicode item size")
 
 
-class UnicodeOpAssembler(object):
-
-    _mixin_ = True
-
-    emit_op_unicodelen = StrOpAssembler.emit_op_strlen
+    emit_op_unicodelen = emit_op_strlen
 
     def emit_op_unicodegetitem(self, op, arglocs, regalloc, fcond):
         res, base_loc, ofs_loc, scale, basesize, itemsize = arglocs
@@ -1032,10 +1002,6 @@
         return fcond
 
 
-class ForceOpAssembler(object):
-
-    _mixin_ = True
-
     def emit_op_force_token(self, op, arglocs, regalloc, fcond):
         res_loc = arglocs[0]
         self.mc.MOV_rr(res_loc.value, r.fp.value)
@@ -1248,10 +1214,6 @@
         self.mc.STR_ri(r.ip.value, r.fp.value)
 
 
-class AllocOpAssembler(object):
-
-    _mixin_ = True
-
     def emit_op_call_malloc_gc(self, op, arglocs, regalloc, fcond):
         self.emit_op_call(op, arglocs, regalloc, fcond)
         self.propagate_memoryerror_if_r0_is_null()
@@ -1282,9 +1244,6 @@
         self.mc.NOP()
 
 
-class FloatOpAssemlber(object):
-    _mixin_ = True
-
     emit_op_float_add = gen_emit_float_op('float_add', 'VADD')
     emit_op_float_sub = gen_emit_float_op('float_sub', 'VSUB')
     emit_op_float_mul = gen_emit_float_op('float_mul', 'VMUL')
@@ -1324,10 +1283,3 @@
         return fcond
 
 
-class ResOpAssembler(GuardOpAssembler, IntOpAsslember,
-                    OpAssembler, UnaryIntOpAssembler,
-                    FieldOpAssembler, ArrayOpAssember,
-                    StrOpAssembler, UnicodeOpAssembler,
-                    ForceOpAssembler, AllocOpAssembler,
-                    FloatOpAssemlber):
-    pass
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -543,7 +543,7 @@
             oopspecindex = effectinfo.oopspecindex
             if oopspecindex == EffectInfo.OS_MATH_SQRT:
                 args = self.prepare_op_math_sqrt(op, fcond)
-                self.assembler.emit_op_math_sqrt(op, args, self, fcond)
+                self.perform_math(op, args, fcond)
                 return
         return self._prepare_call(op)
 

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:01 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:01 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: merge default
Message-ID: <20120405121901.6AF608208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54196:02a8668f1393
Date: 2012-03-27 08:14 +0000
http://bitbucket.org/pypy/pypy/changeset/02a8668f1393/

Log:	merge default

diff --git a/lib-python/modified-2.7/test/test_set.py b/lib-python/modified-2.7/test/test_set.py
--- a/lib-python/modified-2.7/test/test_set.py
+++ b/lib-python/modified-2.7/test/test_set.py
@@ -1568,7 +1568,7 @@
             for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint):
                 for g in (G, I, Ig, L, R):
                     expected = meth(data)
-                    actual = meth(G(data))
+                    actual = meth(g(data))
                     if isinstance(expected, bool):
                         self.assertEqual(actual, expected)
                     else:
diff --git a/pypy/doc/discussion/win64_todo.txt b/pypy/doc/discussion/win64_todo.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/discussion/win64_todo.txt
@@ -0,0 +1,9 @@
+2011-11-04
+ll_os.py has a problem with the file rwin32.py.
+Temporarily disabled for the win64_gborg branch. This needs to be
+investigated and re-enabled.
+Resolved, enabled.
+
+2011-11-05
+test_typed.py needs explicit tests to ensure that we
+handle word sizes right.
\ No newline at end of file
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -18,7 +18,8 @@
 Edition.  Other configurations may work as well.
 
 The translation scripts will set up the appropriate environment variables
-for the compiler.  They will attempt to locate the same compiler version that
+for the compiler, so you do not need to run vcvars before translation.  
+They will attempt to locate the same compiler version that
 was used to build the Python interpreter doing the
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
@@ -26,7 +27,7 @@
 using a 32 bit Python and vice versa.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
-will be aborted in this case.
+will fail in this case.
 
 The compiler is all you need to build pypy-c, but it will miss some
 modules that relies on third-party libraries.  See below how to get
@@ -57,7 +58,8 @@
 install third-party libraries.  We chose to install them in the parent
 directory of the pypy checkout.  For example, if you installed pypy in
 ``d:\pypy\trunk\`` (This directory contains a README file), the base
-directory is ``d:\pypy``.
+directory is ``d:\pypy``. You may choose different values by setting the
+INCLUDE, LIB and PATH (for DLLs)
 
 The Boehm garbage collector
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -126,18 +128,54 @@
 ------------------------
 
 You can compile pypy with the mingw compiler, using the --cc=mingw32 option;
-mingw.exe must be on the PATH.
+gcc.exe must be on the PATH. If the -cc flag does not begin with "ming", it should be
+the name of a valid gcc-derivative compiler, i.e. x86_64-w64-mingw32-gcc for the 64 bit
+compiler creating a 64 bit target.
 
-libffi for the mingw32 compiler
+libffi for the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To enable the _rawffi (and ctypes) module, you need to compile a mingw32
-version of libffi.  I downloaded the `libffi source files`_, and extracted
-them in the base directory.  Then run::
+To enable the _rawffi (and ctypes) module, you need to compile a mingw
+version of libffi.  Here is one way to do this, wich should allow you to try
+to build for win64 or win32:
+
+#. Download and unzip a `mingw32 build`_ or `mingw64 build`_, say into c:\mingw
+#. If you do not use cygwin, you will need msys to provide make, 
+   autoconf tools and other goodies.
+
+    #. Download and unzip a `msys for mingw`_, say into c:\msys
+    #. Edit the c:\msys\etc\fstab file to mount c:\mingw
+
+#. Download and unzip the `libffi source files`_, and extract
+   them in the base directory.  
+#. Run c:\msys\msys.bat or a cygwin shell which should make you
+   feel better since it is a shell prompt with shell tools.
+#. From inside the shell, cd to the libffi directory and do::
 
     sh ./configure
     make
     cp .libs/libffi-5.dll <somewhere on the PATH>
 
+If you can't find the dll, and the libtool issued a warning about 
+"undefined symbols not allowed", you will need to edit the libffi
+Makefile in the toplevel directory. Add the flag -no-undefined to
+the definition of libffi_la_LDFLAGS
+
+If you wish to experiment with win64, you must run configure with flags::
+
+    sh ./configure --build=x86_64-w64-mingw32 --host=x86_64-w64-mingw32
+
+or such, depending on your mingw64 download.
+
+hacking on Pypy with the mingw compiler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Since hacking on Pypy means running tests, you will need a way to specify
+the mingw compiler when hacking (as opposed to translating). As of
+March 2012, --cc is not a valid option for pytest.py. However if you set an
+environment variable CC it will allow you to choose a compiler.
+
+.. _'mingw32 build': http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Automated%20Builds
+.. _`mingw64 build`: http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Automated%20Builds
+.. _`msys for mingw`: http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/MSYS%20%2832-bit%29   
 .. _`libffi source files`: http://sourceware.org/libffi/
 .. _`RPython translation toolchain`: translation.html
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/you-want-to-help.rst
@@ -0,0 +1,86 @@
+
+You want to help with PyPy, now what?
+=====================================
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  the development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy
+  approach is great if you do TDD, not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below
+
+Architecture
+============
+
+PyPy has layers. The 100 miles view:
+
+* `RPython`_ is the language in which we write interpreters. Not the entire
+  PyPy project is written in RPython, only the parts that are compiled in
+  the translation process. The interesting point is that RPython has no parser,
+  it's compiled from the live python objects, which make it possible to do
+  all kinds of metaprogramming during import time. In short, Python is a meta
+  programming language for RPython.
+
+  The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+.. _`RPython`: coding-guide.html#RPython
+
+* The translation toolchain - this is the part that takes care about translating
+  RPython to flow graphs and then to C. There is more in the `architecture`_
+  document written about it.
+
+  It mostly lives in ``rpython``, ``annotator`` and ``objspace/flow``.
+
+.. _`architecture`: architecture.html 
+
+* Python Interpreter
+
+  xxx
+
+* Python modules
+
+  xxx
+
+* Just-in-Time Compiler (JIT): `we have a tracing JIT`_ that traces the
+  interpreter written in RPython, rather than the user program that it
+  interprets.  As a result it applies to any interpreter, i.e. any
+  language.  But getting it to work correctly is not trivial: it
+  requires a small number of precise "hints" and possibly some small
+  refactorings of the interpreter.  The JIT itself also has several
+  almost-independent parts: the tracer itself in ``jit/metainterp``, the
+  optimizer in ``jit/metainterp/optimizer`` that optimizes a list of
+  residual operations, and the backend in ``jit/backend/<machine-name>``
+  that turns it into machine code.  Writing a new backend is a
+  traditional way to get into the project.
+
+.. _`we have a tracing JIT`: jit/index.html
+
+* Garbage Collectors (GC): as you can notice if you are used to CPython's
+  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+  `Garbage collection in PyPy`_ is inserted
+  during translation.  Moreover, this is not reference counting; it is a real
+  GC written as more RPython code.  The best one we have so far is in
+  ``rpython/memory/gc/minimark.py``.
+
+.. _`Garbage collection in PyPy`: garbage_collection.html
+
+
+Toolset
+=======
+
+xxx
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -10,16 +10,6 @@
 from pypy.interpreter.astcompiler import ast, consts
 
 
-try:
-    all
-except NameError:
-    def all(iterable):
-        for x in iterable:
-            if not x:
-                return False
-        return True
-
-
 class TestAstBuilder:
 
     def setup_class(cls):
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -715,8 +715,9 @@
             self.Perform(op, [loc0], loc1)
             self.xrm.possibly_free_var(op.getarg(0))
         else:
-            loc0 = self.xrm.loc(op.getarg(0))
-            loc1 = self.xrm.force_allocate_reg(op.result)
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.loc(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
             self.Perform(op, [loc0], loc1)
             self.xrm.possibly_free_var(op.getarg(0))
 
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -16,13 +16,15 @@
     appleveldefs = {}
     interpleveldefs = {}
     if sys.platform.startswith("linux"):
+        from pypy.module.__pypy__ import interp_time
         interpleveldefs["clock_gettime"] = "interp_time.clock_gettime"
         interpleveldefs["clock_getres"] = "interp_time.clock_getres"
         for name in [
             "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW",
             "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID"
         ]:
-            interpleveldefs[name] = "space.wrap(interp_time.%s)" % name
+            if getattr(interp_time, name) is not None:
+                interpleveldefs[name] = "space.wrap(interp_time.%s)" % name
 
 
 class Module(MixedModule):
diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
--- a/pypy/module/__pypy__/interp_time.py
+++ b/pypy/module/__pypy__/interp_time.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import sys
 
 from pypy.interpreter.error import exception_from_errno
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -16,7 +16,7 @@
         a[i][i] = 1
     return a
 
-def sum(a,axis=None):
+def sum(a,axis=None, out=None):
     '''sum(a, axis=None)
     Sum of array elements over a given axis.
 
@@ -43,17 +43,17 @@
     # TODO: add to doc (once it's implemented): cumsum : Cumulative sum of array elements.
     if not hasattr(a, "sum"):
         a = _numpypy.array(a)
-    return a.sum(axis)
+    return a.sum(axis=axis, out=out)
 
-def min(a, axis=None):
+def min(a, axis=None, out=None):
     if not hasattr(a, "min"):
         a = _numpypy.array(a)
-    return a.min(axis)
+    return a.min(axis=axis, out=out)
 
-def max(a, axis=None):
+def max(a, axis=None, out=None):
     if not hasattr(a, "max"):
         a = _numpypy.array(a)
-    return a.max(axis)
+    return a.max(axis=axis, out=out)
 
 def arange(start, stop=None, step=1, dtype=None):
     '''arange([start], stop[, step], dtype=None)
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -62,21 +62,24 @@
         return space.wrap(dtype.itemtype.bool(self))
 
     def _binop_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                            [self, w_other, w_out])
         return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
 
     def _binop_right_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, 
+                                                            [w_other, self, w_out])
         return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
 
     def _unaryop_impl(ufunc_name):
-        def impl(self, space):
+        def impl(self, space, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                                    [self, w_out])
         return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name)
 
     descr_add = _binop_impl("add")
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -269,7 +269,7 @@
 
     def apply_transformations(self, arr, transformations):
         v = BaseIterator.apply_transformations(self, arr, transformations)
-        if len(arr.shape) == 1:
+        if len(arr.shape) == 1 and len(v.res_shape) == 1:
             return OneDimIterator(self.offset, self.strides[0],
                                   self.res_shape[0])
         return v
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -83,8 +83,9 @@
         return space.wrap(W_NDimArray(shape[:], dtype=dtype))
 
     def _unaryop_impl(ufunc_name):
-        def impl(self, space):
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
+        def impl(self, space, w_out=None):
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                                [self, w_out])
         return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name)
 
     descr_pos = _unaryop_impl("positive")
@@ -93,8 +94,9 @@
     descr_invert = _unaryop_impl("invert")
 
     def _binop_impl(ufunc_name):
-        def impl(self, space, w_other):
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+        def impl(self, space, w_other, w_out=None):
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                        [self, w_other, w_out])
         return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
 
     descr_add = _binop_impl("add")
@@ -124,12 +126,12 @@
         return space.newtuple([w_quotient, w_remainder])
 
     def _binop_right_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             w_other = scalar_w(space,
                 interp_ufuncs.find_dtype_for_scalar(space, w_other, self.find_dtype()),
                 w_other
             )
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self, w_out])
         return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
 
     descr_radd = _binop_right_impl("add")
@@ -152,13 +154,21 @@
         return space.newtuple([w_quotient, w_remainder])
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
-        def impl(self, space, w_axis=None):
+        def impl(self, space, w_axis=None, w_out=None):
             if space.is_w(w_axis, space.w_None):
                 axis = -1
             else:
                 axis = space.int_w(w_axis)
+            if space.is_w(w_out, space.w_None) or not w_out:
+                out = None
+            elif not isinstance(w_out, BaseArray):
+                raise OperationError(space.w_TypeError, space.wrap( 
+                        'output must be an array'))
+            else:
+                out = w_out
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
-                                        self, True, promote_to_largest, axis)
+                                        self, True, promote_to_largest, axis,
+                                                                   False, out)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
@@ -213,6 +223,7 @@
     def descr_dot(self, space, w_other):
         other = convert_to_array(space, w_other)
         if isinstance(other, Scalar):
+            #Note: w_out is not modified, this is numpy compliant.
             return self.descr_mul(space, other)
         elif len(self.shape) < 2 and len(other.shape) < 2:
             w_res = self.descr_mul(space, other)
@@ -514,14 +525,14 @@
             )
         return w_result
 
-    def descr_mean(self, space, w_axis=None):
+    def descr_mean(self, space, w_axis=None, w_out=None):
         if space.is_w(w_axis, space.w_None):
             w_axis = space.wrap(-1)
             w_denom = space.wrap(support.product(self.shape))
         else:
             dim = space.int_w(w_axis)
             w_denom = space.wrap(self.shape[dim])
-        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
+        return space.div(self.descr_sum_promote(space, w_axis, w_out), w_denom)
 
     def descr_var(self, space, w_axis=None):
         return get_appbridge_cache(space).call_method(space, '_var', self,
@@ -714,11 +725,12 @@
     """
     Class for representing virtual arrays, such as binary ops or ufuncs
     """
-    def __init__(self, name, shape, res_dtype):
+    def __init__(self, name, shape, res_dtype, out_arg=None):
         BaseArray.__init__(self, shape)
         self.forced_result = None
         self.res_dtype = res_dtype
         self.name = name
+        self.res = out_arg
         self.size = support.product(self.shape) * res_dtype.get_size()
 
     def _del_sources(self):
@@ -727,13 +739,18 @@
         raise NotImplementedError
 
     def compute(self):
-        ra = ResultArray(self, self.shape, self.res_dtype)
+        ra = ResultArray(self, self.shape, self.res_dtype, self.res)
         loop.compute(ra)
+        if self.res:
+            broadcast_dims = len(self.res.shape) - len(self.shape)
+            chunks = [Chunk(0,0,0,0)] * broadcast_dims + \
+                     [Chunk(0, i, 1, i) for i in self.shape]
+            return Chunks(chunks).apply(self.res)
         return ra.left
 
     def force_if_needed(self):
         if self.forced_result is None:
-            self.forced_result = self.compute()
+            self.forced_result = self.compute().get_concrete()
             self._del_sources()
 
     def get_concrete(self):
@@ -773,8 +790,9 @@
 
 
 class Call1(VirtualArray):
-    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values):
-        VirtualArray.__init__(self, name, shape, res_dtype)
+    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values,
+                                                            out_arg=None):
+        VirtualArray.__init__(self, name, shape, res_dtype, out_arg)
         self.values = values
         self.size = values.size
         self.ufunc = ufunc
@@ -786,6 +804,12 @@
     def create_sig(self):
         if self.forced_result is not None:
             return self.forced_result.create_sig()
+        if self.shape != self.values.shape:
+            #This happens if out arg is used
+            return signature.BroadcastUfunc(self.ufunc, self.name,
+                                            self.calc_dtype,
+                                            self.values.create_sig(),
+                                            self.res.create_sig())
         return signature.Call1(self.ufunc, self.name, self.calc_dtype,
                                self.values.create_sig())
 
@@ -793,8 +817,9 @@
     """
     Intermediate class for performing binary operations.
     """
-    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
-        VirtualArray.__init__(self, name, shape, res_dtype)
+    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right,
+            out_arg=None):
+        VirtualArray.__init__(self, name, shape, res_dtype, out_arg)
         self.ufunc = ufunc
         self.left = left
         self.right = right
@@ -832,8 +857,13 @@
         Call2.__init__(self, None, 'assign', shape, dtype, dtype, res, child)
 
     def create_sig(self):
-        return signature.ResultSignature(self.res_dtype, self.left.create_sig(),
-                                         self.right.create_sig())
+        if self.left.shape != self.right.shape:
+            sig = signature.BroadcastResultSignature(self.res_dtype,
+                        self.left.create_sig(), self.right.create_sig())
+        else:
+            sig = signature.ResultSignature(self.res_dtype, 
+                        self.left.create_sig(), self.right.create_sig())
+        return sig
 
 class ToStringArray(Call1):
     def __init__(self, child):
@@ -842,9 +872,9 @@
         self.s = StringBuilder(child.size * self.item_size)
         Call1.__init__(self, None, 'tostring', child.shape, dtype, dtype,
                        child)
-        self.res = W_NDimArray([1], dtype, 'C')
-        self.res_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
-                                    self.res.storage)
+        self.res_str = W_NDimArray([1], dtype, order='C')
+        self.res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
+                                    self.res_str.storage)
 
     def create_sig(self):
         return signature.ToStringSignature(self.calc_dtype,
@@ -950,7 +980,7 @@
 
     def setitem(self, item, value):
         self.invalidated()
-        self.dtype.setitem(self, item, value)
+        self.dtype.setitem(self, item, value.convert_to(self.dtype))
 
     def calc_strides(self, shape):
         dtype = self.find_dtype()
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -28,26 +28,38 @@
         return self.identity
 
     def descr_call(self, space, __args__):
+	from interp_numarray import BaseArray
         args_w, kwds_w = __args__.unpack()
         # it occurs to me that we don't support any datatypes that
         # require casting, change it later when we do
         kwds_w.pop('casting', None)
         w_subok = kwds_w.pop('subok', None)
         w_out = kwds_w.pop('out', space.w_None)
-        if ((w_subok is not None and space.is_true(w_subok)) or
-            not space.is_w(w_out, space.w_None)):
+        # Setup a default value for out
+        if space.is_w(w_out, space.w_None):
+            out = None
+        else:
+            out = w_out
+        if (w_subok is not None and space.is_true(w_subok)):
             raise OperationError(space.w_NotImplementedError,
                                  space.wrap("parameters unsupported"))
         if kwds_w or len(args_w) < self.argcount:
             raise OperationError(space.w_ValueError,
                 space.wrap("invalid number of arguments")
             )
-        elif len(args_w) > self.argcount:
-            # The extra arguments should actually be the output array, but we
-            # don't support that yet.
+        elif (len(args_w) > self.argcount and out is not None) or \
+             (len(args_w) > self.argcount + 1):
             raise OperationError(space.w_TypeError,
                 space.wrap("invalid number of arguments")
             )
+        # Override the default out value, if it has been provided in w_wargs
+        if len(args_w) > self.argcount:
+            out = args_w[-1]
+        else:
+            args_w = args_w[:] + [out]
+        if out is not None and not isinstance(out, BaseArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                                            'output must be an array'))
         return self.call(space, args_w)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
@@ -105,28 +117,33 @@
         array([[ 1,  5],
                [ 9, 13]])
         """
-        if not space.is_w(w_out, space.w_None):
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                "out not supported"))
+        from pypy.module.micronumpy.interp_numarray import BaseArray
         if w_axis is None:
             axis = 0
         elif space.is_w(w_axis, space.w_None):
             axis = -1
         else:
             axis = space.int_w(w_axis)
-        return self.reduce(space, w_obj, False, False, axis, keepdims)
+        if space.is_w(w_out, space.w_None):
+            out = None
+        elif not isinstance(w_out, BaseArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                                                'output must be an array'))
+        else:
+            out = w_out
+        return self.reduce(space, w_obj, False, False, axis, keepdims, out)
 
-    def reduce(self, space, w_obj, multidim, promote_to_largest, dim,
-               keepdims=False):
+    def reduce(self, space, w_obj, multidim, promote_to_largest, axis,
+               keepdims=False, out=None):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, \
-                                                           Scalar, ReduceArray
+                                             Scalar, ReduceArray, W_NDimArray
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
         assert isinstance(self, W_Ufunc2)
         obj = convert_to_array(space, w_obj)
-        if dim >= len(obj.shape):
-            raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim))
+        if axis >= len(obj.shape):
+            raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % axis))
         if isinstance(obj, Scalar):
             raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
                 "on a scalar"))
@@ -144,21 +161,55 @@
         if self.identity is None and size == 0:
             raise operationerrfmt(space.w_ValueError, "zero-size array to "
                     "%s.reduce without identity", self.name)
-        if shapelen > 1 and dim >= 0:
-            return self.do_axis_reduce(obj, dtype, dim, keepdims)
-        arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
-        return loop.compute(arr)
+        if shapelen > 1 and axis >= 0:
+            if keepdims:
+                shape = obj.shape[:axis] + [1] + obj.shape[axis + 1:]
+            else:
+                shape = obj.shape[:axis] + obj.shape[axis + 1:]
+            if out:
+                #Test for shape agreement
+                if len(out.shape) > len(shape):
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter for reduction operation %s' +
+                        ' has too many dimensions', self.name)
+                elif len(out.shape) < len(shape):
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter for reduction operation %s' +
+                        ' does not have enough dimensions', self.name)
+                elif out.shape != shape:
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter shape mismatch, expecting [%s]' +
+                        ' , got [%s]', 
+                        ",".join([str(x) for x in shape]),
+                        ",".join([str(x) for x in out.shape]),
+                        )
+                #Test for dtype agreement, perhaps create an itermediate
+                #if out.dtype != dtype:
+                #    raise OperationError(space.w_TypeError, space.wrap(
+                #        "mismatched  dtypes"))
+                return self.do_axis_reduce(obj, out.find_dtype(), axis, out)
+            else:
+                result = W_NDimArray(shape, dtype)
+                return self.do_axis_reduce(obj, dtype, axis, result)
+        if out:
+            if len(out.shape)>0:
+                raise operationerrfmt(space.w_ValueError, "output parameter "
+                              "for reduction operation %s has too many"
+                              " dimensions",self.name)
+            arr = ReduceArray(self.func, self.name, self.identity, obj,
+                                                            out.find_dtype())
+            val = loop.compute(arr)
+            assert isinstance(out, Scalar)
+            out.value = val
+        else:
+            arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
+            val = loop.compute(arr)
+        return val 
 
-    def do_axis_reduce(self, obj, dtype, dim, keepdims):
-        from pypy.module.micronumpy.interp_numarray import AxisReduce,\
-             W_NDimArray
-        if keepdims:
-            shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
-        else:
-            shape = obj.shape[:dim] + obj.shape[dim + 1:]
-        result = W_NDimArray(shape, dtype)
+    def do_axis_reduce(self, obj, dtype, axis, result):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce
         arr = AxisReduce(self.func, self.name, self.identity, obj.shape, dtype,
-                         result, obj, dim)
+                         result, obj, axis)
         loop.compute(arr)
         return arr.left
 
@@ -176,24 +227,55 @@
         self.bool_result = bool_result
 
     def call(self, space, args_w):
-        from pypy.module.micronumpy.interp_numarray import (Call1,
-            convert_to_array, Scalar)
-
-        [w_obj] = args_w
+        from pypy.module.micronumpy.interp_numarray import (Call1, BaseArray,
+            convert_to_array, Scalar, shape_agreement)
+        if len(args_w)<2:
+            [w_obj] = args_w
+            out = None
+        else:
+            [w_obj, out] = args_w
+            if space.is_w(out, space.w_None):
+                out = None
         w_obj = convert_to_array(space, w_obj)
         calc_dtype = find_unaryop_result_dtype(space,
                                   w_obj.find_dtype(),
                                   promote_to_float=self.promote_to_float,
                                   promote_bools=self.promote_bools)
-        if self.bool_result:
+        if out:
+            if not isinstance(out, BaseArray):
+                raise OperationError(space.w_TypeError, space.wrap(
+                                                'output must be an array'))
+            res_dtype = out.find_dtype()
+        elif self.bool_result:
             res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
         if isinstance(w_obj, Scalar):
-            return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype)))
-
-        w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype,
-                      w_obj)
+            arr = self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))
+            if isinstance(out,Scalar):
+                out.value=arr
+            elif isinstance(out, BaseArray):
+                out.fill(space, arr)
+            else:
+                out = arr
+            return space.wrap(out)
+        if out:
+            assert isinstance(out, BaseArray) # For translation
+            broadcast_shape =  shape_agreement(space, w_obj.shape, out.shape)
+            if not broadcast_shape or broadcast_shape != out.shape:
+                raise operationerrfmt(space.w_ValueError,
+                    'output parameter shape mismatch, could not broadcast [%s]' +
+                    ' to [%s]', 
+                    ",".join([str(x) for x in w_obj.shape]),
+                    ",".join([str(x) for x in out.shape]),
+                    )
+            w_res = Call1(self.func, self.name, out.shape, calc_dtype,
+                                         res_dtype, w_obj, out)
+            #Force it immediately
+            w_res.get_concrete()
+        else:
+            w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
+                                         res_dtype, w_obj)
         w_obj.add_invalidates(w_res)
         return w_res
 
@@ -212,32 +294,61 @@
 
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
-            convert_to_array, Scalar, shape_agreement)
-
-        [w_lhs, w_rhs] = args_w
+            convert_to_array, Scalar, shape_agreement, BaseArray)
+        if len(args_w)>2:
+            [w_lhs, w_rhs, w_out] = args_w
+        else:
+            [w_lhs, w_rhs] = args_w
+            w_out = None
         w_lhs = convert_to_array(space, w_lhs)
         w_rhs = convert_to_array(space, w_rhs)
-        calc_dtype = find_binop_result_dtype(space,
-            w_lhs.find_dtype(), w_rhs.find_dtype(),
-            int_only=self.int_only,
-            promote_to_float=self.promote_to_float,
-            promote_bools=self.promote_bools,
-        )
+        if space.is_w(w_out, space.w_None) or w_out is None:
+            out = None
+            calc_dtype = find_binop_result_dtype(space,
+                w_lhs.find_dtype(), w_rhs.find_dtype(),
+                int_only=self.int_only,
+                promote_to_float=self.promote_to_float,
+                promote_bools=self.promote_bools,
+            )
+        elif not isinstance(w_out, BaseArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                    'output must be an array'))
+        else:
+            out = w_out
+            calc_dtype = out.find_dtype()
         if self.comparison_func:
             res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
         if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar):
-            return space.wrap(self.func(calc_dtype,
+            arr = self.func(calc_dtype,
                 w_lhs.value.convert_to(calc_dtype),
                 w_rhs.value.convert_to(calc_dtype)
-            ))
+            )
+            if isinstance(out,Scalar):
+                out.value=arr
+            elif isinstance(out, BaseArray):
+                out.fill(space, arr)
+            else:
+                out = arr
+            return space.wrap(out)
         new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape)
+        # Test correctness of out.shape
+        if out and out.shape != shape_agreement(space, new_shape, out.shape):
+            raise operationerrfmt(space.w_ValueError,
+                'output parameter shape mismatch, could not broadcast [%s]' +
+                ' to [%s]', 
+                ",".join([str(x) for x in new_shape]),
+                ",".join([str(x) for x in out.shape]),
+                )
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
-                      res_dtype, w_lhs, w_rhs)
+                      res_dtype, w_lhs, w_rhs, out)
         w_lhs.add_invalidates(w_res)
         w_rhs.add_invalidates(w_res)
+        if out:
+            #out.add_invalidates(w_res) #causes a recursion loop
+            w_res.get_concrete()
         return w_res
 
 
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -216,13 +216,14 @@
         return self.child.eval(frame, arr.child)
 
 class Call1(Signature):
-    _immutable_fields_ = ['unfunc', 'name', 'child', 'dtype']
+    _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
 
-    def __init__(self, func, name, dtype, child):
+    def __init__(self, func, name, dtype, child, res=None):
         self.unfunc = func
         self.child = child
         self.name = name
         self.dtype = dtype
+        self.res  = res
 
     def hash(self):
         return compute_hash(self.name) ^ intmask(self.child.hash() << 1)
@@ -256,6 +257,29 @@
         v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype)
         return self.unfunc(arr.calc_dtype, v)
 
+
+class BroadcastUfunc(Call1):
+    def _invent_numbering(self, cache, allnumbers):
+        self.res._invent_numbering(cache, allnumbers)
+        self.child._invent_numbering(new_cache(), allnumbers)
+
+    def debug_repr(self):
+        return 'BroadcastUfunc(%s, %s)' % (self.name, self.child.debug_repr())
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import Call1
+
+        assert isinstance(arr, Call1)
+        vtransforms = transforms + [BroadcastTransform(arr.values.shape)]
+        self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
+        self.res._create_iter(iterlist, arraylist, arr.res, transforms)
+
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_numarray import Call1
+        assert isinstance(arr, Call1)
+        v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype)
+        return self.unfunc(arr.calc_dtype, v)
+
 class Call2(Signature):
     _immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right']
 
@@ -316,7 +340,17 @@
 
         assert isinstance(arr, ResultArray)
         offset = frame.get_final_iter().offset
-        arr.left.setitem(offset, self.right.eval(frame, arr.right))
+        val = self.right.eval(frame, arr.right)
+        arr.left.setitem(offset, val)
+
+class BroadcastResultSignature(ResultSignature):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import ResultArray
+
+        assert isinstance(arr, ResultArray)
+        rtransforms = transforms + [BroadcastTransform(arr.left.shape)]
+        self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
 class ToStringSignature(Call1):
     def __init__(self, dtype, child):
@@ -327,10 +361,10 @@
         from pypy.module.micronumpy.interp_numarray import ToStringArray
 
         assert isinstance(arr, ToStringArray)
-        arr.res.setitem(0, self.child.eval(frame, arr.values).convert_to(
+        arr.res_str.setitem(0, self.child.eval(frame, arr.values).convert_to(
             self.dtype))
         for i in range(arr.item_size):
-            arr.s.append(arr.res_casted[i])
+            arr.s.append(arr.res_str_casted[i])
 
 class BroadcastLeft(Call2):
     def _invent_numbering(self, cache, allnumbers):
@@ -455,6 +489,5 @@
             cur = arr.left.getitem(iterator.offset)
             value = self.binfunc(self.calc_dtype, cur, v)
         arr.left.setitem(iterator.offset, value)
-    
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -995,6 +995,10 @@
         assert a.sum() == 5
 
         raises(TypeError, 'a.sum(2, 3)')
+        d = array(0.)
+        b = a.sum(out=d)
+        assert b == d
+        assert isinstance(b, float)
 
     def test_reduce_nd(self):
         from numpypy import arange, array, multiply
@@ -1495,8 +1499,6 @@
         a = array([[1, 2], [3, 4], [5, 6], [7, 8],
                    [9, 10], [11, 12], [13, 14]])
         b = a[::2]
-        print a
-        print b
         assert (b == [[1, 2], [5, 6], [9, 10], [13, 14]]).all()
         c = b + b
         assert c[1][1] == 12
diff --git a/pypy/module/micronumpy/test/test_outarg.py b/pypy/module/micronumpy/test/test_outarg.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_outarg.py
@@ -0,0 +1,126 @@
+import py
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestOutArg(BaseNumpyAppTest):
+    def test_reduce_out(self):
+        from numpypy import arange, zeros, array
+        a = arange(15).reshape(5, 3)
+        b = arange(12).reshape(4,3)
+        c = a.sum(0, out=b[1])
+        assert (c == [30, 35, 40]).all()
+        assert (c == b[1]).all()
+        raises(ValueError, 'a.prod(0, out=arange(10))')
+        a=arange(12).reshape(3,2,2)
+        raises(ValueError, 'a.sum(0, out=arange(12).reshape(3,2,2))')
+        raises(ValueError, 'a.sum(0, out=arange(3))')
+        c = array([-1, 0, 1]).sum(out=zeros([], dtype=bool))
+        #You could argue that this should product False, but
+        # that would require an itermediate result. Cpython numpy
+        # gives True.
+        assert c == True
+        a = array([[-1, 0, 1], [1, 0, -1]])
+        c = a.sum(0, out=zeros((3,), dtype=bool))
+        assert (c == [True, False, True]).all()
+        c = a.sum(1, out=zeros((2,), dtype=bool))
+        assert (c == [True, True]).all()
+
+    def test_reduce_intermediary(self):
+        from numpypy import arange, array
+        a = arange(15).reshape(5, 3)
+        b = array(range(3), dtype=bool)
+        c = a.prod(0, out=b)
+        assert(b == [False,  True,  True]).all()
+
+    def test_ufunc_out(self):
+        from _numpypy import array, negative, zeros, sin
+        from math import sin as msin
+        a = array([[1, 2], [3, 4]])
+        c = zeros((2,2,2))
+        b = negative(a + a, out=c[1])
+        #test for view, and also test that forcing out also forces b
+        assert (c[:, :, 1] == [[0, 0], [-4, -8]]).all()
+        assert (b == [[-2, -4], [-6, -8]]).all()
+        #Test broadcast, type promotion
+        b = negative(3, out=a)
+        assert (a == -3).all()
+        c = zeros((2, 2), dtype=float)
+        b = negative(3, out=c)
+        assert b.dtype.kind == c.dtype.kind
+        assert b.shape == c.shape
+        a = array([1, 2])
+        b = sin(a, out=c)
+        assert(c == [[msin(1), msin(2)]] * 2).all()
+        b = sin(a, out=c+c)
+        assert (c == b).all()
+
+        #Test shape agreement
+        a = zeros((3,4))
+        b = zeros((3,5))
+        raises(ValueError, 'negative(a, out=b)')
+        b = zeros((1,4))
+        raises(ValueError, 'negative(a, out=b)')
+
+    def test_binfunc_out(self):
+        from _numpypy import array, add
+        a = array([[1, 2], [3, 4]])
+        out = array([[1, 2], [3, 4]])
+        c = add(a, a, out=out)
+        assert (c == out).all()
+        assert c.shape == a.shape
+        assert c.dtype is a.dtype
+        c[0,0] = 100
+        assert out[0, 0] == 100
+        out[:] = 100
+        raises(ValueError, 'c = add(a, a, out=out[1])')
+        c = add(a[0], a[1], out=out[1])
+        assert (c == out[1]).all()
+        assert (c == [4, 6]).all()
+        assert (out[0] == 100).all()
+        c = add(a[0], a[1], out=out)
+        assert (c == out[1]).all()
+        assert (c == out[0]).all()
+        out = array(16, dtype=int)
+        b = add(10, 10, out=out)
+        assert b==out
+        assert b.dtype == out.dtype
+        
+    def test_applevel(self):
+        from _numpypy import array, sum, max, min
+        a = array([[1, 2], [3, 4]])
+        out = array([[0, 0], [0, 0]])
+        c = sum(a, axis=0, out=out[0])
+        assert (c == [4, 6]).all()
+        assert (c == out[0]).all()
+        assert (c != out[1]).all()
+        c = max(a, axis=1, out=out[0])
+        assert (c == [2, 4]).all()
+        assert (c == out[0]).all()
+        assert (c != out[1]).all()
+        
+    def test_ufunc_cast(self):
+        from _numpypy import array, negative, add, sum
+        a = array(16, dtype = int)
+        c = array(0, dtype = float)
+        b = negative(a, out=c)
+        assert b == c
+        b = add(a, a, out=c)
+        assert b == c
+        d = array([16, 16], dtype=int)
+        b = sum(d, out=c)
+        assert b == c
+        try:
+            from _numpypy import version
+            v = version.version.split('.')
+        except:
+            v = ['1', '6', '0'] # numpypy is api compatable to what version?
+        if v[0]<'2':
+            b = negative(c, out=a)
+            assert b == a
+            b = add(c, c, out=a)
+            assert b == a
+            b = sum(array([16, 16], dtype=float), out=a)
+            assert b == a
+        else:
+            cast_error = raises(TypeError, negative, c, a)
+            assert str(cast_error.value) == \
+            "Cannot cast ufunc negative output from dtype('float64') to dtype('int64') with casting rule 'same_kind'"
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -131,7 +131,7 @@
         #            bogus. We need to improve the situation somehow.
         self.check_simple_loop({'getinteriorfield_raw': 2,
                                 'setinteriorfield_raw': 1,
-                                'arraylen_gc': 1,
+                                'arraylen_gc': 2,
                                 'guard_true': 1,
                                 'int_lt': 1,
                                 'jump': 1,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -128,3 +128,82 @@
         loop, = log.loops_by_filename(self.filepath)
         ops = loop.ops_by_id('look')
         assert 'call' not in log.opnames(ops)
+
+    #XXX the following tests only work with strategies enabled
+
+    def test_should_not_create_intobject_with_sets(self):
+        def main(n):
+            i = 0
+            s = set()
+            while i < n:
+                s.add(i)
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_should_not_create_stringobject_with_sets(self):
+        def main(n):
+            i = 0
+            s = set()
+            while i < n:
+                s.add(str(i))
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_should_not_create_intobject_with_lists(self):
+        def main(n):
+            i = 0
+            l = []
+            while i < n:
+                l.append(i)
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_should_not_create_stringobject_with_lists(self):
+        def main(n):
+            i = 0
+            l = []
+            while i < n:
+                l.append(str(i))
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_optimized_create_list_from_string(self):
+        def main(n):
+            i = 0
+            l = []
+            while i < n:
+                l = list("abc" * i)
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_optimized_create_set_from_list(self):
+        def main(n):
+            i = 0
+            while i < n:
+                s = set([1,2,3])
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py
--- a/pypy/objspace/flow/test/test_objspace.py
+++ b/pypy/objspace/flow/test/test_objspace.py
@@ -1,6 +1,6 @@
 from __future__ import with_statement
 import new
-import py
+import py, sys
 from pypy.objspace.flow.model import Constant, Block, Link, Variable
 from pypy.objspace.flow.model import mkentrymap, c_last_exception
 from pypy.interpreter.argument import Arguments
@@ -893,6 +893,8 @@
         """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG
         bytecode
         """
+        if sys.version_info < (2, 7):
+            py.test.skip("2.7 only test")
         self.patch_opcodes('BUILD_LIST_FROM_ARG')
         try:
             def f():
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -111,9 +111,15 @@
     length = len(data)
     start, stop, step, slicelength = w_slice.indices4(space, length)
     assert slicelength >= 0
-    newdata = [data[start + i*step] for i in range(slicelength)]
+    if step == 1 and 0 <= start <= stop:
+        newdata = data[start:stop]
+    else:
+        newdata = _getitem_slice_multistep(data, start, step, slicelength)
     return W_BytearrayObject(newdata)
 
+def _getitem_slice_multistep(data, start, step, slicelength):
+    return [data[start + i*step] for i in range(slicelength)]
+
 def contains__Bytearray_Int(space, w_bytearray, w_char):
     char = space.int_w(w_char)
     if not 0 <= char < 256:
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -127,10 +127,10 @@
     def iter(self, w_dict):
         return ModuleDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
+    def w_keys(self, w_dict):
         space = self.space
-        iterator = self.unerase(w_dict.dstorage).iteritems
-        return [space.wrap(key) for key, cell in iterator()]
+        l = self.unerase(w_dict.dstorage).keys()
+        return space.newlist_str(l)
 
     def values(self, w_dict):
         iterator = self.unerase(w_dict.dstorage).itervalues
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -90,9 +90,9 @@
 def _add_indirections():
     dict_methods = "setitem setitem_str getitem \
                     getitem_str delitem length \
-                    clear keys values \
+                    clear w_keys values \
                     items iter setdefault \
-                    popitem".split()
+                    popitem listview_str listview_int".split()
 
     def make_method(method):
         def f(self, *args):
@@ -113,7 +113,7 @@
     def get_empty_storage(self):
         raise NotImplementedError
 
-    def keys(self, w_dict):
+    def w_keys(self, w_dict):
         iterator = self.iter(w_dict)
         result = []
         while 1:
@@ -121,7 +121,7 @@
             if w_key is not None:
                 result.append(w_key)
             else:
-                return result
+                return self.space.newlist(result)
 
     def values(self, w_dict):
         iterator = self.iter(w_dict)
@@ -160,6 +160,11 @@
         w_dict.strategy = strategy
         w_dict.dstorage = storage
 
+    def listview_str(self, w_dict):
+        return None
+
+    def listview_int(self, w_dict):
+        return None
 
 class EmptyDictStrategy(DictStrategy):
 
@@ -371,8 +376,9 @@
             self.switch_to_object_strategy(w_dict)
             return w_dict.getitem(w_key)
 
-    def keys(self, w_dict):
-        return [self.wrap(key) for key in self.unerase(w_dict.dstorage).iterkeys()]
+    def w_keys(self, w_dict):
+        l = [self.wrap(key) for key in self.unerase(w_dict.dstorage).iterkeys()]
+        return self.space.newlist(l)
 
     def values(self, w_dict):
         return self.unerase(w_dict.dstorage).values()
@@ -425,8 +431,8 @@
     def iter(self, w_dict):
         return ObjectIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class StringDictStrategy(AbstractTypedStrategy, DictStrategy):
@@ -469,9 +475,15 @@
         assert key is not None
         return self.unerase(w_dict.dstorage).get(key, None)
 
+    def listview_str(self, w_dict):
+        return self.unerase(w_dict.dstorage).keys()
+
     def iter(self, w_dict):
         return StrIteratorImplementation(self.space, self, w_dict)
 
+    def w_keys(self, w_dict):
+        return self.space.newlist_str(self.listview_str(w_dict))
+
 
 class _WrappedIteratorMixin(object):
     _mixin_ = True
@@ -534,6 +546,14 @@
     def iter(self, w_dict):
         return IntIteratorImplementation(self.space, self, w_dict)
 
+    def listview_int(self, w_dict):
+        return self.unerase(w_dict.dstorage).keys()
+
+    def w_keys(self, w_dict):
+        # XXX there is no space.newlist_int yet
+        space = self.space
+        return space.call_function(space.w_list, w_dict)
+
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass
 
@@ -688,7 +708,7 @@
     return space.newlist(w_self.items())
 
 def dict_keys__DictMulti(space, w_self):
-    return space.newlist(w_self.keys())
+    return w_self.w_keys()
 
 def dict_values__DictMulti(space, w_self):
     return space.newlist(w_self.values())
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -76,7 +76,7 @@
 
     def keys(self, w_dict):
         space = self.space
-        return [space.wrap(key) for key in self.unerase(w_dict.dstorage).dict_w.iterkeys()]
+        return space.newlist_str(self.unerase(w_dict.dstorage).dict_w.keys())
 
     def values(self, w_dict):
         return [unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues()]
diff --git a/pypy/objspace/std/dicttype.py b/pypy/objspace/std/dicttype.py
--- a/pypy/objspace/std/dicttype.py
+++ b/pypy/objspace/std/dicttype.py
@@ -62,8 +62,14 @@
         w_fill = space.w_None
     if space.is_w(w_type, space.w_dict):
         w_dict = W_DictMultiObject.allocate_and_init_instance(space, w_type)
-        for w_key in space.listview(w_keys):
-            w_dict.setitem(w_key, w_fill)
+
+        strlist = space.listview_str(w_keys)
+        if strlist is not None:
+            for key in strlist:
+                w_dict.setitem_str(key, w_fill)
+        else:
+            for w_key in space.listview(w_keys):
+                w_dict.setitem(w_key, w_fill)
     else:
         w_dict = space.call_function(w_type)
         for w_key in space.listview(w_keys):
diff --git a/pypy/objspace/std/frozensettype.py b/pypy/objspace/std/frozensettype.py
--- a/pypy/objspace/std/frozensettype.py
+++ b/pypy/objspace/std/frozensettype.py
@@ -39,13 +39,11 @@
 def descr__frozenset__new__(space, w_frozensettype,
                             w_iterable=gateway.NoneNotWrapped):
     from pypy.objspace.std.setobject import W_FrozensetObject
-    from pypy.objspace.std.setobject import make_setdata_from_w_iterable
     if (space.is_w(w_frozensettype, space.w_frozenset) and
         w_iterable is not None and type(w_iterable) is W_FrozensetObject):
         return w_iterable
     w_obj = space.allocate_instance(W_FrozensetObject, w_frozensettype)
-    data = make_setdata_from_w_iterable(space, w_iterable)
-    W_FrozensetObject.__init__(w_obj, space, data)
+    W_FrozensetObject.__init__(w_obj, space, w_iterable)
     return w_obj
 
 frozenset_typedef = StdTypeDef("frozenset",
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -29,9 +29,8 @@
 class W_SeqIterObject(W_AbstractSeqIterObject):
     """Sequence iterator implementation for general sequences."""
 
-class W_FastListIterObject(W_AbstractSeqIterObject):
-    """Sequence iterator specialized for lists, accessing directly their
-    RPython-level list of wrapped objects.
+class W_FastListIterObject(W_AbstractSeqIterObject): # XXX still needed
+    """Sequence iterator specialized for lists.
     """
 
 class W_FastTupleIterObject(W_AbstractSeqIterObject):
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -139,6 +139,16 @@
         new erased object as storage"""
         self.strategy.init_from_list_w(self, list_w)
 
+    def clear(self, space):
+        """Initializes (or overrides) the listobject as empty."""
+        self.space = space
+        if space.config.objspace.std.withliststrategies:
+            strategy = space.fromcache(EmptyListStrategy)
+        else:
+            strategy = space.fromcache(ObjectListStrategy)
+        self.strategy = strategy
+        strategy.clear(self)
+
     def clone(self):
         """Returns a clone by creating a new listobject
         with the same strategy and a copy of the storage"""
@@ -200,6 +210,11 @@
         """ Return the items in the list as unwrapped strings. If the list does
         not use the list strategy, return None. """
         return self.strategy.getitems_str(self)
+
+    def getitems_int(self):
+        """ Return the items in the list as unwrapped ints. If the list does
+        not use the list strategy, return None. """
+        return self.strategy.getitems_int(self)
     # ___________________________________________________
 
 
@@ -300,6 +315,9 @@
     def getitems_str(self, w_list):
         return None
 
+    def getitems_int(self, w_list):
+        return None
+
     def getstorage_copy(self, w_list):
         raise NotImplementedError
 
@@ -358,6 +376,9 @@
         assert len(list_w) == 0
         w_list.lstorage = self.erase(None)
 
+    def clear(self, w_list):
+        w_list.lstorage = self.erase(None)
+
     erase, unerase = rerased.new_erasing_pair("empty")
     erase = staticmethod(erase)
     unerase = staticmethod(unerase)
@@ -516,6 +537,9 @@
             raise IndexError
         return start + i * step
 
+    def getitems_int(self, w_list):
+        return self._getitems_range(w_list, False)
+
     def getitem(self, w_list, i):
         return self.wrap(self._getitem_unwrapped(w_list, i))
 
@@ -696,6 +720,7 @@
             for i in l:
                 if i == obj:
                     return True
+            return False
         return ListStrategy.contains(self, w_list, w_obj)
 
     def length(self, w_list):
@@ -937,6 +962,9 @@
     def init_from_list_w(self, w_list, list_w):
         w_list.lstorage = self.erase(list_w)
 
+    def clear(self, w_list):
+        w_list.lstorage = self.erase([])
+
     def contains(self, w_list, w_obj):
         return ListStrategy.contains(self, w_list, w_obj)
 
@@ -970,6 +998,9 @@
         if reverse:
             l.reverse()
 
+    def getitems_int(self, w_list):
+        return self.unerase(w_list.lstorage)
+
 class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = 0.0
     _applevel_repr = "float"
@@ -1027,37 +1058,49 @@
     def getitems_str(self, w_list):
         return self.unerase(w_list.lstorage)
 
-
 # _______________________________________________________
 
 init_signature = Signature(['sequence'], None, None)
 init_defaults = [None]
 
 def init__List(space, w_list, __args__):
-    from pypy.objspace.std.tupleobject import W_TupleObject
+    from pypy.objspace.std.tupleobject import W_AbstractTupleObject
     # this is on the silly side
     w_iterable, = __args__.parse_obj(
             None, 'list', init_signature, init_defaults)
-    w_list.__init__(space, [])
+    w_list.clear(space)
     if w_iterable is not None:
-        # unfortunately this is duplicating space.unpackiterable to avoid
-        # assigning a new RPython list to 'wrappeditems', which defeats the
-        # W_FastListIterObject optimization.
-        if isinstance(w_iterable, W_ListObject):
-            w_list.extend(w_iterable)
-        elif isinstance(w_iterable, W_TupleObject):
-            w_list.extend(W_ListObject(space, w_iterable.wrappeditems[:]))
-        else:
-            _init_from_iterable(space, w_list, w_iterable)
+        if type(w_iterable) is W_ListObject:
+            w_iterable.copy_into(w_list)
+            return
+        elif isinstance(w_iterable, W_AbstractTupleObject):
+            w_list.__init__(space, w_iterable.getitems_copy())
+            return
+
+        intlist = space.listview_int(w_iterable)
+        if intlist is not None:
+            w_list.strategy = strategy = space.fromcache(IntegerListStrategy)
+             # need to copy because intlist can share with w_iterable
+            w_list.lstorage = strategy.erase(intlist[:])
+            return
+
+        strlist = space.listview_str(w_iterable)
+        if strlist is not None:
+            w_list.strategy = strategy = space.fromcache(StringListStrategy)
+             # need to copy because intlist can share with w_iterable
+            w_list.lstorage = strategy.erase(strlist[:])
+            return
+
+        # xxx special hack for speed
+        from pypy.interpreter.generator import GeneratorIterator
+        if isinstance(w_iterable, GeneratorIterator):
+            w_iterable.unpack_into_w(w_list)
+            return
+        # /xxx
+        _init_from_iterable(space, w_list, w_iterable)
 
 def _init_from_iterable(space, w_list, w_iterable):
     # in its own function to make the JIT look into init__List
-    # xxx special hack for speed
-    from pypy.interpreter.generator import GeneratorIterator
-    if isinstance(w_iterable, GeneratorIterator):
-        w_iterable.unpack_into_w(w_list)
-        return
-    # /xxx
     w_iterator = space.iter(w_iterable)
     while True:
         try:
diff --git a/pypy/objspace/std/listtype.py b/pypy/objspace/std/listtype.py
--- a/pypy/objspace/std/listtype.py
+++ b/pypy/objspace/std/listtype.py
@@ -43,7 +43,7 @@
 def descr__new__(space, w_listtype, __args__):
     from pypy.objspace.std.listobject import W_ListObject
     w_obj = space.allocate_instance(W_ListObject, w_listtype)
-    W_ListObject.__init__(w_obj, space, [])
+    w_obj.clear(space)
     return w_obj
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -694,6 +694,8 @@
         self.delitem(w_dict, w_key)
         return (w_key, w_value)
 
+    # XXX could implement a more efficient w_keys based on space.newlist_str
+
 def materialize_r_dict(space, obj, dict_w):
     map = obj._get_mapdict_map()
     new_obj = map.materialize_r_dict(space, obj, dict_w)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -227,10 +227,7 @@
             return W_ComplexObject(x.real, x.imag)
 
         if isinstance(x, set):
-            rdict_w = r_dict(self.eq_w, self.hash_w)
-            for item in x:
-                rdict_w[self.wrap(item)] = None
-            res = W_SetObject(self, rdict_w)
+            res = W_SetObject(self, self.newlist([self.wrap(item) for item in x]))
             return res
 
         if isinstance(x, frozenset):
@@ -325,7 +322,7 @@
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
-        return W_SetObject(self, newset(self))
+        return W_SetObject(self, None)
 
     def newslice(self, w_start, w_end, w_step):
         return W_SliceObject(w_start, w_end, w_step)
@@ -403,7 +400,7 @@
     def unpackiterable(self, w_obj, expected_length=-1):
         if isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
-        elif isinstance(w_obj, W_ListObject):
+        elif type(w_obj) is W_ListObject:
             t = w_obj.getitems_copy()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
@@ -417,7 +414,7 @@
         """
         if isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.tolist()
-        elif isinstance(w_obj, W_ListObject):
+        elif type(w_obj) is W_ListObject:
             if unroll:
                 t = w_obj.getitems_unroll()
             else:
@@ -438,7 +435,7 @@
         return self.fixedview(w_obj, expected_length, unroll=True)
 
     def listview(self, w_obj, expected_length=-1):
-        if isinstance(w_obj, W_ListObject):
+        if type(w_obj) is W_ListObject:
             t = w_obj.getitems()
         elif isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
@@ -449,8 +446,25 @@
         return t
 
     def listview_str(self, w_obj):
-        if isinstance(w_obj, W_ListObject):
+        # note: uses exact type checking for objects with strategies,
+        # and isinstance() for others.  See test_listobject.test_uses_custom...
+        if type(w_obj) is W_ListObject:
             return w_obj.getitems_str()
+        if type(w_obj) is W_DictMultiObject:
+            return w_obj.listview_str()
+        if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
+            return w_obj.listview_str()
+        if isinstance(w_obj, W_StringObject):
+            return w_obj.listview_str()
+        return None
+
+    def listview_int(self, w_obj):
+        if type(w_obj) is W_ListObject:
+            return w_obj.getitems_int()
+        if type(w_obj) is W_DictMultiObject:
+            return w_obj.listview_int()
+        if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
+            return w_obj.listview_int()
         return None
 
     def sliceindices(self, w_slice, w_length):
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -7,6 +7,12 @@
 from pypy.interpreter.argument import Signature
 from pypy.objspace.std.settype import set_typedef as settypedef
 from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef
+from pypy.rlib import rerased
+from pypy.rlib.objectmodel import instantiate
+from pypy.interpreter.generator import GeneratorIterator
+from pypy.objspace.std.listobject import W_ListObject
+from pypy.objspace.std.intobject import W_IntObject
+from pypy.objspace.std.stringobject import W_StringObject
 
 class W_BaseSetObject(W_Object):
     typedef = None
@@ -20,88 +26,859 @@
             return True
         return False
 
-
-    def __init__(w_self, space, setdata):
+    def __init__(w_self, space, w_iterable=None):
         """Initialize the set by taking ownership of 'setdata'."""
-        assert setdata is not None
-        w_self.setdata = setdata
+        w_self.space = space
+        set_strategy_and_setdata(space, w_self, w_iterable)
 
     def __repr__(w_self):
         """representation for debugging purposes"""
-        reprlist = [repr(w_item) for w_item in w_self.setdata.keys()]
+        reprlist = [repr(w_item) for w_item in w_self.getkeys()]
         return "<%s(%s)>" % (w_self.__class__.__name__, ', '.join(reprlist))
 
+    def from_storage_and_strategy(w_self, storage, strategy):
+        obj = w_self._newobj(w_self.space, None)
+        assert isinstance(obj, W_BaseSetObject)
+        obj.strategy = strategy
+        obj.sstorage = storage
+        return obj
+
     _lifeline_ = None
     def getweakref(self):
         return self._lifeline_
+
     def setweakref(self, space, weakreflifeline):
         self._lifeline_ = weakreflifeline
     def delweakref(self):
         self._lifeline_ = None
 
+    def switch_to_object_strategy(self, space):
+        d = self.strategy.getdict_w(self)
+        self.strategy = strategy = space.fromcache(ObjectSetStrategy)
+        self.sstorage = strategy.erase(d)
+
+    def switch_to_empty_strategy(self):
+        self.strategy = strategy = self.space.fromcache(EmptySetStrategy)
+        self.sstorage = strategy.get_empty_storage()
+
+    # _____________ strategy methods ________________
+
+
+    def clear(self):
+        """ Removes all elements from the set. """
+        self.strategy.clear(self)
+
+    def copy_real(self):
+        """ Returns a clone of the set. Frozensets storages are also copied."""
+        return self.strategy.copy_real(self)
+
+    def length(self):
+        """ Returns the number of items inside the set. """
+        return self.strategy.length(self)
+
+    def add(self, w_key):
+        """ Adds an element to the set. The element must be wrapped. """
+        self.strategy.add(self, w_key)
+
+    def remove(self, w_item):
+        """ Removes the given element from the set. Element must be wrapped. """
+        return self.strategy.remove(self, w_item)
+
+    def getdict_w(self):
+        """ Returns a dict with all elements of the set. Needed only for switching to ObjectSetStrategy. """
+        return self.strategy.getdict_w(self)
+
+    def listview_str(self):
+        """ If this is a string set return its contents as a list of uwnrapped strings. Otherwise return None. """
+        return self.strategy.listview_str(self)
+
+    def listview_int(self):
+        """ If this is an int set return its contents as a list of uwnrapped ints. Otherwise return None. """
+        return self.strategy.listview_int(self)
+
+    def get_storage_copy(self):
+        """ Returns a copy of the storage. Needed when we want to clone all elements from one set and
+        put them into another. """
+        return self.strategy.get_storage_copy(self)
+
+    def getkeys(self):
+        """ Returns a list of all elements inside the set. Only used in __repr__. Use as less as possible."""
+        return self.strategy.getkeys(self)
+
+    def difference(self, w_other):
+        """ Returns a set with all items that are in this set, but not in w_other. W_other must be a set."""
+        return self.strategy.difference(self, w_other)
+
+    def difference_update(self, w_other):
+        """ As difference but overwrites the sets content with the result. W_other must be a set."""
+        self.strategy.difference_update(self, w_other)
+
+    def symmetric_difference(self, w_other):
+        """ Returns a set with all items that are either in this set or in w_other, but not in both. W_other must be a set. """
+        return self.strategy.symmetric_difference(self, w_other)
+
+    def symmetric_difference_update(self, w_other):
+        """ As symmetric_difference but overwrites the content of the set with the result. W_other must be a set."""
+        self.strategy.symmetric_difference_update(self, w_other)
+
+    def intersect(self, w_other):
+        """ Returns a set with all items that exists in both sets, this set and in w_other. W_other must be a set. """
+        return self.strategy.intersect(self, w_other)
+
+    def intersect_update(self, w_other):
+        """ Keeps only those elements found in both sets, removing all other elements. W_other must be a set."""
+        self.strategy.intersect_update(self, w_other)
+
+    def issubset(self, w_other):
+        """ Checks wether this set is a subset of w_other. W_other must be a set. """
+        return self.strategy.issubset(self, w_other)
+
+    def isdisjoint(self, w_other):
+        """ Checks wether this set and the w_other are completly different, i.e. have no equal elements. W_other must be a set."""
+        return self.strategy.isdisjoint(self, w_other)
+
+    def update(self, w_other):
+        """ Appends all elements from the given set to this set. W_other must be a set."""
+        self.strategy.update(self, w_other)
+
+    def has_key(self, w_key):
+        """ Checks wether this set contains the given wrapped key."""
+        return self.strategy.has_key(self, w_key)
+
+    def equals(self, w_other):
+        """ Checks wether this set and the given set are equal, i.e. contain the same elements. W_other must be a set."""
+        return self.strategy.equals(self, w_other)
+
+    def iter(self):
+        """ Returns an iterator of the elements from this set. """
+        return self.strategy.iter(self)
+
+    def popitem(self):
+        """ Removes an arbitrary element from the set. May raise KeyError if set is empty."""
+        return self.strategy.popitem(self)
+
 class W_SetObject(W_BaseSetObject):
     from pypy.objspace.std.settype import set_typedef as typedef
 
-    def _newobj(w_self, space, rdict_w):
-        """Make a new set by taking ownership of 'rdict_w'."""
+    def _newobj(w_self, space, w_iterable):
+        """Make a new set by taking ownership of 'w_iterable'."""
         if type(w_self) is W_SetObject:
-            return W_SetObject(space, rdict_w)
+            return W_SetObject(space, w_iterable)
         w_type = space.type(w_self)
         w_obj = space.allocate_instance(W_SetObject, w_type)
-        W_SetObject.__init__(w_obj, space, rdict_w)
+        W_SetObject.__init__(w_obj, space, w_iterable)
         return w_obj
 
 class W_FrozensetObject(W_BaseSetObject):
     from pypy.objspace.std.frozensettype import frozenset_typedef as typedef
     hash = 0
 
-    def _newobj(w_self, space, rdict_w):
-        """Make a new frozenset by taking ownership of 'rdict_w'."""
+    def _newobj(w_self, space, w_iterable):
+        """Make a new frozenset by taking ownership of 'w_iterable'."""
         if type(w_self) is W_FrozensetObject:
-            return W_FrozensetObject(space, rdict_w)
+            return W_FrozensetObject(space, w_iterable)
         w_type = space.type(w_self)
         w_obj = space.allocate_instance(W_FrozensetObject, w_type)
-        W_FrozensetObject.__init__(w_obj, space, rdict_w)
+        W_FrozensetObject.__init__(w_obj, space, w_iterable)
         return w_obj
 
 registerimplementation(W_BaseSetObject)
 registerimplementation(W_SetObject)
 registerimplementation(W_FrozensetObject)
 
-class W_SetIterObject(W_Object):
-    from pypy.objspace.std.settype import setiter_typedef as typedef
+class SetStrategy(object):
+    def __init__(self, space):
+        self.space = space
 
-    def __init__(w_self, setdata):
-        w_self.content = content = setdata
-        w_self.len = len(content)
-        w_self.pos = 0
-        w_self.iterator = w_self.content.iterkeys()
+    def get_empty_dict(self):
+        """ Returns an empty dictionary depending on the strategy. Used to initalize a new storage. """
+        raise NotImplementedError
 
-    def next_entry(w_self):
-        for w_key in w_self.iterator:
+    def get_empty_storage(self):
+        """ Returns an empty storage (erased) object. Used to initialize an empty set."""
+        raise NotImplementedError
+
+    def listview_str(self, w_set):
+        return None
+
+    def listview_int(self, w_set):
+        return None
+
+    #def erase(self, storage):
+    #    raise NotImplementedError
+
+    #def unerase(self, storage):
+    #    raise NotImplementedError
+
+    # __________________ methods called on W_SetObject _________________
+
+    def clear(self, w_set):
+        raise NotImplementedError
+
+    def copy_real(self, w_set):
+        raise NotImplementedError
+
+    def length(self, w_set):
+        raise NotImplementedError
+
+    def add(self, w_set, w_key):
+        raise NotImplementedError
+
+    def remove(self, w_set, w_item):
+        raise NotImplementedError
+
+    def getdict_w(self, w_set):
+        raise NotImplementedError
+
+    def get_storage_copy(self, w_set):
+        raise NotImplementedError
+
+    def getkeys(self, w_set):
+        raise NotImplementedError
+
+    def difference(self, w_set, w_other):
+        raise NotImplementedError
+
+    def difference_update(self, w_set, w_other):
+        raise NotImplementedError
+
+    def symmetric_difference(self, w_set, w_other):
+        raise NotImplementedError
+
+    def symmetric_difference_update(self, w_set, w_other):
+        raise NotImplementedError
+
+    def intersect(self, w_set, w_other):
+        raise NotImplementedError
+
+    def intersect_update(self, w_set, w_other):
+        raise NotImplementedError
+
+    def issubset(self, w_set, w_other):
+        raise NotImplementedError
+
+    def isdisjoint(self, w_set, w_other):
+        raise NotImplementedError
+
+    def update(self, w_set, w_other):
+        raise NotImplementedError
+
+    def has_key(self, w_set, w_key):
+        raise NotImplementedError
+
+    def equals(self, w_set, w_other):
+        raise NotImplementedError
+
+    def iter(self, w_set):
+        raise NotImplementedError
+
+    def popitem(self, w_set):
+        raise NotImplementedError
+
+class EmptySetStrategy(SetStrategy):
+
+    erase, unerase = rerased.new_erasing_pair("empty")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def get_empty_storage(self):
+        return self.erase(None)
+
+    def is_correct_type(self, w_key):
+        return False
+
+    def length(self, w_set):
+        return 0
+
+    def clear(self, w_set):
+        pass
+
+    def copy_real(self, w_set):
+        storage = self.erase(None)
+        clone = w_set.from_storage_and_strategy(storage, self)
+        return clone
+
+    def add(self, w_set, w_key):
+        if type(w_key) is W_IntObject:
+            strategy = self.space.fromcache(IntegerSetStrategy)
+        elif type(w_key) is W_StringObject:
+            strategy = self.space.fromcache(StringSetStrategy)
+        else:
+            strategy = self.space.fromcache(ObjectSetStrategy)
+        w_set.strategy = strategy
+        w_set.sstorage = strategy.get_empty_storage()
+        w_set.add(w_key)
+
+    def remove(self, w_set, w_item):
+        return False
+
+    def getdict_w(self, w_set):
+        return newset(self.space)
+
+    def get_storage_copy(self, w_set):
+        return w_set.sstorage
+
+    def getkeys(self, w_set):
+        return []
+
+    def has_key(self, w_set, w_key):
+        return False
+
+    def equals(self, w_set, w_other):
+        if w_other.strategy is self or w_other.length() == 0:
+            return True
+        return False
+
+    def difference(self, w_set, w_other):
+        return w_set.copy_real()
+
+    def difference_update(self, w_set, w_other):
+        pass
+
+    def intersect(self, w_set, w_other):
+        return w_set.copy_real()
+
+    def intersect_update(self, w_set, w_other):
+        pass
+
+    def isdisjoint(self, w_set, w_other):
+        return True
+
+    def issubset(self, w_set, w_other):
+        return True
+
+    def symmetric_difference(self, w_set, w_other):
+        return w_other.copy_real()
+
+    def symmetric_difference_update(self, w_set, w_other):
+        w_set.strategy = w_other.strategy
+        w_set.sstorage = w_other.get_storage_copy()
+
+    def update(self, w_set, w_other):
+        w_set.strategy = w_other.strategy
+        w_set.sstorage = w_other.get_storage_copy()
+
+    def iter(self, w_set):
+        return EmptyIteratorImplementation(self.space, w_set)
+
+    def popitem(self, w_set):
+        raise OperationError(self.space.w_KeyError,
+                                self.space.wrap('pop from an empty set'))
+
+class AbstractUnwrappedSetStrategy(object):
+    _mixin_ = True
+
+    def is_correct_type(self, w_key):
+        """ Checks wether the given wrapped key fits this strategy."""
+        raise NotImplementedError
+
+    def unwrap(self, w_item):
+        """ Returns the unwrapped value of the given wrapped item."""
+        raise NotImplementedError
+
+    def wrap(self, item):
+        """ Returns a wrapped version of the given unwrapped item. """
+        raise NotImplementedError
+
+    def get_storage_from_list(self, list_w):
+        setdata = self.get_empty_dict()
+        for w_item in list_w:
+            setdata[self.unwrap(w_item)] = None
+        return self.erase(setdata)
+
+    def get_storage_from_unwrapped_list(self, items):
+        setdata = self.get_empty_dict()
+        for item in items:
+            setdata[item] = None
+        return self.erase(setdata)
+
+    def length(self, w_set):
+        return len(self.unerase(w_set.sstorage))
+
+    def clear(self, w_set):
+        w_set.switch_to_empty_strategy()
+
+    def copy_real(self, w_set):
+        # may be used internally on frozen sets, although frozenset().copy()
+        # returns self in frozenset_copy__Frozenset.
+        strategy = w_set.strategy
+        d = self.unerase(w_set.sstorage)
+        storage = self.erase(d.copy())
+        clone = w_set.from_storage_and_strategy(storage, strategy)
+        return clone
+
+    def add(self, w_set, w_key):
+        if self.is_correct_type(w_key):
+            d = self.unerase(w_set.sstorage)
+            d[self.unwrap(w_key)] = None
+        else:
+            w_set.switch_to_object_strategy(self.space)
+            w_set.add(w_key)
+
+    def remove(self, w_set, w_item):
+        from pypy.objspace.std.dictmultiobject import _never_equal_to_string
+        d = self.unerase(w_set.sstorage)
+        if not self.is_correct_type(w_item):
+            #XXX check type of w_item and immediately return False in some cases
+            w_set.switch_to_object_strategy(self.space)
+            return w_set.remove(w_item)
+
+        key = self.unwrap(w_item)
+        try:
+            del d[key]
+            return True
+        except KeyError:
+            return False
+
+    def getdict_w(self, w_set):
+        result = newset(self.space)
+        keys = self.unerase(w_set.sstorage).keys()
+        for key in keys:
+            result[self.wrap(key)] = None
+        return result
+
+    def get_storage_copy(self, w_set):
+        d = self.unerase(w_set.sstorage)
+        copy = self.erase(d.copy())
+        return copy
+
+    def getkeys(self, w_set):
+        keys = self.unerase(w_set.sstorage).keys()
+        keys_w = [self.wrap(key) for key in keys]
+        return keys_w
+
+    def has_key(self, w_set, w_key):
+        from pypy.objspace.std.dictmultiobject import _never_equal_to_string
+        if not self.is_correct_type(w_key):
+            #XXX check type of w_item and immediately return False in some cases
+            w_set.switch_to_object_strategy(self.space)
+            return w_set.has_key(w_key)
+        d = self.unerase(w_set.sstorage)
+        return self.unwrap(w_key) in d
+
+    def equals(self, w_set, w_other):
+        if w_set.length() != w_other.length():
+            return False
+        items = self.unerase(w_set.sstorage).keys()
+        for key in items:
+            if not w_other.has_key(self.wrap(key)):
+                return False
+        return True
+
+    def _difference_wrapped(self, w_set, w_other):
+        strategy = self.space.fromcache(ObjectSetStrategy)
+
+        d_new = strategy.get_empty_dict()
+        for obj in self.unerase(w_set.sstorage):
+            w_item = self.wrap(obj)
+            if not w_other.has_key(w_item):
+                d_new[w_item] = None
+
+        return strategy.erase(d_new)
+
+    def _difference_unwrapped(self, w_set, w_other):
+        iterator = self.unerase(w_set.sstorage).iterkeys()
+        other_dict = self.unerase(w_other.sstorage)
+        result_dict = self.get_empty_dict()
+        for key in iterator:
+            if key not in other_dict:
+                result_dict[key] = None
+        return self.erase(result_dict)
+
+    def _difference_base(self, w_set, w_other):
+        if self is w_other.strategy:
+            strategy = w_set.strategy
+            storage = self._difference_unwrapped(w_set, w_other)
+        elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
+            strategy = w_set.strategy
+            storage = w_set.sstorage
+        else:
+            strategy = self.space.fromcache(ObjectSetStrategy)
+            storage = self._difference_wrapped(w_set, w_other)
+        return storage, strategy
+
+    def difference(self, w_set, w_other):
+        storage, strategy = self._difference_base(w_set, w_other)
+        w_newset = w_set.from_storage_and_strategy(storage, strategy)
+        return w_newset
+
+    def difference_update(self, w_set, w_other):
+        storage, strategy = self._difference_base(w_set, w_other)
+        w_set.strategy = strategy
+        w_set.sstorage = storage
+
+    def _symmetric_difference_unwrapped(self, w_set, w_other):
+        d_new = self.get_empty_dict()
+        d_this = self.unerase(w_set.sstorage)
+        d_other = self.unerase(w_other.sstorage)
+        for key in d_other.keys():
+            if not key in d_this:
+                d_new[key] = None
+        for key in d_this.keys():
+            if not key in d_other:
+                d_new[key] = None
+
+        storage = self.erase(d_new)
+        return storage
+
+    def _symmetric_difference_wrapped(self, w_set, w_other):
+        newsetdata = newset(self.space)
+        for obj in self.unerase(w_set.sstorage):
+            w_item = self.wrap(obj)
+            if not w_other.has_key(w_item):
+                newsetdata[w_item] = None
+
+        w_iterator = w_other.iter()
+        while True:
+            w_item = w_iterator.next_entry()
+            if w_item is None:
+                break
+            if not w_set.has_key(w_item):
+                newsetdata[w_item] = None
+
+        strategy = self.space.fromcache(ObjectSetStrategy)
+        return strategy.erase(newsetdata)
+
+    def _symmetric_difference_base(self, w_set, w_other):
+        if self is w_other.strategy:
+            strategy = w_set.strategy
+            storage = self._symmetric_difference_unwrapped(w_set, w_other)
+        else:
+            strategy = self.space.fromcache(ObjectSetStrategy)
+            storage = self._symmetric_difference_wrapped(w_set, w_other)
+        return storage, strategy
+
+    def symmetric_difference(self, w_set, w_other):
+        storage, strategy = self._symmetric_difference_base(w_set, w_other)
+        return w_set.from_storage_and_strategy(storage, strategy)
+
+    def symmetric_difference_update(self, w_set, w_other):
+        storage, strategy = self._symmetric_difference_base(w_set, w_other)
+        w_set.strategy = strategy
+        w_set.sstorage = storage
+
+    def _intersect_base(self, w_set, w_other):
+        if self is w_other.strategy:
+            strategy = w_set.strategy
+            storage = strategy._intersect_unwrapped(w_set, w_other)
+        elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
+            strategy = self.space.fromcache(EmptySetStrategy)
+            storage = strategy.get_empty_storage()
+        else:
+            strategy = self.space.fromcache(ObjectSetStrategy)
+            storage = self._intersect_wrapped(w_set, w_other)
+        return storage, strategy
+
+    def _intersect_wrapped(self, w_set, w_other):
+        result = newset(self.space)
+        for key in self.unerase(w_set.sstorage):
+            w_key = self.wrap(key)
+            if w_other.has_key(w_key):
+                result[w_key] = None
+
+        strategy = self.space.fromcache(ObjectSetStrategy)
+        return strategy.erase(result)
+
+    def _intersect_unwrapped(self, w_set, w_other):
+        result = self.get_empty_dict()
+        d_this = self.unerase(w_set.sstorage)
+        d_other = self.unerase(w_other.sstorage)
+        for key in d_this:
+            if key in d_other:
+                result[key] = None
+        return self.erase(result)
+
+    def intersect(self, w_set, w_other):
+        if w_set.length() > w_other.length():
+            return w_other.intersect(w_set)
+
+        storage, strategy = self._intersect_base(w_set, w_other)
+        return w_set.from_storage_and_strategy(storage, strategy)
+
+    def intersect_update(self, w_set, w_other):
+        if w_set.length() > w_other.length():
+            w_intersection = w_other.intersect(w_set)
+            strategy = w_intersection.strategy
+            storage = w_intersection.sstorage
+        else:
+            storage, strategy = self._intersect_base(w_set, w_other)
+        w_set.strategy = strategy
+        w_set.sstorage = storage
+
+    def _issubset_unwrapped(self, w_set, w_other):
+        d_other = self.unerase(w_other.sstorage)
+        for item in self.unerase(w_set.sstorage):
+            if not item in d_other:
+                return False
+        return True
+
+    def _issubset_wrapped(self, w_set, w_other):
+        for obj in self.unerase(w_set.sstorage):
+            w_item = self.wrap(obj)
+            if not w_other.has_key(w_item):
+                return False
+        return True
+
+    def issubset(self, w_set, w_other):
+        if w_set.length() == 0:
+            return True
+
+        if w_set.strategy is w_other.strategy:
+            return self._issubset_unwrapped(w_set, w_other)
+        elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
+            return False
+        else:
+            return self._issubset_wrapped(w_set, w_other)
+
+    def _isdisjoint_unwrapped(self, w_set, w_other):
+        d_set = self.unerase(w_set.sstorage)
+        d_other = self.unerase(w_other.sstorage)
+        for key in d_set:
+            if key in d_other:
+                return False
+        return True
+
+    def _isdisjoint_wrapped(self, w_set, w_other):
+        d = self.unerase(w_set.sstorage)
+        for key in d:
+            if w_other.has_key(self.wrap(key)):
+                return False
+        return True
+
+    def isdisjoint(self, w_set, w_other):
+        if w_other.length() == 0:
+            return True
+        if w_set.length() > w_other.length():
+            return w_other.isdisjoint(w_set)
+
+        if w_set.strategy is w_other.strategy:
+            return self._isdisjoint_unwrapped(w_set, w_other)
+        elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
+            return True
+        else:
+            return self._isdisjoint_wrapped(w_set, w_other)
+
+    def update(self, w_set, w_other):
+        if self is w_other.strategy:
+            d_set = self.unerase(w_set.sstorage)
+            d_other = self.unerase(w_other.sstorage)
+            d_set.update(d_other)
+            return
+
+        w_set.switch_to_object_strategy(self.space)
+        w_set.update(w_other)
+
+    def popitem(self, w_set):
+        storage = self.unerase(w_set.sstorage)
+        try:
+            # this returns a tuple because internally sets are dicts
+            result = storage.popitem()
+        except KeyError:
+            # strategy may still be the same even if dict is empty
+            raise OperationError(self.space.w_KeyError,
+                            self.space.wrap('pop from an empty set'))
+        return self.wrap(result[0])
+
+class StringSetStrategy(AbstractUnwrappedSetStrategy, SetStrategy):
+    erase, unerase = rerased.new_erasing_pair("string")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def get_empty_storage(self):
+        return self.erase({})
+
+    def get_empty_dict(self):
+        return {}
+
+    def listview_str(self, w_set):
+        return self.unerase(w_set.sstorage).keys()
+
+    def is_correct_type(self, w_key):
+        return type(w_key) is W_StringObject
+
+    def may_contain_equal_elements(self, strategy):
+        if strategy is self.space.fromcache(IntegerSetStrategy):
+            return False
+        if strategy is self.space.fromcache(EmptySetStrategy):
+            return False
+        return True
+
+    def unwrap(self, w_item):
+        return self.space.str_w(w_item)
+
+    def wrap(self, item):
+        return self.space.wrap(item)
+
+    def iter(self, w_set):
+        return StringIteratorImplementation(self.space, self, w_set)
+
+class IntegerSetStrategy(AbstractUnwrappedSetStrategy, SetStrategy):
+    erase, unerase = rerased.new_erasing_pair("integer")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def get_empty_storage(self):
+        return self.erase({})
+
+    def get_empty_dict(self):
+        return {}
+
+    def listview_int(self, w_set):
+        return self.unerase(w_set.sstorage).keys()
+
+    def is_correct_type(self, w_key):
+        from pypy.objspace.std.intobject import W_IntObject
+        return type(w_key) is W_IntObject
+
+    def may_contain_equal_elements(self, strategy):
+        if strategy is self.space.fromcache(StringSetStrategy):
+            return False
+        if strategy is self.space.fromcache(EmptySetStrategy):
+            return False
+        return True
+
+    def unwrap(self, w_item):
+        return self.space.int_w(w_item)
+
+    def wrap(self, item):
+        return self.space.wrap(item)
+
+    def iter(self, w_set):
+        return IntegerIteratorImplementation(self.space, self, w_set)
+
+class ObjectSetStrategy(AbstractUnwrappedSetStrategy, SetStrategy):
+    erase, unerase = rerased.new_erasing_pair("object")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def get_empty_storage(self):
+        return self.erase(self.get_empty_dict())
+
+    def get_empty_dict(self):
+        return newset(self.space)
+
+    def is_correct_type(self, w_key):
+        return True
+
+    def may_contain_equal_elements(self, strategy):
+        if strategy is self.space.fromcache(EmptySetStrategy):
+            return False
+        return True
+
+    def unwrap(self, w_item):
+        return w_item
+
+    def wrap(self, item):
+        return item
+
+    def iter(self, w_set):
+        return RDictIteratorImplementation(self.space, self, w_set)
+
+    def update(self, w_set, w_other):
+        d_obj = self.unerase(w_set.sstorage)
+        w_iterator = w_other.iter()
+        while True:
+            w_item = w_iterator.next_entry()
+            if w_item is None:
+                break
+            d_obj[w_item] = None
+
+class IteratorImplementation(object):
+    def __init__(self, space, implementation):
+        self.space = space
+        self.setimplementation = implementation
+        self.len = implementation.length()
+        self.pos = 0
+
+    def next(self):
+        if self.setimplementation is None:
+            return None
+        if self.len != self.setimplementation.length():
+            self.len = -1   # Make this error state sticky
+            raise OperationError(self.space.w_RuntimeError,
+                     self.space.wrap("set changed size during iteration"))
+        # look for the next entry
+        if self.pos < self.len:
+            result = self.next_entry()
+            self.pos += 1
+            return result
+        # no more entries
+        self.setimplementation = None
+        return None
+
+    def next_entry(self):
+        """ Purely abstract method
+        """
+        raise NotImplementedError
+
+    def length(self):
+        if self.setimplementation is not None:
+            return self.len - self.pos
+        return 0
+
+class EmptyIteratorImplementation(IteratorImplementation):
+    def next_entry(self):
+        return None
+
+
+class StringIteratorImplementation(IteratorImplementation):
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, w_set)
+        d = strategy.unerase(w_set.sstorage)
+        self.iterator = d.iterkeys()
+
+    def next_entry(self):
+        for key in self.iterator:
+            return self.space.wrap(key)
+        else:
+            return None
+
+class IntegerIteratorImplementation(IteratorImplementation):
+    #XXX same implementation in dictmultiobject on dictstrategy-branch
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, dictimplementation)
+        d = strategy.unerase(dictimplementation.sstorage)
+        self.iterator = d.iterkeys()
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for key in self.iterator:
+            return self.space.wrap(key)
+        else:
+            return None
+
+class RDictIteratorImplementation(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, dictimplementation)
+        d = strategy.unerase(dictimplementation.sstorage)
+        self.iterator = d.iterkeys()
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for w_key in self.iterator:
             return w_key
         else:
             return None
 
+class W_SetIterObject(W_Object):
+    from pypy.objspace.std.settype import setiter_typedef as typedef
+    # XXX this class should be killed, and the various
+    # iterimplementations should be W_Objects directly.
+
+    def __init__(w_self, space, iterimplementation):
+        w_self.space = space
+        w_self.iterimplementation = iterimplementation
+
 registerimplementation(W_SetIterObject)
 
 def iter__SetIterObject(space, w_setiter):
     return w_setiter
 
 def next__SetIterObject(space, w_setiter):
-    content = w_setiter.content
-    if content is not None:
-        if w_setiter.len != len(content):
-            w_setiter.len = -1   # Make this error state sticky
-            raise OperationError(space.w_RuntimeError,
-                     space.wrap("Set changed size during iteration"))
-        # look for the next entry
-        w_result = w_setiter.next_entry()
-        if w_result is not None:
-            w_setiter.pos += 1
-            return w_result
-        # no more entries
-        w_setiter.content = None
+    iterimplementation = w_setiter.iterimplementation
+    w_key = iterimplementation.next()
+    if w_key is not None:
+        return w_key
     raise OperationError(space.w_StopIteration, space.w_None)
 
 # XXX __length_hint__()
@@ -116,107 +893,91 @@
 def newset(space):
     return r_dict(space.eq_w, space.hash_w, force_non_null=True)
 
-def make_setdata_from_w_iterable(space, w_iterable=None):
-    """Return a new r_dict with the content of w_iterable."""
+def set_strategy_and_setdata(space, w_set, w_iterable):
+    from pypy.objspace.std.intobject import W_IntObject
+    if w_iterable is None :
+        w_set.strategy = strategy = space.fromcache(EmptySetStrategy)
+        w_set.sstorage = strategy.get_empty_storage()
+        return
+
     if isinstance(w_iterable, W_BaseSetObject):
-        return w_iterable.setdata.copy()
-    data = newset(space)
-    if w_iterable is not None:
-        for w_item in space.listview(w_iterable):
-            data[w_item] = None
-    return data
+        w_set.strategy = w_iterable.strategy
+        w_set.sstorage = w_iterable.get_storage_copy()
+        return
+
+    stringlist = space.listview_str(w_iterable)
+    if stringlist is not None:
+        strategy = space.fromcache(StringSetStrategy)
+        w_set.strategy = strategy
+        w_set.sstorage = strategy.get_storage_from_unwrapped_list(stringlist)
+        return
+
+    intlist = space.listview_int(w_iterable)
+    if intlist is not None:
+        strategy = space.fromcache(IntegerSetStrategy)
+        w_set.strategy = strategy
+        w_set.sstorage = strategy.get_storage_from_unwrapped_list(intlist)
+        return
+
+    iterable_w = space.listview(w_iterable)
+
+    if len(iterable_w) == 0:
+        w_set.strategy = strategy = space.fromcache(EmptySetStrategy)
+        w_set.sstorage = strategy.get_empty_storage()
+        return
+
+    _pick_correct_strategy(space, w_set, iterable_w)
+
+def _pick_correct_strategy(space, w_set, iterable_w):
+    # check for integers
+    for w_item in iterable_w:
+        if type(w_item) is not W_IntObject:
+            break
+    else:
+        w_set.strategy = space.fromcache(IntegerSetStrategy)
+        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        return
+
+    # check for strings
+    for w_item in iterable_w:
+        if type(w_item) is not W_StringObject:
+            break
+    else:
+        w_set.strategy = space.fromcache(StringSetStrategy)
+        w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
+        return
+
+    w_set.strategy = space.fromcache(ObjectSetStrategy)
+    w_set.sstorage = w_set.strategy.get_storage_from_list(iterable_w)
 
 def _initialize_set(space, w_obj, w_iterable=None):
-    w_obj.setdata.clear()
-    if w_iterable is not None:
-        w_obj.setdata = make_setdata_from_w_iterable(space, w_iterable)
+    w_obj.clear()
+    set_strategy_and_setdata(space, w_obj, w_iterable)
 
 def _convert_set_to_frozenset(space, w_obj):
-    if space.isinstance_w(w_obj, space.w_set):
-        return W_FrozensetObject(space,
-                                 make_setdata_from_w_iterable(space, w_obj))
+    if isinstance(w_obj, W_SetObject):
+        w_frozen = W_FrozensetObject(space, None)
+        w_frozen.strategy = w_obj.strategy
+        w_frozen.sstorage = w_obj.sstorage
+        return w_frozen
+    elif space.isinstance_w(w_obj, space.w_set):
+        w_frz = space.allocate_instance(W_FrozensetObject, space.w_frozenset)
+        W_FrozensetObject.__init__(w_frz, space, w_obj)
+        return w_frz
     else:
         return None
 
-# helper functions for set operation on dicts
-
-def _is_eq(ld, rd):
-    if len(ld) != len(rd):
-        return False
-    for w_key in ld:
-        if w_key not in rd:
-            return False
-    return True
-
-def _difference_dict(space, ld, rd):
-    result = newset(space)
-    for w_key in ld:
-        if w_key not in rd:
-            result[w_key] = None
-    return result
-
-def _difference_dict_update(space, ld, rd):
-    if ld is rd:
-        ld.clear()     # for the case 'a.difference_update(a)'
-    else:
-        for w_key in rd:
-            try:
-                del ld[w_key]
-            except KeyError:
-                pass
-
-def _intersection_dict(space, ld, rd):
-    result = newset(space)
-    if len(ld) > len(rd):
-        ld, rd = rd, ld     # loop over the smaller dict
-    for w_key in ld:
-        if w_key in rd:
-            result[w_key] = None
-    return result
-
-def _isdisjoint_dict(ld, rd):
-    if len(ld) > len(rd):
-        ld, rd = rd, ld     # loop over the smaller dict
-    for w_key in ld:
-        if w_key in rd:
-            return False
-    return True
-
-def _symmetric_difference_dict(space, ld, rd):
-    result = newset(space)
-    for w_key in ld:
-        if w_key not in rd:
-            result[w_key] = None
-    for w_key in rd:
-        if w_key not in ld:
-            result[w_key] = None
-    return result
-
-def _issubset_dict(ldict, rdict):
-    if len(ldict) > len(rdict):
-        return False
-
-    for w_key in ldict:
-        if w_key not in rdict:
-            return False
-    return True
-
-
-#end helper functions
-
 def set_update__Set(space, w_left, others_w):
     """Update a set with the union of itself and another."""
-    ld = w_left.setdata
     for w_other in others_w:
         if isinstance(w_other, W_BaseSetObject):
-            ld.update(w_other.setdata)     # optimization only
+            w_left.update(w_other)     # optimization only
         else:
             for w_key in space.listview(w_other):
-                ld[w_key] = None
+                w_left.add(w_key)
 
 def inplace_or__Set_Set(space, w_left, w_other):
-    ld, rd = w_left.setdata, w_other.setdata
-    ld.update(rd)
+    w_left.update(w_other)
     return w_left
 
 inplace_or__Set_Frozenset = inplace_or__Set_Set
@@ -226,10 +987,10 @@
 
     This has no effect if the element is already present.
     """
-    w_left.setdata[w_other] = None
+    w_left.add(w_other)
 
 def set_copy__Set(space, w_set):
-    return w_set._newobj(space, w_set.setdata.copy())
+    return w_set.copy_real()
 
 def frozenset_copy__Frozenset(space, w_left):
     if type(w_left) is W_FrozensetObject:
@@ -238,63 +999,51 @@
         return set_copy__Set(space, w_left)
 
 def set_clear__Set(space, w_left):
-    w_left.setdata.clear()
+    w_left.clear()
 
 def sub__Set_Set(space, w_left, w_other):
-    ld, rd = w_left.setdata, w_other.setdata
-    new_ld = _difference_dict(space, ld, rd)
-    return w_left._newobj(space, new_ld)
+    return w_left.difference(w_other)
 
 sub__Set_Frozenset = sub__Set_Set
 sub__Frozenset_Set = sub__Set_Set
 sub__Frozenset_Frozenset = sub__Set_Set
 
 def set_difference__Set(space, w_left, others_w):
-    result = w_left.setdata
-    if len(others_w) == 0:
-        result = result.copy()
-    for w_other in others_w:
-        if isinstance(w_other, W_BaseSetObject):
-            rd = w_other.setdata     # optimization only
-        else:
-            rd = make_setdata_from_w_iterable(space, w_other)
-        result = _difference_dict(space, result, rd)
-    return w_left._newobj(space, result)
+    result = w_left.copy_real()
+    set_difference_update__Set(space, result, others_w)
+    return result
 
 frozenset_difference__Frozenset = set_difference__Set
 
 
 def set_difference_update__Set(space, w_left, others_w):
-    ld = w_left.setdata
     for w_other in others_w:
         if isinstance(w_other, W_BaseSetObject):
             # optimization only
-            _difference_dict_update(space, ld, w_other.setdata)
+            w_left.difference_update(w_other)
         else:
-            for w_key in space.listview(w_other):
-                try:
-                    del ld[w_key]
-                except KeyError:
-                    pass
+            w_other_as_set = w_left._newobj(space, w_other)
+            w_left.difference_update(w_other_as_set)
 
 def inplace_sub__Set_Set(space, w_left, w_other):
-    ld, rd = w_left.setdata, w_other.setdata
-    _difference_dict_update(space, ld, rd)
+    w_left.difference_update(w_other)
     return w_left
 
 inplace_sub__Set_Frozenset = inplace_sub__Set_Set
 
 def eq__Set_Set(space, w_left, w_other):
     # optimization only (the general case is eq__Set_settypedef)
-    return space.wrap(_is_eq(w_left.setdata, w_other.setdata))
+    return space.wrap(w_left.equals(w_other))
 
 eq__Set_Frozenset = eq__Set_Set
 eq__Frozenset_Frozenset = eq__Set_Set
 eq__Frozenset_Set = eq__Set_Set
 
 def eq__Set_settypedef(space, w_left, w_other):
-    rd = make_setdata_from_w_iterable(space, w_other)
-    return space.wrap(_is_eq(w_left.setdata, rd))
+    # tested in test_buildinshortcut.py
+    #XXX do not make new setobject here
+    w_other_as_set = w_left._newobj(space, w_other)
+    return space.wrap(w_left.equals(w_other_as_set))
 
 eq__Set_frozensettypedef = eq__Set_settypedef
 eq__Frozenset_settypedef = eq__Set_settypedef
@@ -308,15 +1057,16 @@
 eq__Frozenset_ANY = eq__Set_ANY
 
 def ne__Set_Set(space, w_left, w_other):
-    return space.wrap(not _is_eq(w_left.setdata, w_other.setdata))
+    return space.wrap(not w_left.equals(w_other))
 
 ne__Set_Frozenset = ne__Set_Set
 ne__Frozenset_Frozenset = ne__Set_Set
 ne__Frozenset_Set = ne__Set_Set
 
 def ne__Set_settypedef(space, w_left, w_other):
-    rd = make_setdata_from_w_iterable(space, w_other)
-    return space.wrap(not _is_eq(w_left.setdata, rd))
+    #XXX this is not tested
+    w_other_as_set = w_left._newobj(space, w_other)
+    return space.wrap(not w_left.equals(w_other_as_set))
 
 ne__Set_frozensettypedef = ne__Set_settypedef
 ne__Frozenset_settypedef = ne__Set_settypedef
@@ -331,12 +1081,12 @@
 
 def contains__Set_ANY(space, w_left, w_other):
     try:
-        return space.newbool(w_other in w_left.setdata)
+        return space.newbool(w_left.has_key(w_other))
     except OperationError, e:
         if e.match(space, space.w_TypeError):
             w_f = _convert_set_to_frozenset(space, w_other)
             if w_f is not None:
-                return space.newbool(w_f in w_left.setdata)
+                return space.newbool(w_left.has_key(w_f))
         raise
 
 contains__Frozenset_ANY = contains__Set_ANY
@@ -345,19 +1095,23 @@
     # optimization only (the general case works too)
     if space.is_w(w_left, w_other):
         return space.w_True
-    ld, rd = w_left.setdata, w_other.setdata
-    return space.wrap(_issubset_dict(ld, rd))
+    if w_left.length() > w_other.length():
+        return space.w_False
+    return space.wrap(w_left.issubset(w_other))
 
 set_issubset__Set_Frozenset = set_issubset__Set_Set
 frozenset_issubset__Frozenset_Set = set_issubset__Set_Set
 frozenset_issubset__Frozenset_Frozenset = set_issubset__Set_Set
 
 def set_issubset__Set_ANY(space, w_left, w_other):
-    if space.is_w(w_left, w_other):
-        return space.w_True
+    # not checking whether w_left is w_other here, because if that were the
+    # case the more precise multimethod would have applied.
 
-    ld, rd = w_left.setdata, make_setdata_from_w_iterable(space, w_other)
-    return space.wrap(_issubset_dict(ld, rd))
+    w_other_as_set = w_left._newobj(space, w_other)
+
+    if w_left.length() > w_other_as_set.length():
+        return space.w_False
+    return space.wrap(w_left.issubset(w_other_as_set))
 
 frozenset_issubset__Frozenset_ANY = set_issubset__Set_ANY
 
@@ -370,9 +1124,9 @@
     # optimization only (the general case works too)
     if space.is_w(w_left, w_other):
         return space.w_True
-
-    ld, rd = w_left.setdata, w_other.setdata
-    return space.wrap(_issubset_dict(rd, ld))
+    if w_left.length() < w_other.length():
+        return space.w_False
+    return space.wrap(w_other.issubset(w_left))
 
 set_issuperset__Set_Frozenset = set_issuperset__Set_Set
 set_issuperset__Frozenset_Set = set_issuperset__Set_Set
@@ -382,8 +1136,11 @@
     if space.is_w(w_left, w_other):
         return space.w_True
 
-    ld, rd = w_left.setdata, make_setdata_from_w_iterable(space, w_other)
-    return space.wrap(_issubset_dict(rd, ld))
+    w_other_as_set = w_left._newobj(space, w_other)
+
+    if w_left.length() < w_other_as_set.length():
+        return space.w_False
+    return space.wrap(w_other_as_set.issubset(w_left))
 
 frozenset_issuperset__Frozenset_ANY = set_issuperset__Set_ANY
 
@@ -395,7 +1152,7 @@
 # automatic registration of "lt(x, y)" as "not ge(y, x)" would not give the
 # correct answer here!
 def lt__Set_Set(space, w_left, w_other):
-    if len(w_left.setdata) >= len(w_other.setdata):
+    if w_left.length() >= w_other.length():
         return space.w_False
     else:
         return le__Set_Set(space, w_left, w_other)
@@ -405,7 +1162,7 @@
 lt__Frozenset_Frozenset = lt__Set_Set
 
 def gt__Set_Set(space, w_left, w_other):
-    if len(w_left.setdata) <= len(w_other.setdata):
+    if w_left.length() <= w_other.length():
         return space.w_False
     else:
         return ge__Set_Set(space, w_left, w_other)
@@ -421,26 +1178,19 @@
     Returns True if successfully removed.
     """
     try:
-        del w_left.setdata[w_item]
-        return True
-    except KeyError:
-        return False
+        deleted = w_left.remove(w_item)
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_f = _convert_set_to_frozenset(space, w_item)
-        if w_f is None:
-            raise
+        else:
+            w_f = _convert_set_to_frozenset(space, w_item)
+            if w_f is None:
+                raise
+            deleted = w_left.remove(w_f)
 
-    try:
-        del w_left.setdata[w_f]
-        return True
-    except KeyError:
-        return False
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        return False
+    if w_left.length() == 0:
+        w_left.switch_to_empty_strategy()
+    return deleted
 
 def set_discard__Set_ANY(space, w_left, w_item):
     _discard_from_set(space, w_left, w_item)
@@ -454,8 +1204,12 @@
     if w_set.hash != 0:
         return space.wrap(w_set.hash)
     hash = r_uint(1927868237)
-    hash *= r_uint(len(w_set.setdata) + 1)
-    for w_item in w_set.setdata:
+    hash *= r_uint(w_set.length() + 1)
+    w_iterator = w_set.iter()
+    while True:
+        w_item = w_iterator.next_entry()
+        if w_item is None:
+            break
         h = space.hash_w(w_item)
         value = (r_uint(h ^ (h << 16) ^ 89869747)  * multi)
         hash = hash ^ value
@@ -468,71 +1222,75 @@
     return space.wrap(hash)
 
 def set_pop__Set(space, w_left):
-    try:
-        w_key, _ = w_left.setdata.popitem()
-    except KeyError:
-        raise OperationError(space.w_KeyError,
-                                space.wrap('pop from an empty set'))
-    return w_key
+    return w_left.popitem()
 
 def and__Set_Set(space, w_left, w_other):
-    ld, rd = w_left.setdata, w_other.setdata
-    new_ld = _intersection_dict(space, ld, rd)
-    return w_left._newobj(space, new_ld)
+    new_set = w_left.intersect(w_other)
+    return new_set
 
 and__Set_Frozenset = and__Set_Set
 and__Frozenset_Set = and__Set_Set
 and__Frozenset_Frozenset = and__Set_Set
 
-def _intersection_multiple(space, w_left, others_w):
-    result = w_left.setdata
-    for w_other in others_w:
+def set_intersection__Set(space, w_left, others_w):
+    #XXX find smarter implementations
+    others_w = [w_left] + others_w
+
+    # find smallest set in others_w to reduce comparisons
+    startindex, startlength = 0, -1
+    for i in range(len(others_w)):
+        w_other = others_w[i]
+        try:
+            length = space.int_w(space.len(w_other))
+        except OperationError, e:
+            if (e.match(space, space.w_TypeError) or
+                e.match(space, space.w_AttributeError)):
+                continue
+            raise
+
+        if startlength == -1 or length < startlength:
+            startindex = i
+            startlength = length
+
+    others_w[startindex], others_w[0] = others_w[0], others_w[startindex]
+
+    result = w_left._newobj(space, others_w[0])
+    for i in range(1,len(others_w)):
+        w_other = others_w[i]
         if isinstance(w_other, W_BaseSetObject):
             # optimization only
-            result = _intersection_dict(space, result, w_other.setdata)
+            result.intersect_update(w_other)
         else:
-            result2 = newset(space)
-            for w_key in space.listview(w_other):
-                if w_key in result:
-                    result2[w_key] = None
-            result = result2
+            w_other_as_set = w_left._newobj(space, w_other)
+            result.intersect_update(w_other_as_set)
     return result
 
-def set_intersection__Set(space, w_left, others_w):
-    if len(others_w) == 0:
-        result = w_left.setdata.copy()
-    else:
-        result = _intersection_multiple(space, w_left, others_w)
-    return w_left._newobj(space, result)
-
 frozenset_intersection__Frozenset = set_intersection__Set
 
 def set_intersection_update__Set(space, w_left, others_w):
-    result = _intersection_multiple(space, w_left, others_w)
-    w_left.setdata = result
+    result = set_intersection__Set(space, w_left, others_w)
+    w_left.strategy = result.strategy
+    w_left.sstorage = result.sstorage
+    return
 
 def inplace_and__Set_Set(space, w_left, w_other):
-    ld, rd = w_left.setdata, w_other.setdata
-    new_ld = _intersection_dict(space, ld, rd)
-    w_left.setdata = new_ld
+    w_left.intersect_update(w_other)
     return w_left
 
 inplace_and__Set_Frozenset = inplace_and__Set_Set
 
 def set_isdisjoint__Set_Set(space, w_left, w_other):
     # optimization only (the general case works too)
-    ld, rd = w_left.setdata, w_other.setdata
-    disjoint = _isdisjoint_dict(ld, rd)
-    return space.newbool(disjoint)
+    return space.newbool(w_left.isdisjoint(w_other))
 
 set_isdisjoint__Set_Frozenset = set_isdisjoint__Set_Set
 set_isdisjoint__Frozenset_Frozenset = set_isdisjoint__Set_Set
 set_isdisjoint__Frozenset_Set = set_isdisjoint__Set_Set
 
 def set_isdisjoint__Set_ANY(space, w_left, w_other):
-    ld = w_left.setdata
+    #XXX may be optimized when other strategies are added
     for w_key in space.listview(w_other):
-        if w_key in ld:
+        if w_left.has_key(w_key):
             return space.w_False
     return space.w_True
 
@@ -540,9 +1298,8 @@
 
 def set_symmetric_difference__Set_Set(space, w_left, w_other):
     # optimization only (the general case works too)
-    ld, rd = w_left.setdata, w_other.setdata
-    new_ld = _symmetric_difference_dict(space, ld, rd)
-    return w_left._newobj(space, new_ld)
+    w_result = w_left.symmetric_difference(w_other)
+    return w_result
 
 set_symmetric_difference__Set_Frozenset = set_symmetric_difference__Set_Set
 set_symmetric_difference__Frozenset_Set = set_symmetric_difference__Set_Set
@@ -556,26 +1313,23 @@
 
 
 def set_symmetric_difference__Set_ANY(space, w_left, w_other):
-    ld, rd = w_left.setdata, make_setdata_from_w_iterable(space, w_other)
-    new_ld = _symmetric_difference_dict(space, ld, rd)
-    return w_left._newobj(space, new_ld)
+    w_other_as_set = w_left._newobj(space, w_other)
+    w_result = w_left.symmetric_difference(w_other_as_set)
+    return w_result
 
 frozenset_symmetric_difference__Frozenset_ANY = \
         set_symmetric_difference__Set_ANY
 
 def set_symmetric_difference_update__Set_Set(space, w_left, w_other):
     # optimization only (the general case works too)
-    ld, rd = w_left.setdata, w_other.setdata
-    new_ld = _symmetric_difference_dict(space, ld, rd)
-    w_left.setdata = new_ld
+    w_left.symmetric_difference_update(w_other)
 
 set_symmetric_difference_update__Set_Frozenset = \
                                     set_symmetric_difference_update__Set_Set
 
 def set_symmetric_difference_update__Set_ANY(space, w_left, w_other):
-    ld, rd = w_left.setdata, make_setdata_from_w_iterable(space, w_other)
-    new_ld = _symmetric_difference_dict(space, ld, rd)
-    w_left.setdata = new_ld
+    w_other_as_set = w_left._newobj(space, w_other)
+    w_left.symmetric_difference_update(w_other_as_set)
 
 def inplace_xor__Set_Set(space, w_left, w_other):
     set_symmetric_difference_update__Set_Set(space, w_left, w_other)
@@ -584,34 +1338,33 @@
 inplace_xor__Set_Frozenset = inplace_xor__Set_Set
 
 def or__Set_Set(space, w_left, w_other):
-    ld, rd = w_left.setdata, w_other.setdata
-    result = ld.copy()
-    result.update(rd)
-    return w_left._newobj(space, result)
+    w_copy = w_left.copy_real()
+    w_copy.update(w_other)
+    return w_copy
 
 or__Set_Frozenset = or__Set_Set
 or__Frozenset_Set = or__Set_Set
 or__Frozenset_Frozenset = or__Set_Set
 
 def set_union__Set(space, w_left, others_w):
-    result = w_left.setdata.copy()
+    result = w_left.copy_real()
     for w_other in others_w:
         if isinstance(w_other, W_BaseSetObject):
-            result.update(w_other.setdata)     # optimization only
+            result.update(w_other)     # optimization only
         else:
             for w_key in space.listview(w_other):
-                result[w_key] = None
-    return w_left._newobj(space, result)
+                result.add(w_key)
+    return result
 
 frozenset_union__Frozenset = set_union__Set
 
 def len__Set(space, w_left):
-    return space.newint(len(w_left.setdata))
+    return space.newint(w_left.length())
 
 len__Frozenset = len__Set
 
 def iter__Set(space, w_left):
-    return W_SetIterObject(w_left.setdata)
+    return W_SetIterObject(space, w_left.iter())
 
 iter__Frozenset = iter__Set
 
diff --git a/pypy/objspace/std/settype.py b/pypy/objspace/std/settype.py
--- a/pypy/objspace/std/settype.py
+++ b/pypy/objspace/std/settype.py
@@ -68,7 +68,7 @@
 def descr__new__(space, w_settype, __args__):
     from pypy.objspace.std.setobject import W_SetObject, newset
     w_obj = space.allocate_instance(W_SetObject, w_settype)
-    W_SetObject.__init__(w_obj, space, newset(space))
+    W_SetObject.__init__(w_obj, space)
     return w_obj
 
 set_typedef = StdTypeDef("set",
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -69,6 +69,14 @@
     def str_w(w_self, space):
         return w_self._value
 
+    def listview_str(w_self):
+        return _create_list_from_string(w_self._value)
+
+def _create_list_from_string(value):
+    # need this helper function to allow the jit to look inside and inline
+    # listview_str
+    return [s for s in value]
+
 registerimplementation(W_StringObject)
 
 W_StringObject.EMPTY = W_StringObject('')
diff --git a/pypy/objspace/std/test/test_builtinshortcut.py b/pypy/objspace/std/test/test_builtinshortcut.py
--- a/pypy/objspace/std/test/test_builtinshortcut.py
+++ b/pypy/objspace/std/test/test_builtinshortcut.py
@@ -85,6 +85,20 @@
     def setup_class(cls):
         from pypy import conftest
         cls.space = conftest.gettestobjspace(**WITH_BUILTINSHORTCUT)
+        w_fakeint = cls.space.appexec([], """():
+            class FakeInt(object):
+                def __init__(self, value):
+                    self.value = value
+                def __hash__(self):
+                    return hash(self.value)
+
+                def __eq__(self, other):
+                    if other == self.value:
+                        return True
+                    return False
+            return FakeInt
+            """)
+        cls.w_FakeInt = w_fakeint
 
 class AppTestString(test_stringobject.AppTestStringObject):
     def setup_class(cls):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -131,6 +131,45 @@
         assert self.space.eq_w(space.call_function(get, w("33")), w(None))
         assert self.space.eq_w(space.call_function(get, w("33"), w(44)), w(44))
 
+    def test_fromkeys_fastpath(self):
+        space = self.space
+        w = space.wrap
+
+        w_l = self.space.newlist([w("a"),w("b")])
+        w_l.getitems = None
+        w_d = space.call_method(space.w_dict, "fromkeys", w_l)
+
+        assert space.eq_w(w_d.getitem_str("a"), space.w_None)
+        assert space.eq_w(w_d.getitem_str("b"), space.w_None)
+
+    def test_listview_str_dict(self):
+        w = self.space.wrap
+
+        w_d = self.space.newdict()
+        w_d.initialize_content([(w("a"), w(1)), (w("b"), w(2))])
+
+        assert self.space.listview_str(w_d) == ["a", "b"]
+
+    def test_listview_int_dict(self):
+        w = self.space.wrap
+        w_d = self.space.newdict()
+        w_d.initialize_content([(w(1), w("a")), (w(2), w("b"))])
+
+        assert self.space.listview_int(w_d) == [1, 2]
+
+    def test_keys_on_string_int_dict(self):
+        w = self.space.wrap
+        w_d = self.space.newdict()
+        w_d.initialize_content([(w(1), w("a")), (w(2), w("b"))])
+
+        w_l = self.space.call_method(w_d, "keys")
+        assert sorted(self.space.listview_int(w_l)) == [1,2]
+
+        w_d = self.space.newdict()
+        w_d.initialize_content([(w("a"), w(1)), (w("b"), w(6))])
+
+        w_l = self.space.call_method(w_d, "keys")
+        assert sorted(self.space.listview_str(w_l)) == ["a", "b"]
 
 class AppTest_DictObject:
     def setup_class(cls):
@@ -793,7 +832,9 @@
         return x == y
     eq_w = eq
     def newlist(self, l):
-        return []
+        return l
+    def newlist_str(self, l):
+        return l
     DictObjectCls = W_DictMultiObject
     def type(self, w_obj):
         if isinstance(w_obj, FakeString):
@@ -933,7 +974,7 @@
 
     def test_keys(self):
         self.fill_impl()
-        keys = self.impl.keys()
+        keys = self.impl.w_keys() # wrapped lists = lists in the fake space
         keys.sort()
         assert keys == [self.string, self.string2]
         self.check_not_devolved()
@@ -1011,8 +1052,8 @@
         d.setitem("s", 12)
         d.delitem(F())
 
-        assert "s" not in d.keys()
-        assert F() not in d.keys()
+        assert "s" not in d.w_keys()
+        assert F() not in d.w_keys()
 
 class TestStrDictImplementation(BaseTestRDictImplementation):
     StrategyClass = StringDictStrategy
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -486,6 +486,14 @@
         list.__init__(l, ['a', 'b', 'c'])
         assert l is l0
         assert l == ['a', 'b', 'c']
+        list.__init__(l)
+        assert l == []
+
+    def test_explicit_new_init_more_cases(self):
+        for assignment in [[], (), [3], ["foo"]]:
+            l = [1, 2]
+            l.__init__(assignment)
+            assert l == list(assignment)
 
     def test_extend_list(self):
         l = l0 = [1]
@@ -1173,6 +1181,20 @@
         assert l == []
         assert list(g) == []
 
+    def test_uses_custom_iterator(self):
+        # obscure corner case: space.listview*() must not shortcut subclasses
+        # of dicts, because the OrderedDict in the stdlib relies on this.
+        # we extend the use case to lists and sets, i.e. all types that have
+        # strategies, to avoid surprizes depending on the strategy.
+        for base, arg in [(list, []), (list, [5]), (list, ['x']),
+                          (set, []),  (set,  [5]), (set,  ['x']),
+                          (dict, []), (dict, [(5,6)]), (dict, [('x',7)])]:
+            print base, arg
+            class SubClass(base):
+                def __iter__(self):
+                    return iter("foobar")
+            assert list(SubClass(arg)) == ['f', 'o', 'o', 'b', 'a', 'r']
+
 class AppTestForRangeLists(AppTestW_ListObject):
 
     def setup_class(cls):
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -420,7 +420,7 @@
 
     def test_listview_str(self):
         space = self.space
-        assert space.listview_str(space.wrap("a")) is None
+        assert space.listview_str(space.wrap(1)) == None
         w_l = self.space.newlist([self.space.wrap('a'), self.space.wrap('b')])
         assert space.listview_str(w_l) == ["a", "b"]
 
@@ -463,6 +463,44 @@
         w_res = listobject.list_pop__List_ANY(space, w_l, space.w_None) # does not crash
         assert space.unwrap(w_res) == 3
 
+    def test_create_list_from_set(self):
+        from pypy.objspace.std.setobject import W_SetObject
+        from pypy.objspace.std.setobject import _initialize_set
+
+        space = self.space
+        w = space.wrap
+
+        w_l = W_ListObject(space, [space.wrap(1), space.wrap(2), space.wrap(3)])
+
+        w_set = W_SetObject(self.space)
+        _initialize_set(self.space, w_set, w_l)
+        w_set.iter = None # make sure fast path is used
+
+        w_l2 = W_ListObject(space, [])
+        space.call_method(w_l2, "__init__", w_set)
+
+        w_l2.sort(False)
+        assert space.eq_w(w_l, w_l2)
+
+        w_l = W_ListObject(space, [space.wrap("a"), space.wrap("b"), space.wrap("c")])
+        _initialize_set(self.space, w_set, w_l)
+
+        space.call_method(w_l2, "__init__", w_set)
+
+        w_l2.sort(False)
+        assert space.eq_w(w_l, w_l2)
+
+
+    def test_listview_str_list(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap("a"), space.wrap("b")])
+        assert self.space.listview_str(w_l) == ["a", "b"]
+
+    def test_listview_int_list(self):
+        space = self.space
+        w_l = W_ListObject(space, [space.wrap(1), space.wrap(2), space.wrap(3)])
+        assert self.space.listview_int(w_l) == [1, 2, 3]
+
 
 class TestW_ListStrategiesDisabled:
     def setup_class(cls):
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -8,12 +8,14 @@
 is not too wrong.
 """
 import py.test
-from pypy.objspace.std.setobject import W_SetObject, W_FrozensetObject
+from pypy.objspace.std.setobject import W_SetObject, W_FrozensetObject, IntegerSetStrategy
 from pypy.objspace.std.setobject import _initialize_set
-from pypy.objspace.std.setobject import newset, make_setdata_from_w_iterable
+from pypy.objspace.std.setobject import newset
 from pypy.objspace.std.setobject import and__Set_Set
 from pypy.objspace.std.setobject import set_intersection__Set
 from pypy.objspace.std.setobject import eq__Set_Set
+from pypy.conftest import gettestobjspace
+from pypy.objspace.std.listobject import W_ListObject
 
 letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
 
@@ -29,12 +31,11 @@
         self.false = self.space.w_False
 
     def test_and(self):
-        s = W_SetObject(self.space, newset(self.space))
+        s = W_SetObject(self.space)
         _initialize_set(self.space, s, self.word)
-        t0 = W_SetObject(self.space, newset(self.space))
+        t0 = W_SetObject(self.space)
         _initialize_set(self.space, t0, self.otherword)
-        t1 = W_FrozensetObject(self.space,
-                make_setdata_from_w_iterable(self.space, self.otherword))
+        t1 = W_FrozensetObject(self.space, self.otherword)
         r0 = and__Set_Set(self.space, s, t0)
         r1 = and__Set_Set(self.space, s, t1)
         assert eq__Set_Set(self.space, r0, r1) == self.true
@@ -42,9 +43,9 @@
         assert eq__Set_Set(self.space, r0, sr) == self.true
 
     def test_compare(self):
-        s = W_SetObject(self.space, newset(self.space))
+        s = W_SetObject(self.space)
         _initialize_set(self.space, s, self.word)
-        t = W_SetObject(self.space, newset(self.space))
+        t = W_SetObject(self.space)
         _initialize_set(self.space, t, self.word)
         assert self.space.eq_w(s,t)
         u = self.space.wrap(set('simsalabim'))
@@ -54,7 +55,247 @@
         s = self.space.newset()
         assert self.space.str_w(self.space.repr(s)) == 'set([])'
 
+    def test_intersection_order(self):
+        # theses tests make sure that intersection is done in the correct order
+        # (smallest first)
+        space = self.space
+        a = W_SetObject(self.space)
+        _initialize_set(self.space, a, self.space.wrap("abcdefg"))
+        a.intersect = None
+
+        b = W_SetObject(self.space)
+        _initialize_set(self.space, b, self.space.wrap("abc"))
+
+        result = set_intersection__Set(space, a, [b])
+        assert space.is_true(self.space.eq(result, W_SetObject(space, self.space.wrap("abc"))))
+
+        c = W_SetObject(self.space)
+        _initialize_set(self.space, c, self.space.wrap("e"))
+
+        d = W_SetObject(self.space)
+        _initialize_set(self.space, d, self.space.wrap("ab"))
+
+        # if ordering works correct we should start with set e
+        a.get_storage_copy = None
+        b.get_storage_copy = None
+        d.get_storage_copy = None
+
+        result = set_intersection__Set(space, a, [d,c,b])
+        assert space.is_true(self.space.eq(result, W_SetObject(space, self.space.wrap(""))))
+
+    def test_create_set_from_list(self):
+        from pypy.objspace.std.setobject import ObjectSetStrategy, StringSetStrategy
+        from pypy.objspace.std.floatobject import W_FloatObject
+        from pypy.objspace.std.model import W_Object
+
+        w = self.space.wrap
+        intstr = self.space.fromcache(IntegerSetStrategy)
+        tmp_func = intstr.get_storage_from_list
+        # test if get_storage_from_list is no longer used
+        intstr.get_storage_from_list = None
+
+        w_list = W_ListObject(self.space, [w(1), w(2), w(3)])
+        w_set = W_SetObject(self.space)
+        _initialize_set(self.space, w_set, w_list)
+        assert w_set.strategy is intstr
+        assert intstr.unerase(w_set.sstorage) == {1:None, 2:None, 3:None}
+
+        w_list = W_ListObject(self.space, [w("1"), w("2"), w("3")])
+        w_set = W_SetObject(self.space)
+        _initialize_set(self.space, w_set, w_list)
+        assert w_set.strategy is self.space.fromcache(StringSetStrategy)
+        assert w_set.strategy.unerase(w_set.sstorage) == {"1":None, "2":None, "3":None}
+
+        w_list = W_ListObject(self.space, [w("1"), w(2), w("3")])
+        w_set = W_SetObject(self.space)
+        _initialize_set(self.space, w_set, w_list)
+        assert w_set.strategy is self.space.fromcache(ObjectSetStrategy)
+        for item in w_set.strategy.unerase(w_set.sstorage):
+            assert isinstance(item, W_Object)
+
+        w_list = W_ListObject(self.space, [w(1.0), w(2.0), w(3.0)])
+        w_set = W_SetObject(self.space)
+        _initialize_set(self.space, w_set, w_list)
+        assert w_set.strategy is self.space.fromcache(ObjectSetStrategy)
+        for item in w_set.strategy.unerase(w_set.sstorage):
+            assert isinstance(item, W_FloatObject)
+
+        # changed cached object, need to change it back for other tests to pass
+        intstr.get_storage_from_list = tmp_func
+
+    def test_listview_str_int_on_set(self):
+        w = self.space.wrap
+
+        w_a = W_SetObject(self.space)
+        _initialize_set(self.space, w_a, w("abcdefg"))
+        assert sorted(self.space.listview_str(w_a)) == list("abcdefg")
+        assert self.space.listview_int(w_a) is None
+
+        w_b = W_SetObject(self.space)
+        _initialize_set(self.space, w_b, self.space.newlist([w(1),w(2),w(3),w(4),w(5)]))
+        assert sorted(self.space.listview_int(w_b)) == [1,2,3,4,5]
+        assert self.space.listview_str(w_b) is None
+
 class AppTestAppSetTest:
+
+    def setup_class(self):
+        self.space = gettestobjspace()
+        w_fakeint = self.space.appexec([], """():
+            class FakeInt(object):
+                def __init__(self, value):
+                    self.value = value
+                def __hash__(self):
+                    return hash(self.value)
+
+                def __eq__(self, other):
+                    if other == self.value:
+                        return True
+                    return False
+            return FakeInt
+            """)
+        self.w_FakeInt = w_fakeint
+
+    def test_fakeint(self):
+        f1 = self.FakeInt(4)
+        assert f1 == 4
+        assert hash(f1) == hash(4)
+
+    def test_simple(self):
+        a = set([1,2,3])
+        b = set()
+        b.add(4)
+        c = a.union(b)
+        assert c == set([1,2,3,4])
+
+    def test_generator(self):
+        def foo():
+            for i in [1,2,3,4,5]:
+                yield i
+        b = set(foo())
+        assert b == set([1,2,3,4,5])
+
+        a = set(x for x in [1,2,3])
+        assert a == set([1,2,3])
+
+    def test_generator2(self):
+        def foo():
+            for i in [1,2,3]:
+                yield i
+        class A(set):
+            pass
+        a = A([1,2,3,4,5])
+        b = a.difference(foo())
+        assert b == set([4,5])
+
+    def test_or(self):
+        a = set([0,1,2])
+        b = a | set([1,2,3])
+        assert b == set([0,1,2,3])
+
+        # test inplace or
+        a |= set([1,2,3])
+        assert a == b
+
+    def test_clear(self):
+        a = set([1,2,3])
+        a.clear()
+        assert a == set()
+
+    def test_sub(self):
+        a = set([1,2,3,4,5])
+        b = set([2,3,4])
+        a - b == [1,5]
+        a.__sub__(b) == [1,5]
+
+        #inplace sub
+        a = set([1,2,3,4])
+        b = set([1,4])
+        a -= b
+        assert a == set([2,3])
+
+    def test_issubset(self):
+        a = set([1,2,3,4])
+        b = set([2,3])
+        assert b.issubset(a)
+        c = [1,2,3,4]
+        assert b.issubset(c)
+
+        a = set([1,2,3,4])
+        b = set(['1','2'])
+        assert not b.issubset(a)
+
+    def test_issuperset(self):
+        a = set([1,2,3,4])
+        b = set([2,3])
+        assert a.issuperset(b)
+        c = [2,3]
+        assert a.issuperset(c)
+
+        c = [1,1,1,1,1]
+        assert a.issuperset(c)
+        assert set([1,1,1,1,1]).issubset(a)
+
+        a = set([1,2,3])
+        assert a.issuperset(a)
+        assert not a.issuperset(set([1,2,3,4,5]))
+
+    def test_inplace_and(test):
+        a = set([1,2,3,4])
+        b = set([0,2,3,5,6])
+        a &= b
+        assert a == set([2,3])
+
+    def test_discard_remove(self):
+        a = set([1,2,3,4,5])
+        a.remove(1)
+        assert a == set([2,3,4,5])
+        a.discard(2)
+        assert a == set([3,4,5])
+
+        raises(KeyError, "a.remove(6)")
+
+    def test_pop(self):
+        b = set()
+        raises(KeyError, "b.pop()")
+
+        a = set([1,2,3,4,5])
+        for i in xrange(5):
+            a.pop()
+        assert a == set()
+        raises(KeyError, "a.pop()")
+
+    def test_symmetric_difference(self):
+        a = set([1,2,3])
+        b = set([3,4,5])
+        c = a.symmetric_difference(b)
+        assert c == set([1,2,4,5])
+
+        a = set([1,2,3])
+        b = [3,4,5]
+        c = a.symmetric_difference(b)
+        assert c == set([1,2,4,5])
+
+        a = set([1,2,3])
+        b = set('abc')
+        c = a.symmetric_difference(b)
+        assert c == set([1,2,3,'a','b','c'])
+
+    def test_symmetric_difference_update(self):
+        a = set([1,2,3])
+        b = set([3,4,5])
+        a.symmetric_difference_update(b)
+        assert a == set([1,2,4,5])
+
+        a = set([1,2,3])
+        b = [3,4,5]
+        a.symmetric_difference_update(b)
+        assert a == set([1,2,4,5])
+
+        a = set([1,2,3])
+        b = set([3,4,5])
+        a ^= b
+        assert a == set([1,2,4,5])
+
     def test_subtype(self):
         class subset(set):pass
         a = subset()
@@ -131,6 +372,8 @@
         assert (set('abc') != set('abcd'))
         assert (frozenset('abc') != frozenset('abcd'))
         assert (frozenset('abc') != set('abcd'))
+        assert set() != set('abc')
+        assert set('abc') != set('abd')
 
     def test_libpython_equality(self):
         for thetype in [frozenset, set]:
@@ -178,6 +421,9 @@
         s1 = set('abc')
         s1.update('d', 'ef', frozenset('g'))
         assert s1 == set('abcdefg')
+        s1 = set()
+        s1.update(set('abcd'))
+        assert s1 == set('abcd')
 
     def test_recursive_repr(self):
         class A(object):
@@ -330,6 +576,7 @@
         assert not set([1,2,5]).isdisjoint(frozenset([4,5,6]))
         assert not set([1,2,5]).isdisjoint([4,5,6])
         assert not set([1,2,5]).isdisjoint((4,5,6))
+        assert set([1,2,3]).isdisjoint(set([3.5,4.0]))
 
     def test_intersection(self):
         assert set([1,2,3]).intersection(set([2,3,4])) == set([2,3])
@@ -347,6 +594,35 @@
         assert s.intersection() == s
         assert s.intersection() is not s
 
+    def test_intersection_swap(self):
+        s1 = s3 = set([1,2,3,4,5])
+        s2 = set([2,3,6,7])
+        s1 &= s2
+        assert s1 == set([2,3])
+        assert s3 == set([2,3])
+
+    def test_intersection_generator(self):
+        def foo():
+            for i in range(5):
+                yield i
+
+        s1 = s2 = set([1,2,3,4,5,6])
+        assert s1.intersection(foo()) == set([1,2,3,4])
+        s1.intersection_update(foo())
+        assert s1 == set([1,2,3,4])
+        assert s2 == set([1,2,3,4])
+
+    def test_intersection_string(self):
+        s = set([1,2,3])
+        o = 'abc'
+        assert s.intersection(o) == set()
+
+    def test_intersection_float(self):
+        a = set([1,2,3])
+        b = set([3.0,4.0,5.0])
+        c = a.intersection(b)
+        assert c == set([3.0])
+
     def test_difference(self):
         assert set([1,2,3]).difference(set([2,3,4])) == set([1])
         assert set([1,2,3]).difference(frozenset([2,3,4])) == set([1])
@@ -361,6 +637,9 @@
         s = set([1,2,3])
         assert s.difference() == s
         assert s.difference() is not s
+        assert set([1,2,3]).difference(set([2,3,4,'5'])) == set([1])
+        assert set([1,2,3,'5']).difference(set([2,3,4])) == set([1,'5'])
+        assert set().difference(set([1,2,3])) == set()
 
     def test_intersection_update(self):
         s = set([1,2,3,4,7])
@@ -381,3 +660,250 @@
         assert s == set([2,3])
         s.difference_update(s)
         assert s == set([])
+
+    def test_empty_empty(self):
+        assert set() == set([])
+
+    def test_empty_difference(self):
+        e = set()
+        x = set([1,2,3])
+        assert e.difference(x) == set()
+        assert x.difference(e) == x
+
+        e.difference_update(x)
+        assert e == set()
+        x.difference_update(e)
+        assert x == set([1,2,3])
+
+        assert e.symmetric_difference(x) == x
+        assert x.symmetric_difference(e) == x
+
+        e.symmetric_difference_update(e)
+        assert e == e
+        e.symmetric_difference_update(x)
+        assert e == x
+
+        x.symmetric_difference_update(set())
+        assert x == set([1,2,3])
+
+    def test_fastpath_with_strategies(self):
+        a = set([1,2,3])
+        b = set(["a","b","c"])
+        assert a.difference(b) == a
+        assert b.difference(a) == b
+
+        a = set([1,2,3])
+        b = set(["a","b","c"])
+        assert a.intersection(b) == set()
+        assert b.intersection(a) == set()
+
+        a = set([1,2,3])
+        b = set(["a","b","c"])
+        assert not a.issubset(b)
+        assert not b.issubset(a)
+
+        a = set([1,2,3])
+        b = set(["a","b","c"])
+        assert a.isdisjoint(b)
+        assert b.isdisjoint(a)
+
+    def test_empty_intersect(self):
+        e = set()
+        x = set([1,2,3])
+        assert e.intersection(x) == e
+        assert x.intersection(e) == e
+        assert e & x == e
+        assert x & e == e
+
+        e.intersection_update(x)
+        assert e == set()
+        e &= x
+        assert e == set()
+        x.intersection_update(e)
+        assert x == set()
+
+    def test_empty_issuper(self):
+        e = set()
+        x = set([1,2,3])
+        assert e.issuperset(e) == True
+        assert e.issuperset(x) == False
+        assert x.issuperset(e) == True
+
+        assert e.issuperset(set())
+        assert e.issuperset([])
+
+    def test_empty_issubset(self):
+        e = set()
+        x = set([1,2,3])
+        assert e.issubset(e) == True
+        assert e.issubset(x) == True
+        assert x.issubset(e) == False
+        assert e.issubset([])
+
+    def test_empty_isdisjoint(self):
+        e = set()
+        x = set([1,2,3])
+        assert e.isdisjoint(e) == True
+        assert e.isdisjoint(x) == True
+        assert x.isdisjoint(e) == True
+
+    def test_empty_unhashable(self):
+        s = set()
+        raises(TypeError, s.difference, [[]])
+        raises(TypeError, s.difference_update, [[]])
+        raises(TypeError, s.intersection, [[]])
+        raises(TypeError, s.intersection_update, [[]])
+        raises(TypeError, s.symmetric_difference, [[]])
+        raises(TypeError, s.symmetric_difference_update, [[]])
+        raises(TypeError, s.update, [[]])
+
+    def test_super_with_generator(self):
+        def foo():
+            for i in [1,2,3]:
+                yield i
+        set([1,2,3,4,5]).issuperset(foo())
+
+    def test_isdisjoint_with_generator(self):
+        def foo():
+            for i in [1,2,3]:
+                yield i
+        set([1,2,3,4,5]).isdisjoint(foo())
+
+    def test_fakeint_and_equals(self):
+        s1 = set([1,2,3,4])
+        s2 = set([1,2,self.FakeInt(3), 4])
+        assert s1 == s2
+
+    def test_fakeint_and_discard(self):
+        # test with object strategy
+        s = set([1, 2, 'three', 'four'])
+        s.discard(self.FakeInt(2))
+        assert s == set([1, 'three', 'four'])
+
+        s.remove(self.FakeInt(1))
+        assert s == set(['three', 'four'])
+        raises(KeyError, s.remove, self.FakeInt(16))
+
+        # test with int strategy
+        s = set([1,2,3,4])
+        s.discard(self.FakeInt(4))
+        assert s == set([1,2,3])
+        s.remove(self.FakeInt(3))
+        assert s == set([1,2])
+        raises(KeyError, s.remove, self.FakeInt(16))
+
+    def test_fakeobject_and_has_key(self):
+        s = set([1,2,3,4,5])
+        assert 5 in s
+        assert self.FakeInt(5) in s
+
+    def test_fakeobject_and_pop(self):
+        s = set([1,2,3,self.FakeInt(4),5])
+        assert s.pop()
+        assert s.pop()
+        assert s.pop()
+        assert s.pop()
+        assert s.pop()
+        assert s == set([])
+
+    def test_fakeobject_and_difference(self):
+        s = set([1,2,'3',4])
+        s.difference_update([self.FakeInt(1), self.FakeInt(2)])
+        assert s == set(['3',4])
+
+        s = set([1,2,3,4])
+        s.difference_update([self.FakeInt(1), self.FakeInt(2)])
+        assert s == set([3,4])
+
+    def test_frozenset_behavior(self):
+        s = set([1,2,3,frozenset([4])])
+        raises(TypeError, s.difference_update, [1,2,3,set([4])])
+
+        s = set([1,2,3,frozenset([4])])
+        s.discard(set([4]))
+        assert s == set([1,2,3])
+
+    def test_discard_unhashable(self):
+        s = set([1,2,3,4])
+        raises(TypeError, s.discard, [1])
+
+    def test_discard_evil_compare(self):
+        class Evil(object):
+            def __init__(self, value):
+                self.value = value
+            def __hash__(self):
+                return hash(self.value)
+            def __eq__(self, other):
+                if isinstance(other, frozenset):
+                    raise TypeError
+                if other == self.value:
+                    return True
+                return False
+        s = set([1,2, Evil(frozenset([1]))])
+        raises(TypeError, s.discard, set([1]))
+
+    def test_create_set_from_set(self):
+        # no sharing
+        x = set([1,2,3])
+        y = set(x)
+        a = x.pop()
+        assert y == set([1,2,3])
+        assert len(x) == 2
+        assert x.union(set([a])) == y
+
+    def test_never_change_frozenset(self):
+        a = frozenset([1,2])
+        b = a.copy()
+        assert a is b
+
+        a = frozenset([1,2])
+        b = a.union(set([3,4]))
+        assert b == set([1,2,3,4])
+        assert a == set([1,2])
+
+        a = frozenset()
+        b = a.union(set([3,4]))
+        assert b == set([3,4])
+        assert a == set()
+
+        a = frozenset([1,2])#multiple
+        b = a.union(set([3,4]),[5,6])
+        assert b == set([1,2,3,4,5,6])
+        assert a == set([1,2])
+
+        a = frozenset([1,2,3])
+        b = a.difference(set([3,4,5]))
+        assert b == set([1,2])
+        assert a == set([1,2,3])
+
+        a = frozenset([1,2,3])#multiple
+        b = a.difference(set([3]), [2])
+        assert b == set([1])
+        assert a == set([1,2,3])
+
+        a = frozenset([1,2,3])
+        b = a.symmetric_difference(set([3,4,5]))
+        assert b == set([1,2,4,5])
+        assert a == set([1,2,3])
+
+        a = frozenset([1,2,3])
+        b = a.intersection(set([3,4,5]))
+        assert b == set([3])
+        assert a == set([1,2,3])
+
+        a = frozenset([1,2,3])#multiple
+        b = a.intersection(set([2,3,4]), [2])
+        assert b == set([2])
+        assert a == set([1,2,3])
+
+        raises(AttributeError, "frozenset().update()")
+        raises(AttributeError, "frozenset().difference_update()")
+        raises(AttributeError, "frozenset().symmetric_difference_update()")
+        raises(AttributeError, "frozenset().intersection_update()")
+
+    def test_intersection_obj(self):
+        class Obj:
+            def __getitem__(self, i):
+                return [5, 3, 4][i]
+        s = set([10,3,2]).intersection(Obj())
+        assert list(s) == [3]
diff --git a/pypy/objspace/std/test/test_setstrategies.py b/pypy/objspace/std/test/test_setstrategies.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_setstrategies.py
@@ -0,0 +1,107 @@
+from pypy.objspace.std.setobject import W_SetObject
+from pypy.objspace.std.setobject import IntegerSetStrategy, ObjectSetStrategy, EmptySetStrategy
+from pypy.objspace.std.listobject import W_ListObject
+
+class TestW_SetStrategies:
+
+    def wrapped(self, l):
+        return W_ListObject(self.space, [self.space.wrap(x) for x in l])
+
+    def test_from_list(self):
+        s = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        assert s.strategy is self.space.fromcache(IntegerSetStrategy)
+
+        s = W_SetObject(self.space, self.wrapped([1,"two",3,"four",5]))
+        assert s.strategy is self.space.fromcache(ObjectSetStrategy)
+
+        s = W_SetObject(self.space)
+        assert s.strategy is self.space.fromcache(EmptySetStrategy)
+
+        s = W_SetObject(self.space, self.wrapped([]))
+        assert s.strategy is self.space.fromcache(EmptySetStrategy)
+
+    def test_switch_to_object(self):
+        s = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        s.add(self.space.wrap("six"))
+        assert s.strategy is self.space.fromcache(ObjectSetStrategy)
+
+        s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        s2 = W_SetObject(self.space, self.wrapped(["six", "seven"]))
+        s1.update(s2)
+        assert s1.strategy is self.space.fromcache(ObjectSetStrategy)
+
+    def test_symmetric_difference(self):
+        s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        s2 = W_SetObject(self.space, self.wrapped(["six", "seven"]))
+        s1.symmetric_difference_update(s2)
+        assert s1.strategy is self.space.fromcache(ObjectSetStrategy)
+
+    def test_intersection(self):
+        s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        s2 = W_SetObject(self.space, self.wrapped([4,5, "six", "seven"]))
+        s3 = s1.intersect(s2)
+        skip("for now intersection with ObjectStrategy always results in another ObjectStrategy")
+        assert s3.strategy is self.space.fromcache(IntegerSetStrategy)
+
+    def test_clear(self):
+        s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        s1.clear()
+        assert s1.strategy is self.space.fromcache(EmptySetStrategy)
+
+    def test_remove(self):
+        from pypy.objspace.std.setobject import set_remove__Set_ANY
+        s1 = W_SetObject(self.space, self.wrapped([1]))
+        set_remove__Set_ANY(self.space, s1, self.space.wrap(1))
+        assert s1.strategy is self.space.fromcache(EmptySetStrategy)
+
+    def test_union(self):
+        from pypy.objspace.std.setobject import set_union__Set
+        s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        s2 = W_SetObject(self.space, self.wrapped([4,5,6,7]))
+        s3 = W_SetObject(self.space, self.wrapped([4,'5','6',7]))
+        s4 = set_union__Set(self.space, s1, [s2])
+        s5 = set_union__Set(self.space, s1, [s3])
+        assert s4.strategy is self.space.fromcache(IntegerSetStrategy)
+        assert s5.strategy is self.space.fromcache(ObjectSetStrategy)
+
+    def test_discard(self):
+        class FakeInt(object):
+            def __init__(self, value):
+                self.value = value
+            def __hash__(self):
+                return hash(self.value)
+            def __eq__(self, other):
+                if other == self.value:
+                    return True
+                return False
+
+        from pypy.objspace.std.setobject import set_discard__Set_ANY
+
+        s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        set_discard__Set_ANY(self.space, s1, self.space.wrap("five"))
+        skip("currently not supported")
+        assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
+
+        set_discard__Set_ANY(self.space, s1, self.space.wrap(FakeInt(5)))
+        assert s1.strategy is self.space.fromcache(ObjectSetStrategy)
+
+    def test_has_key(self):
+        class FakeInt(object):
+            def __init__(self, value):
+                self.value = value
+            def __hash__(self):
+                return hash(self.value)
+            def __eq__(self, other):
+                if other == self.value:
+                    return True
+                return False
+
+        from pypy.objspace.std.setobject import set_discard__Set_ANY
+
+        s1 = W_SetObject(self.space, self.wrapped([1,2,3,4,5]))
+        assert not s1.has_key(self.space.wrap("five"))
+        skip("currently not supported")
+        assert s1.strategy is self.space.fromcache(IntegerSetStrategy)
+
+        assert s1.has_key(self.space.wrap(FakeInt(2)))
+        assert s1.strategy is self.space.fromcache(ObjectSetStrategy)
diff --git a/pypy/objspace/std/test/test_stringobject.py b/pypy/objspace/std/test/test_stringobject.py
--- a/pypy/objspace/std/test/test_stringobject.py
+++ b/pypy/objspace/std/test/test_stringobject.py
@@ -85,6 +85,10 @@
         w_slice = space.newslice(w(1), w_None, w(2))
         assert self.space.eq_w(space.getitem(w_str, w_slice), w('el'))
 
+    def test_listview_str(self):
+        w_str = self.space.wrap('abcd')
+        assert self.space.listview_str(w_str) == list("abcd")
+
 class AppTestStringObject:
 
     def test_format_wrongchar(self):
diff --git a/pypy/objspace/std/test/test_userobject.py b/pypy/objspace/std/test/test_userobject.py
--- a/pypy/objspace/std/test/test_userobject.py
+++ b/pypy/objspace/std/test/test_userobject.py
@@ -244,8 +244,12 @@
             skip("disabled")
         if self.runappdirect:
             total = 500000
+            def rand():
+                import random
+                return random.randrange(0, 5)
         else:
             total = 50
+            rand = self.rand
         #
         class A(object):
             hash = None
@@ -256,7 +260,7 @@
             a = A()
             a.next = tail.next
             tail.next = a
-            for j in range(self.rand()):
+            for j in range(rand()):
                 any = any.next
             if any.hash is None:
                 any.hash = hash(any)
diff --git a/pypy/rlib/_rffi_stacklet.py b/pypy/rlib/_rffi_stacklet.py
--- a/pypy/rlib/_rffi_stacklet.py
+++ b/pypy/rlib/_rffi_stacklet.py
@@ -14,7 +14,7 @@
     includes = ['src/stacklet/stacklet.h'],
     separate_module_sources = ['#include "src/stacklet/stacklet.c"\n'],
 )
-if sys.platform == 'win32':
+if 'masm' in dir(eci.platform): # Microsoft compiler
     if is_emulated_long:
         asmsrc = 'switch_x64_msvc.asm'
     else:
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -58,12 +58,12 @@
     header_lines = [
         '#include <WinSock2.h>',
         '#include <WS2tcpip.h>',
+        '#include <Mstcpip.h>',
         # winsock2 defines AF_UNIX, but not sockaddr_un
         '#undef AF_UNIX',
         ]
     if _MSVC:
         header_lines.extend([
-            '#include <Mstcpip.h>',
             # these types do not exist on microsoft compilers
             'typedef int ssize_t;',
             'typedef unsigned __int16 uint16_t;',
@@ -71,6 +71,7 @@
             ])
     else: # MINGW
         includes = ('stdint.h',)
+        """
         header_lines.extend([
             '''\
             #ifndef _WIN32_WINNT
@@ -88,6 +89,7 @@
                 u_long  keepaliveinterval;
             };'''
             ])
+        """
     HEADER = '\n'.join(header_lines)
     COND_HEADER = ''
 constants = {}
diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py
--- a/pypy/rlib/clibffi.py
+++ b/pypy/rlib/clibffi.py
@@ -114,9 +114,10 @@
         )
 
     eci = rffi_platform.configure_external_library(
-        'libffi', eci,
+        'libffi-5', eci,
         [dict(prefix='libffi-',
               include_dir='include', library_dir='.libs'),
+         dict(prefix=r'c:\mingw64', include_dir='include', library_dir='lib'),
          ])
 else:
     libffidir = py.path.local(pypydir).join('translator', 'c', 'src', 'libffi_msvc')
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -6,6 +6,7 @@
 in which it does not work.
 """
 
+from __future__ import with_statement
 from pypy.annotation import model as annmodel
 from pypy.rlib.rarithmetic import r_int64
 from pypy.rpython.lltypesystem import lltype, rffi
diff --git a/pypy/rlib/parsing/pypackrat.py b/pypy/rlib/parsing/pypackrat.py
--- a/pypy/rlib/parsing/pypackrat.py
+++ b/pypy/rlib/parsing/pypackrat.py
@@ -1,6 +1,8 @@
 
 from pypy.rlib.parsing.tree import Nonterminal, Symbol
-from makepackrat import PackratParser, BacktrackException, Status
+from pypy.rlib.parsing.makepackrat import PackratParser, BacktrackException, Status
+
+
 class Parser(object):
     def NAME(self):
         return self._NAME().result
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -713,9 +713,9 @@
     free = c_munmap_safe
 
 elif _MS_WINDOWS:
-    def mmap(fileno, length, flags=0, tagname="", access=_ACCESS_DEFAULT, offset=0):
+    def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
         # XXX flags is or-ed into access by now.
-        
+        flags = 0
         # check size boundaries
         _check_map_size(length)
         map_size = length
diff --git a/pypy/rlib/rwin32.py b/pypy/rlib/rwin32.py
--- a/pypy/rlib/rwin32.py
+++ b/pypy/rlib/rwin32.py
@@ -141,6 +141,10 @@
         cfile = udir.join('dosmaperr.c')
         cfile.write(r'''
                 #include <errno.h>
+                #include  <stdio.h>
+                #ifdef __GNUC__
+                #define _dosmaperr mingw_dosmaperr
+                #endif
                 int main()
                 {
                     int i;
diff --git a/pypy/rlib/test/autopath.py b/pypy/rlib/test/autopath.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/autopath.py
@@ -0,0 +1,131 @@
+"""
+self cloning, automatic path configuration 
+
+copy this into any subdirectory of pypy from which scripts need 
+to be run, typically all of the test subdirs. 
+The idea is that any such script simply issues
+
+    import autopath
+
+and this will make sure that the parent directory containing "pypy"
+is in sys.path. 
+
+If you modify the master "autopath.py" version (in pypy/tool/autopath.py) 
+you can directly run it which will copy itself on all autopath.py files
+it finds under the pypy root directory. 
+
+This module always provides these attributes:
+
+    pypydir    pypy root directory path 
+    this_dir   directory where this autopath.py resides 
+
+"""
+
+def __dirinfo(part):
+    """ return (partdir, this_dir) and insert parent of partdir
+    into sys.path.  If the parent directories don't have the part
+    an EnvironmentError is raised."""
+
+    import sys, os
+    try:
+        head = this_dir = os.path.realpath(os.path.dirname(__file__))
+    except NameError:
+        head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0]))
+
+    error = None
+    while head:
+        partdir = head
+        head, tail = os.path.split(head)
+        if tail == part:
+            checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py')
+            if not os.path.exists(checkfile):
+                error = "Cannot find %r" % (os.path.normpath(checkfile),)
+            break
+    else:
+        error = "Cannot find the parent directory %r of the path %r" % (
+            partdir, this_dir)
+    if not error:
+        # check for bogus end-of-line style (e.g. files checked out on
+        # Windows and moved to Unix)
+        f = open(__file__.replace('.pyc', '.py'), 'r')
+        data = f.read()
+        f.close()
+        if data.endswith('\r\n') or data.endswith('\r'):
+            error = ("Bad end-of-line style in the .py files. Typically "
+                     "caused by a zip file or a checkout done on Windows and "
+                     "moved to Unix or vice-versa.")
+    if error:
+        raise EnvironmentError("Invalid source tree - bogus checkout! " +
+                               error)
+    
+    pypy_root = os.path.join(head, '')
+    try:
+        sys.path.remove(head)
+    except ValueError:
+        pass
+    sys.path.insert(0, head)
+
+    munged = {}
+    for name, mod in sys.modules.items():
+        if '.' in name:
+            continue
+        fn = getattr(mod, '__file__', None)
+        if not isinstance(fn, str):
+            continue
+        newname = os.path.splitext(os.path.basename(fn))[0]
+        if not newname.startswith(part + '.'):
+            continue
+        path = os.path.join(os.path.dirname(os.path.realpath(fn)), '')
+        if path.startswith(pypy_root) and newname != part:
+            modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep)
+            if newname != '__init__':
+                modpaths.append(newname)
+            modpath = '.'.join(modpaths)
+            if modpath not in sys.modules:
+                munged[modpath] = mod
+
+    for name, mod in munged.iteritems():
+        if name not in sys.modules:
+            sys.modules[name] = mod
+        if '.' in name:
+            prename = name[:name.rfind('.')]
+            postname = name[len(prename)+1:]
+            if prename not in sys.modules:
+                __import__(prename)
+                if not hasattr(sys.modules[prename], postname):
+                    setattr(sys.modules[prename], postname, mod)
+
+    return partdir, this_dir
+
+def __clone():
+    """ clone master version of autopath.py into all subdirs """
+    from os.path import join, walk
+    if not this_dir.endswith(join('pypy','tool')):
+        raise EnvironmentError("can only clone master version "
+                               "'%s'" % join(pypydir, 'tool',_myname))
+
+
+    def sync_walker(arg, dirname, fnames):
+        if _myname in fnames:
+            fn = join(dirname, _myname)
+            f = open(fn, 'rwb+')
+            try:
+                if f.read() == arg:
+                    print "checkok", fn
+                else:
+                    print "syncing", fn
+                    f = open(fn, 'w')
+                    f.write(arg)
+            finally:
+                f.close()
+    s = open(join(pypydir, 'tool', _myname), 'rb').read()
+    walk(pypydir, sync_walker, s)
+
+_myname = 'autopath.py'
+
+# set guaranteed attributes
+
+pypydir, this_dir = __dirinfo('pypy')
+
+if __name__ == '__main__':
+    __clone()
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -427,6 +427,10 @@
 ##    assert type(x) is int
 ##    return llmemory.cast_int_to_adr(x)
 
+def op_convert_float_bytes_to_longlong(a):
+    from pypy.rlib.longlong2float import float2longlong
+    return float2longlong(a)
+
 
 def op_unichar_eq(x, y):
     assert isinstance(x, unicode) and len(x) == 1
diff --git a/pypy/rpython/rstr.py b/pypy/rpython/rstr.py
--- a/pypy/rpython/rstr.py
+++ b/pypy/rpython/rstr.py
@@ -165,6 +165,7 @@
         v_char = hop.inputarg(rstr.char_repr, arg=1)
         v_left = hop.inputconst(Bool, left)
         v_right = hop.inputconst(Bool, right)
+        hop.exception_is_here()
         return hop.gendirectcall(self.ll.ll_strip, v_str, v_char, v_left, v_right)
 
     def rtype_method_lstrip(self, hop):
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -637,13 +637,16 @@
     def _make_split_test(self, split_fn):
         const = self.const
         def fn(i):
-            s = [const(''), const('0.1.2.4.8'), const('.1.2'), const('1.2.'), const('.1.2.4.')][i]
-            l = getattr(s, split_fn)(const('.'))
-            sum = 0
-            for num in l:
-                if len(num):
-                    sum += ord(num[0]) - ord(const('0')[0])
-            return sum + len(l) * 100
+            try:
+                s = [const(''), const('0.1.2.4.8'), const('.1.2'), const('1.2.'), const('.1.2.4.')][i]
+                l = getattr(s, split_fn)(const('.'))
+                sum = 0
+                for num in l:
+                    if len(num):
+                        sum += ord(num[0]) - ord(const('0')[0])
+                return sum + len(l) * 100
+            except MemoryError:
+                return 42
         return fn
 
     def test_split(self):
diff --git a/pypy/rpython/tool/rffi_platform.py b/pypy/rpython/tool/rffi_platform.py
--- a/pypy/rpython/tool/rffi_platform.py
+++ b/pypy/rpython/tool/rffi_platform.py
@@ -660,8 +660,8 @@
     if isinstance(fieldtype, lltype.FixedSizeArray):
         size, _ = expected_size_and_sign
         return lltype.FixedSizeArray(fieldtype.OF, size/_sizeof(fieldtype.OF))
-    raise TypeError("conflicting field type %r for %r" % (fieldtype,
-                                                          fieldname))
+    raise TypeError("conflict between translating python and compiler field"
+                    " type %r for %r" % (fieldtype, fieldname))
 
 def expose_value_as_rpython(value):
     if intmask(value) == value:
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -58,9 +58,13 @@
     binaries = [(pypy_c, rename_pypy_c)]
     #
     if sys.platform == 'win32':
+        #Don't include a mscvrXX.dll, users should get their own.
+        #Instructions are provided on the website.
+
         # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        
         for extra in ['libpypy-c.dll',
-                      'libexpat.dll', 'sqlite3.dll', 'msvcr100.dll',
+                      'libexpat.dll', 'sqlite3.dll', 
                       'libeay32.dll', 'ssleay32.dll']:
             p = pypy_c.dirpath().join(extra)
             if not p.check():
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -6,6 +6,7 @@
 from pypy.annotation.listdef import s_list_of_strings
 from pypy import conftest
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.translator.platform import platform as compiler
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib.entrypoint import entrypoint, secondary_entrypoints
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -17,6 +18,8 @@
 
     @classmethod
     def make_config(cls):
+        if compiler.name == "msvc":
+            py.test.skip("all asmgcroot tests disabled for MSVC")
         from pypy.config.pypyoption import get_pypy_config
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -484,7 +484,9 @@
         'shl', 'shr', 'sal', 'sar', 'rol', 'ror', 'mul', 'imul', 'div', 'idiv',
         'bswap', 'bt', 'rdtsc',
         'punpck', 'pshufd', 'pcmp', 'pand', 'psllw', 'pslld', 'psllq',
-        'paddq', 'pinsr', 'pmul', 'psrl', 'vmul',
+        'paddq', 'pinsr', 'pmul', 'psrl',
+        # all vectors don't produce pointers
+        'v',
         # sign-extending moves should not produce GC pointers
         'cbtw', 'cwtl', 'cwtd', 'cltd', 'cltq', 'cqto',
         # zero-extending moves should not produce GC pointers
diff --git a/pypy/translator/c/src/libffi_msvc/win64.asm b/pypy/translator/c/src/libffi_msvc/win64.asm
new file mode 100644
--- /dev/null
+++ b/pypy/translator/c/src/libffi_msvc/win64.asm
@@ -0,0 +1,156 @@
+PUBLIC	ffi_call_AMD64
+
+EXTRN	__chkstk:NEAR
+EXTRN	ffi_closure_SYSV:NEAR
+
+_TEXT	SEGMENT
+
+;;; ffi_closure_OUTER will be called with these registers set:
+;;;    rax points to 'closure'
+;;;    r11 contains a bit mask that specifies which of the
+;;;    first four parameters are float or double
+;;;
+;;; It must move the parameters passed in registers to their stack location,
+;;; call ffi_closure_SYSV for the actual work, then return the result.
+;;; 
+ffi_closure_OUTER PROC FRAME
+	;; save actual arguments to their stack space.
+	test	r11, 1
+	jne	first_is_float	
+	mov	QWORD PTR [rsp+8], rcx
+	jmp	second
+first_is_float:
+	movlpd	QWORD PTR [rsp+8], xmm0
+
+second:
+	test	r11, 2
+	jne	second_is_float	
+	mov	QWORD PTR [rsp+16], rdx
+	jmp	third
+second_is_float:
+	movlpd	QWORD PTR [rsp+16], xmm1
+
+third:
+	test	r11, 4
+	jne	third_is_float	
+	mov	QWORD PTR [rsp+24], r8
+	jmp	forth
+third_is_float:
+	movlpd	QWORD PTR [rsp+24], xmm2
+
+forth:
+	test	r11, 8
+	jne	forth_is_float	
+	mov	QWORD PTR [rsp+32], r9
+	jmp	done
+forth_is_float:
+	movlpd	QWORD PTR [rsp+32], xmm3
+
+done:
+.ALLOCSTACK 40
+	sub	rsp, 40
+.ENDPROLOG
+	mov	rcx, rax	; context is first parameter
+	mov	rdx, rsp	; stack is second parameter
+	add	rdx, 40		; correct our own area
+	mov	rax, ffi_closure_SYSV
+	call	rax		; call the real closure function
+	;; Here, code is missing that handles float return values
+	add	rsp, 40
+	movd	xmm0, rax	; In case the closure returned a float.
+	ret	0
+ffi_closure_OUTER ENDP
+
+
+;;; ffi_call_AMD64
+
+stack$ = 0
+prepfunc$ = 32
+ecif$ = 40
+bytes$ = 48
+flags$ = 56
+rvalue$ = 64
+fn$ = 72
+
+ffi_call_AMD64 PROC FRAME
+
+	mov	QWORD PTR [rsp+32], r9
+	mov	QWORD PTR [rsp+24], r8
+	mov	QWORD PTR [rsp+16], rdx
+	mov	QWORD PTR [rsp+8], rcx
+.PUSHREG rbp
+	push	rbp
+.ALLOCSTACK 48
+	sub	rsp, 48					; 00000030H
+.SETFRAME rbp, 32
+	lea	rbp, QWORD PTR [rsp+32]
+.ENDPROLOG
+
+	mov	eax, DWORD PTR bytes$[rbp]
+	add	rax, 15
+	and	rax, -16
+	call	__chkstk
+	sub	rsp, rax
+	lea	rax, QWORD PTR [rsp+32]
+	mov	QWORD PTR stack$[rbp], rax
+
+	mov	rdx, QWORD PTR ecif$[rbp]
+	mov	rcx, QWORD PTR stack$[rbp]
+	call	QWORD PTR prepfunc$[rbp]
+
+	mov	rsp, QWORD PTR stack$[rbp]
+
+	movlpd	xmm3, QWORD PTR [rsp+24]
+	movd	r9, xmm3
+
+	movlpd	xmm2, QWORD PTR [rsp+16]
+	movd	r8, xmm2
+
+	movlpd	xmm1, QWORD PTR [rsp+8]
+	movd	rdx, xmm1
+
+	movlpd	xmm0, QWORD PTR [rsp]
+	movd	rcx, xmm0
+
+	call	QWORD PTR fn$[rbp]
+ret_int$:
+ 	cmp	DWORD PTR flags$[rbp], 1 ; FFI_TYPE_INT
+ 	jne	ret_float$
+
+	mov	rcx, QWORD PTR rvalue$[rbp]
+	mov	DWORD PTR [rcx], eax
+	jmp	SHORT ret_nothing$
+
+ret_float$:
+ 	cmp	DWORD PTR flags$[rbp], 2 ; FFI_TYPE_FLOAT
+ 	jne	SHORT ret_double$
+
+ 	mov	rax, QWORD PTR rvalue$[rbp]
+ 	movlpd	QWORD PTR [rax], xmm0
+ 	jmp	SHORT ret_nothing$
+
+ret_double$:
+ 	cmp	DWORD PTR flags$[rbp], 3 ; FFI_TYPE_DOUBLE
+ 	jne	SHORT ret_int64$
+
+ 	mov	rax, QWORD PTR rvalue$[rbp]
+ 	movlpd	QWORD PTR [rax], xmm0
+ 	jmp	SHORT ret_nothing$
+
+ret_int64$:
+  	cmp	DWORD PTR flags$[rbp], 12 ; FFI_TYPE_SINT64
+  	jne	ret_nothing$
+
+ 	mov	rcx, QWORD PTR rvalue$[rbp]
+ 	mov	QWORD PTR [rcx], rax
+ 	jmp	SHORT ret_nothing$
+	
+ret_nothing$:
+	xor	eax, eax
+
+	lea	rsp, QWORD PTR [rbp+16]
+	pop	rbp
+	ret	0
+ffi_call_AMD64 ENDP
+_TEXT	ENDS
+END
diff --git a/pypy/translator/platform/__init__.py b/pypy/translator/platform/__init__.py
--- a/pypy/translator/platform/__init__.py
+++ b/pypy/translator/platform/__init__.py
@@ -304,6 +304,8 @@
     global platform
     log.msg("Setting platform to %r cc=%s" % (new_platform,cc))
     platform = pick_platform(new_platform, cc)
+    if not platform:
+        raise ValueError("pick_platform failed")
 
     if new_platform == 'host':
         global host
diff --git a/pypy/translator/platform/windows.py b/pypy/translator/platform/windows.py
--- a/pypy/translator/platform/windows.py
+++ b/pypy/translator/platform/windows.py
@@ -7,15 +7,27 @@
 from pypy.translator.platform import log, _run_subprocess
 from pypy.translator.platform import Platform, posix
 
+def _get_compiler_type(cc, x64_flag):
+    import subprocess
+    if not cc:
+        cc = os.environ.get('CC','')
+    if not cc:
+        return MsvcPlatform(cc=cc, x64=x64_flag)
+    elif cc.startswith('mingw'):
+        return MingwPlatform(cc)
+    try:
+        subprocess.check_output([cc, '--version'])
+    except:
+        raise ValueError,"Could not find compiler specified by cc option" + \
+                " '%s', it must be a valid exe file on your path"%cc
+    return MingwPlatform(cc)
+
 def Windows(cc=None):
-    if cc == 'mingw32':
-        return MingwPlatform(cc)
-    else:
-        return MsvcPlatform(cc, False)
+    return _get_compiler_type(cc, False)
+
+def Windows_x64(cc=None):
+    return _get_compiler_type(cc, True)
     
-def Windows_x64(cc=None):
-    return MsvcPlatform(cc, True)
-
 def _get_msvc_env(vsver, x64flag):
     try:
         toolsdir = os.environ['VS%sCOMNTOOLS' % vsver]
@@ -31,14 +43,16 @@
         vcvars = os.path.join(toolsdir, 'vsvars32.bat')
 
     import subprocess
-    popen = subprocess.Popen('"%s" & set' % (vcvars,),
+    try:
+        popen = subprocess.Popen('"%s" & set' % (vcvars,),
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)
 
-    stdout, stderr = popen.communicate()
-    if popen.wait() != 0:
-        return
-
+        stdout, stderr = popen.communicate()
+        if popen.wait() != 0:
+            return None
+    except:
+        return None
     env = {}
 
     stdout = stdout.replace("\r\n", "\n")
@@ -395,7 +409,9 @@
     so_ext = 'dll'
 
     def __init__(self, cc=None):
-        Platform.__init__(self, 'gcc')
+        if not cc:
+            cc = 'gcc'
+        Platform.__init__(self, cc)
 
     def _args_for_shared(self, args):
         return ['-shared'] + args
diff --git a/pypy/translator/test/test_unsimplify.py b/pypy/translator/test/test_unsimplify.py
--- a/pypy/translator/test/test_unsimplify.py
+++ b/pypy/translator/test/test_unsimplify.py
@@ -78,7 +78,7 @@
             return x * 6
         def hello_world():
             if we_are_translated():
-                fd = os.open(tmpfile, os.O_WRONLY | os.O_CREAT, 0)
+                fd = os.open(tmpfile, os.O_WRONLY | os.O_CREAT, 0644)
                 os.close(fd)
         graph, t = translate(f, [int], type_system)
         call_initial_function(t, hello_world)
@@ -97,7 +97,7 @@
             return x * 6
         def goodbye_world():
             if we_are_translated():
-                fd = os.open(tmpfile, os.O_WRONLY | os.O_CREAT, 0)
+                fd = os.open(tmpfile, os.O_WRONLY | os.O_CREAT, 0644)
                 os.close(fd)
         graph, t = translate(f, [int], type_system)
         call_final_function(t, goodbye_world)

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:04 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:04 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: start implementing the long long
	support
Message-ID: <20120405121904.009178208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54198:7dbbed165924
Date: 2012-03-27 12:23 +0000
http://bitbucket.org/pypy/pypy/changeset/7dbbed165924/

Log:	start implementing the long long support

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -925,6 +925,19 @@
             return False
         return True
 
+    def regalloc_emit_llong(self, op, arglocs, fcond, regalloc):
+        effectinfo = op.getdescr().get_extra_info()
+        oopspecindex = effectinfo.oopspecindex
+        asm_llong_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
+	return fcond 
+
+    def regalloc_emit_math(self, op, arglocs, fcond, regalloc):
+        effectinfo = op.getdescr().get_extra_info()
+        oopspecindex = effectinfo.oopspecindex
+        asm_math_operations[oopspecindex](self, op, arglocs, resloc)
+        return fcond
+
+
     def _insert_checks(self, mc=None):
         if not we_are_translated() and self._debug:
             if mc is None:
@@ -1265,6 +1278,7 @@
 
 asm_operations = [notimplemented_op] * (rop._LAST + 1)
 asm_operations_with_guard = [notimplemented_op_with_guard] * (rop._LAST + 1)
+asm_llong_operations = {}
 asm_math_operations = {}
 
 for name, value in ResOpAssembler.__dict__.iteritems():
diff --git a/pypy/jit/backend/arm/instruction_builder.py b/pypy/jit/backend/arm/instruction_builder.py
--- a/pypy/jit/backend/arm/instruction_builder.py
+++ b/pypy/jit/backend/arm/instruction_builder.py
@@ -350,6 +350,36 @@
             self.write32(instr)
     return f
 
+def define_simd_instructions_3regs_func(name, table):
+    n = 0x79 << 25
+    if 'A' in table:
+        n |= (table['A'] & 0xF) << 8
+    if 'B' in table:
+        n |= (table['B'] & 0x1) << 4
+    if 'U' in table:
+        n |= (table['U'] & 0x1) << 24
+    if 'C' in table:
+        n |= (table['C'] & 0x3) << 20
+    if name == 'VADD_i64' or name == 'VSUB_i64':
+        size = 0x3
+        n |= size << 20
+    def f(self, dd, dn, dm):
+	N = (dn >> 4) & 0x1
+	M = (dm >> 4) & 0x1
+	D = (dd >> 4) & 0x1
+	Q = 0 # we want doubleword regs
+        instr = (n
+                | D << 22
+                | (dn & 0xf) << 16
+                | (dd & 0xf) << 12
+                | N << 7
+                | Q << 6
+                | M << 5
+                | (dm & 0xf))
+	
+        self.write32(instr)
+    return f
+
 
 def imm_operation(rt, rn, imm):
     return ((rn & 0xFF) << 16
@@ -368,6 +398,7 @@
 
 def define_instruction(builder, key, val, target):
     f = builder(key, val)
+    f.__name__ = key
     setattr(target, key, f)
 
 
@@ -378,7 +409,6 @@
             continue
         try:
             func = globals()['define_%s_func' % name]
-            func.__name__ = name
         except KeyError:
             print 'No instr generator for %s instructions' % name
             continue
diff --git a/pypy/jit/backend/arm/instructions.py b/pypy/jit/backend/arm/instructions.py
--- a/pypy/jit/backend/arm/instructions.py
+++ b/pypy/jit/backend/arm/instructions.py
@@ -140,3 +140,11 @@
     'VSQRT' : {'opc1':0xB, 'opc2':0x1, 'opc3':0x3, 'base': False},
     #'VCVT' : {'opc1':0xB, 'opc2':0xE, 'opc3':0x1, 'base': False},
 }
+
+simd_instructions_3regs = {
+    'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0},
+    'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1},
+    'VAND_i64': {'A': 0x1, 'B': 1, 'U': 0, 'C': 0},
+    'VORR_i64': {'A': 0x1, 'B': 1, 'U': 0, 'C': 0x10},
+    'VEOR_i64': {'A': 0x1, 'B': 1, 'U': 1, 'C': 0x0},
+}
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -1282,4 +1282,9 @@
         self.mc.VCVT_int_to_float(res.value, temp.value)
         return fcond
 
+    emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
+    emit_op_llong_sub = gen_emit_float_op('llong_sub', 'VSUB_i64')
+    emit_op_llong_and = gen_emit_float_op('llong_and', 'VAND_i64')
+    emit_op_llong_or = gen_emit_float_op('llong_or', 'VORR_i64')
+    emit_op_llong_xor = gen_emit_float_op('llong_xor', 'VEOR_i64')
 
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -374,6 +374,12 @@
         # is also used on op args, which is a non-resizable list
         self.possibly_free_vars(list(inputargs))
 
+    def perform_llong(self, op, args, fcond):
+        return self.assembler.regalloc_emit_llong(op, args, fcond, self)
+    
+    def perform_math(self, op, args, fcond):
+	return self.assembler.regalloc_emit_math(op, args, self, fcond)
+
     def force_spill_var(self, var):
         if var.type == FLOAT:
             self.vfprm.force_spill_var(var)
@@ -541,6 +547,18 @@
         effectinfo = op.getdescr().get_extra_info()
         if effectinfo is not None:
             oopspecindex = effectinfo.oopspecindex
+            if oopspecindex in (EffectInfo.OS_LLONG_ADD,
+            		    EffectInfo.OS_LLONG_SUB,
+            		    EffectInfo.OS_LLONG_AND,
+            		    EffectInfo.OS_LLONG_OR,
+            		    EffectInfo.OS_LLONG_XOR):
+                args = self._prepare_llong_binop_xx(op, fcond)
+		self.perform_llong(op, args, fcond)
+                return
+            if oopspecindex == EffectInfo.OS_LLONG_TO_INT:
+                args = self._prepare_llong_to_int(op, fcond)
+		self.perform_llong(op, args, fcond)
+                return
             if oopspecindex == EffectInfo.OS_MATH_SQRT:
                 args = self.prepare_op_math_sqrt(op, fcond)
                 self.perform_math(op, args, fcond)
@@ -568,6 +586,15 @@
     def prepare_op_call_malloc_gc(self, op, fcond):
         return self._prepare_call(op)
 
+    def _prepare_llong_binop_xx(self, op, fcond):
+	# arg 0 is the address of the function
+        loc0 = self._ensure_value_is_boxed(op.getarg(1))
+	loc1 = self._ensure_value_is_boxed(op.getarg(2))
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
+        res = self.vfprm.force_allocate_reg(op.result)
+        return [loc0, loc1, res]
+
     def _prepare_guard(self, op, args=None):
         if args is None:
             args = []
diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py
--- a/pypy/jit/backend/arm/runner.py
+++ b/pypy/jit/backend/arm/runner.py
@@ -9,6 +9,7 @@
 class ArmCPU(AbstractLLCPU):
 
     supports_floats = True
+    supports_longlong = True
 
     def __init__(self, rtyper, stats, opts=None, translate_support_code=False,
                  gcdescr=None):

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:05 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:05 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: refactor cast_float_to_int and
 cast_int_to_float using the VMOV operation
Message-ID: <20120405121905.A4C898208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54199:cf3f78de7344
Date: 2012-03-27 17:02 +0000
http://bitbucket.org/pypy/pypy/changeset/cf3f78de7344/

Log:	refactor cast_float_to_int and cast_int_to_float using the VMOV
	operation

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -1268,18 +1268,20 @@
     emit_guard_float_ge = gen_emit_float_cmp_op_guard('float_ge', c.GE)
 
     def emit_op_cast_float_to_int(self, op, arglocs, regalloc, fcond):
-        arg, temp, res = arglocs
-        self.mc.VCVT_float_to_int(temp.value, arg.value)
-        self.mc.VPUSH([temp.value])
-        # res is lower register than r.ip
-        self.mc.POP([res.value, r.ip.value])
+        arg, res = arglocs
+        assert arg.is_vfp_reg()
+        assert res.is_reg()
+        self.mc.VCVT_float_to_int(r.vfp_ip.value, arg.value)
+	self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
         return fcond
 
     def emit_op_cast_int_to_float(self, op, arglocs, regalloc, fcond):
-        arg, temp, res = arglocs
-        self.mc.PUSH([arg.value, r.ip.value])
-        self.mc.VPOP([temp.value])
-        self.mc.VCVT_int_to_float(res.value, temp.value)
+        arg, res = arglocs
+        assert res.is_vfp_reg()
+        assert arg.is_reg()
+	self.mc.MOV_ri(r.ip.value, 0)
+        self.mc.VMOV_cr(res.value, arg.value, r.ip.value)
+        self.mc.VCVT_int_to_float(res.value, res.value)
         return fcond
 
     emit_op_llong_add = gen_emit_float_op('llong_add', 'VADD_i64')
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -1186,19 +1186,13 @@
 
     def prepare_op_cast_float_to_int(self, op, fcond):
         loc1 = self._ensure_value_is_boxed(op.getarg(0))
-        temp_loc = self.get_scratch_reg(FLOAT)
-        self.possibly_free_vars_for_op(op)
-        self.free_temp_vars()
         res = self.rm.force_allocate_reg(op.result)
-        return [loc1, temp_loc, res]
+        return [loc1, res]
 
     def prepare_op_cast_int_to_float(self, op, fcond):
         loc1 = self._ensure_value_is_boxed(op.getarg(0))
-        temp_loc = self.get_scratch_reg(FLOAT)
-        self.possibly_free_vars_for_op(op)
-        self.free_temp_vars()
         res = self.vfprm.force_allocate_reg(op.result)
-        return [loc1, temp_loc, res]
+        return [loc1, res]
 
     def prepare_force_spill(self, op, fcond):
         self.force_spill_var(op.getarg(0))

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:07 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:07 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: Add tests for SIMD instructions
	and fix VORR
Message-ID: <20120405121907.090658208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54200:563b0debb79e
Date: 2012-03-28 10:32 +0000
http://bitbucket.org/pypy/pypy/changeset/563b0debb79e/

Log:	Add tests for SIMD instructions and fix VORR

diff --git a/pypy/jit/backend/arm/instructions.py b/pypy/jit/backend/arm/instructions.py
--- a/pypy/jit/backend/arm/instructions.py
+++ b/pypy/jit/backend/arm/instructions.py
@@ -145,6 +145,6 @@
     'VADD_i64': {'A': 0x8, 'B': 0, 'U': 0},
     'VSUB_i64': {'A': 0x8, 'B': 0, 'U': 1},
     'VAND_i64': {'A': 0x1, 'B': 1, 'U': 0, 'C': 0},
-    'VORR_i64': {'A': 0x1, 'B': 1, 'U': 0, 'C': 0x10},
+    'VORR_i64': {'A': 0x1, 'B': 1, 'U': 0, 'C': 0x2},
     'VEOR_i64': {'A': 0x1, 'B': 1, 'U': 1, 'C': 0x0},
 }
diff --git a/pypy/jit/backend/arm/test/test_instr_codebuilder.py b/pypy/jit/backend/arm/test/test_instr_codebuilder.py
--- a/pypy/jit/backend/arm/test/test_instr_codebuilder.py
+++ b/pypy/jit/backend/arm/test/test_instr_codebuilder.py
@@ -284,6 +284,10 @@
             tests.append((asm, (r.r3.value, range(regs+1))))
     return tests
 
+def gen_test_simd_instructions_3regs_func(name, table):
+    op_name = name[:name.index('_')]
+    return  [('d1, d2, d3', (r.d1.value, r.d2.value, r.d3.value), {}, '.i64')]
+
 def build_tests():
     cls = TestInstrCodeBuilderForGeneratedInstr
     test_name = 'test_generated_%s'

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:08 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:08 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: add llong_to_int implementation
Message-ID: <20120405121908.541518208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54201:9f746b2b318e
Date: 2012-03-28 11:22 +0000
http://bitbucket.org/pypy/pypy/changeset/9f746b2b318e/

Log:	add llong_to_int implementation

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -1290,3 +1290,10 @@
     emit_op_llong_or = gen_emit_float_op('llong_or', 'VORR_i64')
     emit_op_llong_xor = gen_emit_float_op('llong_xor', 'VEOR_i64')
 
+    def emit_op_llong_to_int(self, op, arglocs, regalloc, fcond):
+        loc = arglocs[0]
+        res = arglocs[1]
+        assert loc.is_vfp_reg()
+        assert res.is_reg()
+        self.mc.VMOV_rc(res.value, r.ip.value, loc.value)
+        return fcond
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -595,6 +595,12 @@
         res = self.vfprm.force_allocate_reg(op.result)
         return [loc0, loc1, res]
 
+    def _prepare_llong_to_int(self, op, fcond):
+        loc0 = self._ensure_value_is_boxed(op.getarg(1))
+        res = self.force_allocate_reg(op.result)
+        return [loc0, res]
+
+
     def _prepare_guard(self, op, args=None):
         if args is None:
             args = []

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:09 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:09 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: implement
	convert_float_bytes_to_longlong
Message-ID: <20120405121909.909B18208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54202:fdde7a2ddb35
Date: 2012-03-28 11:35 +0000
http://bitbucket.org/pypy/pypy/changeset/fdde7a2ddb35/

Log:	implement convert_float_bytes_to_longlong

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -1297,3 +1297,5 @@
         assert res.is_reg()
         self.mc.VMOV_rc(res.value, r.ip.value, loc.value)
         return fcond
+
+    emit_op_convert_float_bytes_to_longlong = gen_emit_unary_float_op('float_bytes_to_longlong', 'VMOV_cc')
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -1204,6 +1204,9 @@
         self.force_spill_var(op.getarg(0))
         return []
 
+    prepare_op_convert_float_bytes_to_longlong = prepare_float_op(base=False,
+			    name='prepare_op_convert_float_bytes_to_longlong')
+
 
 def add_none_argument(fn):
     return lambda self, op, fcond: fn(self, op, None, fcond)

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:10 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:10 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: mark test_convert_float_bytes as
 requiring longlong support
Message-ID: <20120405121910.D10898208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54203:20eca32afd29
Date: 2012-03-28 11:37 +0000
http://bitbucket.org/pypy/pypy/changeset/20eca32afd29/

Log:	mark test_convert_float_bytes as requiring longlong support

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1776,6 +1776,8 @@
         assert res == -19
 
     def test_convert_float_bytes(self):
+        if not self.cpu.supports_longlong:
+            py.test.skip("longlong test")
         t = 'int' if longlong.is_64_bit else 'float'
         res = self.execute_operation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG,
                                      [boxfloat(2.5)], t).value

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:12 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:12 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: fix merge
Message-ID: <20120405121912.26EF78208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54204:43c292b41c22
Date: 2012-03-28 11:40 +0000
http://bitbucket.org/pypy/pypy/changeset/43c292b41c22/

Log:	fix merge

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1657,6 +1657,7 @@
     def test_read_timestamp(self):
         if not self.cpu.supports_longlong:
             py.test.skip("longlong test")
+        if sys.platform == 'win32':
             # so we stretch the time a little bit.
             # On my virtual Parallels machine in a 2GHz Core i7 Mac Mini,
             # the test starts working at delay == 21670 and stops at 20600000.

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:13 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:13 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: implement the read timestamp
 operation. But it only works in privileged mode. Bah.
Message-ID: <20120405121913.8F6CA8208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54205:81fad4c1e587
Date: 2012-04-05 12:07 +0000
http://bitbucket.org/pypy/pypy/changeset/81fad4c1e587/

Log:	implement the read timestamp operation. But it only works in
	privileged mode. Bah.

diff --git a/pypy/jit/backend/arm/instruction_builder.py b/pypy/jit/backend/arm/instruction_builder.py
--- a/pypy/jit/backend/arm/instruction_builder.py
+++ b/pypy/jit/backend/arm/instruction_builder.py
@@ -223,6 +223,7 @@
     n = (0x3 << 26 | (table['op1'] & 0x3F) << 20 | (table['op'] & 0x1) << 4)
 
     def f(self, coproc, opc1, rt, crn, crm, opc2=0, cond=cond.AL):
+        assert coproc & 0xE != 0xA
         self.write32(n
                     | cond << 28
                     | (opc1 & 0x7) << 21
diff --git a/pypy/jit/backend/arm/instructions.py b/pypy/jit/backend/arm/instructions.py
--- a/pypy/jit/backend/arm/instructions.py
+++ b/pypy/jit/backend/arm/instructions.py
@@ -93,6 +93,7 @@
 
 supervisor_and_coproc = {
     'MCR': {'op1': 0x20, 'op': 1, 'rn':0, 'coproc':0},
+    'MRC': {'op1': 0x21, 'op': 1, 'rn':0, 'coproc':0},
 }
 
 block_data = {
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -1299,3 +1299,11 @@
         return fcond
 
     emit_op_convert_float_bytes_to_longlong = gen_emit_unary_float_op('float_bytes_to_longlong', 'VMOV_cc')
+
+    def emit_op_read_timestamp(self, op, arglocs, regalloc, fcond):
+	tmp = arglocs[0]
+        res = arglocs[1]
+        self.mc.MRC(15, 0, tmp.value, 15, 12, 1)
+	self.mc.MOV_ri(r.ip.value, 0)
+        self.mc.VMOV_cr(res.value, tmp.value, r.ip.value)
+        return fcond
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -1207,6 +1207,11 @@
     prepare_op_convert_float_bytes_to_longlong = prepare_float_op(base=False,
 			    name='prepare_op_convert_float_bytes_to_longlong')
 
+    def prepare_op_read_timestamp(self, op, fcond):
+        loc = self.get_scratch_reg(INT)
+        res = self.vfprm.force_allocate_reg(op.result)
+        return [loc, res]
+
 
 def add_none_argument(fn):
     return lambda self, op, fcond: fn(self, op, None, fcond)

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:14 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:14 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: disable longlong support until I
 find a way to implement read_timestamp on ARM
Message-ID: <20120405121914.D54FD8208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54206:b3ce03aac72d
Date: 2012-04-05 12:14 +0000
http://bitbucket.org/pypy/pypy/changeset/b3ce03aac72d/

Log:	disable longlong support until I find a way to implement
	read_timestamp on ARM

diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py
--- a/pypy/jit/backend/arm/runner.py
+++ b/pypy/jit/backend/arm/runner.py
@@ -9,7 +9,8 @@
 class ArmCPU(AbstractLLCPU):
 
     supports_floats = True
-    supports_longlong = True
+    supports_longlong = False # XXX requires an implementation of
+                              # read_timestamp that works in user mode
 
     def __init__(self, rtyper, stats, opts=None, translate_support_code=False,
                  gcdescr=None):

From noreply at buildbot.pypy.org  Thu Apr  5 14:19:16 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Thu,  5 Apr 2012 14:19:16 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: backout 20eca32afd29
Message-ID: <20120405121916.1CE668208A@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54207:30c3a37d897a
Date: 2012-04-05 12:17 +0000
http://bitbucket.org/pypy/pypy/changeset/30c3a37d897a/

Log:	backout 20eca32afd29

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1777,8 +1777,6 @@
         assert res == -19
 
     def test_convert_float_bytes(self):
-        if not self.cpu.supports_longlong:
-            py.test.skip("longlong test")
         t = 'int' if longlong.is_64_bit else 'float'
         res = self.execute_operation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG,
                                      [boxfloat(2.5)], t).value

From pullrequests-noreply at bitbucket.org  Thu Apr  5 15:59:34 2012
From: pullrequests-noreply at bitbucket.org (Amaury Forgeot d'Arc)
Date: Thu, 05 Apr 2012 13:59:34 -0000
Subject: [pypy-commit] [pypy/pypy] PyFile_FromFile implementation (pull
	request #68)
In-Reply-To: <819aec1e6e3998acaa7b35b136186f8a@bitbucket.org>
References: <819aec1e6e3998acaa7b35b136186f8a@bitbucket.org>
Message-ID: <20120405135934.7050.41402@bitbucket12.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/68/pyfile_fromfile-implementation#comment-4832

Amaury Forgeot d'Arc (amauryfa) said:

Looks good. I'd fflush() the FILE* though, to avoid a potential loss of data.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Thu Apr  5 16:55:29 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  5 Apr 2012 16:55:29 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: simplify
Message-ID: <20120405145529.8A75E8208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54208:f1aae6ad9af7
Date: 2012-04-05 17:53 +0300
http://bitbucket.org/pypy/pypy/changeset/f1aae6ad9af7/

Log:	simplify

diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -455,6 +455,6 @@
     return intmask(time), intmask(nsec)
 
 def time_t_to_FILE_TIME(time, filetime):
-    ft = rffi.r_longlong(time * 10000000) + secs_between_epochs * 10000000
+    ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
     filetime.c_dwLowDateTime = rffi.r_uint(ft)    # masking off high bits

From noreply at buildbot.pypy.org  Thu Apr  5 16:55:30 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu,  5 Apr 2012 16:55:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix float-to-time conversion in windows
Message-ID: <20120405145530.C374A8208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54209:a7b45b1ec178
Date: 2012-04-05 17:54 +0300
http://bitbucket.org/pypy/pypy/changeset/a7b45b1ec178/

Log:	fix float-to-time conversion in windows

diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -455,6 +455,6 @@
     return intmask(time), intmask(nsec)
 
 def time_t_to_FILE_TIME(time, filetime):
-    ft = (rffi.r_longlong(time) + secs_between_epochs) * 10000000
+    ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
     filetime.c_dwLowDateTime = rffi.r_uint(ft)    # masking off high bits

From pullrequests-noreply at bitbucket.org  Thu Apr  5 17:00:42 2012
From: pullrequests-noreply at bitbucket.org (Roberto De Ioris)
Date: Thu, 05 Apr 2012 15:00:42 -0000
Subject: [pypy-commit] [pypy/pypy] PyFile_FromFile implementation (pull
	request #68)
In-Reply-To: <819aec1e6e3998acaa7b35b136186f8a@bitbucket.org>
References: <819aec1e6e3998acaa7b35b136186f8a@bitbucket.org>
Message-ID: <20120405150042.13043.27165@bitbucket15.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/68/pyfile_fromfile-implementation#comment-4836

Roberto De Ioris (unbit) said:

that should be enough:
https://bitbucket.org/unbit/pypy/changeset/d356d169e770

i will remake the pull-request later as i am about to commit another couple of functions

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Thu Apr  5 18:27:22 2012
From: noreply at buildbot.pypy.org (ctismer)
Date: Thu,  5 Apr 2012 18:27:22 +0200 (CEST)
Subject: [pypy-commit] pypy win64-stage1: Closed branch win64-stage1
Message-ID: <20120405162722.C5E238208A@wyvern.cs.uni-duesseldorf.de>

Author: Christian Tismer  <tismer at stackless.com>
Branch: win64-stage1
Changeset: r54210:e37b6d1dd329
Date: 2012-04-05 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/e37b6d1dd329/

Log:	Closed branch win64-stage1


From noreply at buildbot.pypy.org  Thu Apr  5 23:39:04 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu,  5 Apr 2012 23:39:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Try to better support -fPIC compiler
 option in trackgcroot.
Message-ID: <20120405213904.7F41F8208A@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54211:10f1ff48184b
Date: 2012-04-05 23:38 +0200
http://bitbucket.org/pypy/pypy/changeset/10f1ff48184b/

Log:	Try to better support -fPIC compiler option in trackgcroot.

diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -847,6 +847,10 @@
                 if sources:
                     target, = sources
 
+        if target.endswith('@PLT'):
+            # In -fPIC mode, all functions calls have this suffix
+            target = target[:-4]
+
         if target in self.FUNCTIONS_NOT_RETURNING:
             return [InsnStop(target)]
         if self.format == 'mingw32' and target == '__alloca':
@@ -1137,7 +1141,7 @@
     r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$")
 
     r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$")
-    r_jmptable_item = re.compile(r"\t.quad\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$")
+    r_jmptable_item = re.compile(r"\t.(?:quad|long)\t"+LABEL+"(-\"[A-Za-z0-9$]+\"|-"+LABEL+")?\s*$")
     r_jmptable_end  = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL)
 
     r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/")

From noreply at buildbot.pypy.org  Fri Apr  6 09:02:42 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  6 Apr 2012 09:02:42 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge from default
Message-ID: <20120406070242.485108208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54212:b06aa6be5f37
Date: 2012-04-05 18:05 +0300
http://bitbucket.org/pypy/pypy/changeset/b06aa6be5f37/

Log:	merge from default

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -968,8 +968,7 @@
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
+        return NotImplemented    # note that this doesn't work on CPython 2.2
 
     __radd__ = __add__
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -149,6 +149,22 @@
 exported.  This would give us a one-size-fits-all generic .so file to be
 imported by any application that wants to load .so files :-)
 
+Optimising cpyext (CPython C-API compatibility layer)
+-----------------------------------------------------
+
+A lot of work has gone into PyPy's implementation of CPython's C-API over
+the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However,
+there are still many edges and corner cases where it misbehaves, and it has
+not received any substantial optimisation so far.
+
+The objective of this project is to fix bugs in cpyext and to optimise
+several performance critical parts of it, such as the reference counting
+support and other heavily used C-API functions. The net result would be to
+have CPython extensions run much faster on PyPy than they currently do, or
+to make them work at all if they currently don't. A part of this work would
+be to get cpyext into a shape where it supports running Cython generated
+extensions.
 
 .. _`issue tracker`: http://bugs.pypy.org
 .. _`mailing list`: http://mail.python.org/mailman/listinfo/pypy-dev
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -199,17 +199,11 @@
 The following features (present in some past Stackless version of PyPy)
 are for the time being not supported any more:
 
-* Tasklets and channels (currently ``stackless.py`` seems to import,
-  but you have tasklets on top of coroutines on top of greenlets on
-  top of continulets on top of stacklets, and it's probably not too
-  hard to cut two of these levels by adapting ``stackless.py`` to
-  use directly continulets)
-
 * Coroutines (could be rewritten at app-level)
 
-* Pickling and unpickling continulets (*)
-
-* Continuing execution of a continulet in a different thread (*)
+* Continuing execution of a continulet in a different thread
+  (but if it is "simple enough", you can pickle it and unpickle it
+  in the other thread).
 
 * Automatic unlimited stack (must be emulated__ so far)
 
@@ -217,15 +211,6 @@
 
 .. __: `recursion depth limit`_
 
-(*) Pickling, as well as changing threads, could be implemented by using
-a "soft" stack switching mode again.  We would get either "hard" or
-"soft" switches, similarly to Stackless Python 3rd version: you get a
-"hard" switch (like now) when the C stack contains non-trivial C frames
-to save, and a "soft" switch (like previously) when it contains only
-simple calls from Python to Python.  Soft-switched continulets would
-also consume a bit less RAM, and the switch might be a bit faster too
-(unsure about that; what is the Stackless Python experience?).
-
 
 Recursion depth limit
 +++++++++++++++++++++
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -296,6 +296,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self.frame_trace_action = FrameTraceAction(self)
+        self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
         self.our_magic = default_magic
@@ -467,9 +468,9 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: time2 or rctime take precedence over time
+        # a bit of custom logic: rctime take precedence over time
         # XXX this could probably be done as a "requires" in the config
-        if ('time2' in modules or 'rctime' in modules) and 'time' in modules:
+        if 'rctime' in modules and 'time' in modules:
             modules.remove('time')
 
         if not self.config.objspace.nofaking:
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -154,6 +154,7 @@
         #operationerr.print_detailed_traceback(self.space)
 
     def _convert_exc(self, operr):
+        # Only for the flow object space
         return operr
 
     def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
@@ -166,6 +167,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -113,6 +113,12 @@
         from pypy.interpreter.pycode import PyCode
 
         code = self.getcode() # hook for the jit
+        #
+        if (jit.we_are_jitted() and code is self.space._code_of_sys_exc_info
+                                and nargs == 0):
+            from pypy.module.sys.vm import exc_info_direct
+            return exc_info_direct(self.space, frame)
+        #
         fast_natural_arity = code.fast_natural_arity
         if nargs == fast_natural_arity:
             if nargs == 0:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -874,6 +874,12 @@
             fn.add_to_table()
         if gateway.as_classmethod:
             fn = ClassMethod(space.wrap(fn))
+        #
+        from pypy.module.sys.vm import exc_info
+        if code._bltin is exc_info:
+            assert space._code_of_sys_exc_info is None
+            space._code_of_sys_exc_info = code
+        #
         return fn
 
 
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -2,34 +2,39 @@
 from pypy.interpreter import unicodehelper
 from pypy.rlib.rstring import StringBuilder
 
-def parsestr(space, encoding, s, unicode_literals=False):
-    # compiler.transformer.Transformer.decode_literal depends on what 
-    # might seem like minor details of this function -- changes here 
-    # must be reflected there.
+def parsestr(space, encoding, s, unicode_literal=False):
+    """Parses a string or unicode literal, and return a wrapped value.
+
+    If encoding=iso8859-1, the source string is also in this encoding.
+    If encoding=None, the source string is ascii only.
+    In other cases, the source string is in utf-8 encoding.
+
+    When a bytes string is returned, it will be encoded with the
+    original encoding.
+
+    Yes, it's very inefficient.
+    Yes, CPython has very similar code.
+    """
 
     # we use ps as "pointer to s"
     # q is the virtual last char index of the string
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = unicode_literals
 
     # string decoration handling
-    o = ord(quote)
-    isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
-    if isalpha or quote == '_':
-        if quote == 'b' or quote == 'B':
-            ps += 1
-            quote = s[ps]
-            unicode = False
-        elif quote == 'u' or quote == 'U':
-            ps += 1
-            quote = s[ps]
-            unicode = True
-        if quote == 'r' or quote == 'R':
-            ps += 1
-            quote = s[ps]
-            rawmode = True
+    if quote == 'b' or quote == 'B':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = False
+    elif quote == 'u' or quote == 'U':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = True
+    if quote == 'r' or quote == 'R':
+        ps += 1
+        quote = s[ps]
+        rawmode = True
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
                              'Internal error: parser passed unquoted literal')
@@ -46,21 +51,28 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
-    if unicode: # XXX Py_UnicodeFlag is ignored for now
+    if unicode_literal: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None or encoding == "iso-8859-1":
+            # 'unicode_escape' expects latin-1 bytes, string is ready.
             buf = s
             bufp = ps
             bufq = q
             u = None
         else:
-            # "\XX" may become "\u005c\uHHLL" (12 bytes)
+            # String is utf8-encoded, but 'unicode_escape' expects
+            # latin-1; So multibyte sequences must be escaped.
             lis = [] # using a list to assemble the value
             end = q
+            # Worst case: "\XX" may become "\u005c\uHHLL" (12 bytes)
             while ps < end:
                 if s[ps] == '\\':
                     lis.append(s[ps])
                     ps += 1
                     if ord(s[ps]) & 0x80:
+                        # A multibyte sequence will follow, it will be
+                        # escaped like \u1234. To avoid confusion with
+                        # the backslash we just wrote, we emit "\u005c"
+                        # instead.
                         lis.append("u005c")
                 if ord(s[ps]) & 0x80: # XXX inefficient
                     w, ps = decode_utf8(space, s, ps, end, "utf-16-be")
@@ -86,13 +98,11 @@
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
-    # XXX add strchr like interface to rtyper
     assert 0 <= ps <= q
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
             w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
-            #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1797,6 +1797,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1813,6 +1814,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -27,6 +27,12 @@
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
 
+def boxlonglong(ll):
+    if longlong.is_64_bit:
+        return BoxInt(ll)
+    else:
+        return BoxFloat(ll)
+
 
 class Runner(object):
 
@@ -1623,6 +1629,11 @@
                                      [boxfloat(2.5)], t).value
         assert res == longlong2float.float2longlong(2.5)
 
+        bytes = longlong2float.float2longlong(2.5)
+        res = self.execute_operation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT,
+                                     [boxlonglong(res)], 'float').value
+        assert longlong.getrealfloat(res) == 2.5
+
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -328,6 +328,15 @@
     def produce_into(self, builder, r):
         self.put(builder, [r.choice(builder.intvars)])
 
+class CastLongLongToFloatOperation(AbstractFloatOperation):
+    def produce_into(self, builder, r):
+        if longlong.is_64_bit:
+            self.put(builder, [r.choice(builder.intvars)])
+        else:
+            if not builder.floatvars:
+                raise CannotProduceOperation
+            self.put(builder, [r.choice(builder.floatvars)])
+
 class CastFloatToIntOperation(AbstractFloatOperation):
     def produce_into(self, builder, r):
         if not builder.floatvars:
@@ -450,6 +459,7 @@
 OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT))
 OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT))
 OPERATIONS.append(CastFloatToIntOperation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG))
+OPERATIONS.append(CastLongLongToFloatOperation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT))
 
 OperationBuilder.OPERATIONS = OPERATIONS
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1251,6 +1251,15 @@
         else:
             self.mov(loc0, resloc)
 
+    def genop_convert_longlong_bytes_to_float(self, op, arglocs, resloc):
+        loc0, = arglocs
+        if longlong.is_64_bit:
+            assert isinstance(resloc, RegLoc)
+            assert isinstance(loc0, RegLoc)
+            self.mc.MOVD(resloc, loc0)
+        else:
+            self.mov(loc0, resloc)
+
     def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
         guard_opnum = guard_op.getopnum()
         self.mc.CMP(arglocs[0], imm0)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -777,7 +777,20 @@
             loc0 = self.xrm.loc(arg0)
             loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
             self.Perform(op, [loc0], loc1)
-            self.xrm.possibly_free_var(op.getarg(0))
+            self.xrm.possibly_free_var(arg0)
+
+    def consider_convert_longlong_bytes_to_float(self, op):
+        if longlong.is_64_bit:
+            loc0 = self.rm.make_sure_var_in_reg(op.getarg(0))
+            loc1 = self.xrm.force_allocate_reg(op.result)
+            self.Perform(op, [loc0], loc1)
+            self.rm.possibly_free_var(op.getarg(0))
+        else:
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.make_sure_var_in_reg(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
 
     def _consider_llong_binop_xx(self, op):
         # must force both arguments into xmm registers, because we don't
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -295,6 +295,7 @@
         return op
 
     rewrite_op_convert_float_bytes_to_longlong = _noop_rewrite
+    rewrite_op_convert_longlong_bytes_to_float = _noop_rewrite
 
     # ----------
     # Various kinds of calls
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -968,20 +968,22 @@
             int_return %i2
         """, transform=True)
 
-    def test_convert_float_bytes_to_int(self):
-        from pypy.rlib.longlong2float import float2longlong
+    def test_convert_float_bytes(self):
+        from pypy.rlib.longlong2float import float2longlong, longlong2float
         def f(x):
-            return float2longlong(x)
+            ll = float2longlong(x)
+            return longlong2float(ll)
         if longlong.is_64_bit:
-            result_var = "%i0"
-            return_op = "int_return"
+            tmp_var = "%i0"
+            result_var = "%f1"
         else:
-            result_var = "%f1"
-            return_op = "float_return"
+            tmp_var = "%f1"
+            result_var = "%f2"
         self.encoding_test(f, [25.0], """
-            convert_float_bytes_to_longlong %%f0 -> %(result_var)s
-            %(return_op)s %(result_var)s
-        """ % {"result_var": result_var, "return_op": return_op})
+            convert_float_bytes_to_longlong %%f0 -> %(tmp_var)s
+            convert_longlong_bytes_to_float %(tmp_var)s -> %(result_var)s
+            float_return %(result_var)s
+        """ % {"result_var": result_var, "tmp_var": tmp_var}, transform=True)
 
 
 def check_force_cast(FROM, TO, operations, value):
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -672,6 +672,11 @@
         a = longlong.getrealfloat(a)
         return longlong2float.float2longlong(a)
 
+    @arguments(LONGLONG_TYPECODE, returns="f")
+    def bhimpl_convert_longlong_bytes_to_float(a):
+        a = longlong2float.longlong2float(a)
+        return longlong.getfloatstorage(a)
+
     # ----------
     # control flow operations
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -224,6 +224,7 @@
                     'float_neg', 'float_abs',
                     'cast_ptr_to_int', 'cast_int_to_ptr',
                     'convert_float_bytes_to_longlong',
+                    'convert_longlong_bytes_to_float',
                     ]:
         exec py.code.Source('''
             @arguments("box")
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -420,6 +420,7 @@
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     'CONVERT_FLOAT_BYTES_TO_LONGLONG/1',
+    'CONVERT_LONGLONG_BYTES_TO_FLOAT/1',
     #
     'INT_LT/2b',
     'INT_LE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,3 +1,4 @@
+import math
 import sys
 
 import py
@@ -15,7 +16,7 @@
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
     isconstant, isvirtual, promote_string, set_param, record_known_class)
-from pypy.rlib.longlong2float import float2longlong
+from pypy.rlib.longlong2float import float2longlong, longlong2float
 from pypy.rlib.rarithmetic import ovfcheck, is_valid_int
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3795,15 +3796,15 @@
         res = self.interp_operations(g, [1])
         assert res == 3
 
-    def test_float2longlong(self):
+    def test_float_bytes(self):
         def f(n):
-            return float2longlong(n)
+            ll = float2longlong(n)
+            return longlong2float(ll)
 
         for x in [2.5, float("nan"), -2.5, float("inf")]:
             # There are tests elsewhere to verify the correctness of this.
-            expected = float2longlong(x)
             res = self.interp_operations(f, [x])
-            assert longlong.getfloatstorage(res) == expected
+            assert res == x or math.isnan(x) and math.isnan(res)
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -432,7 +432,8 @@
                     raise _ssl_seterror(self.space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return self.space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return self.space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                              length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -11,7 +11,7 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.objectmodel import specialize
+from pypy.rlib.objectmodel import specialize, keepalive_until_here
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
@@ -145,18 +145,24 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(RWBuffer):
-    def __init__(self, data, bytes):
-        self.data = data
-        self.len = bytes
+    def __init__(self, array):
+        self.array = array
 
     def getlength(self):
-        return self.len
+        return self.array.len * self.array.itemsize
 
     def getitem(self, index):
-        return self.data[index]
+        array = self.array
+        data = array._charbuf_start()
+        char = data[index]
+        array._charbuf_stop()
+        return char
 
     def setitem(self, index, char):
-        self.data[index] = char
+        array = self.array
+        data = array._charbuf_start()
+        data[index] = char
+        array._charbuf_stop()
 
 
 def make_array(mytype):
@@ -278,9 +284,10 @@
             oldlen = self.len
             new = len(s) / mytype.bytes
             self.setlen(oldlen + new)
-            cbuf = self.charbuf()
+            cbuf = self._charbuf_start()
             for i in range(len(s)):
                 cbuf[oldlen * mytype.bytes + i] = s[i]
+            self._charbuf_stop()
 
         def fromlist(self, w_lst):
             s = self.len
@@ -310,8 +317,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def charbuf(self):
-            return  rffi.cast(rffi.CCHARP, self.buffer)
+        def _charbuf_start(self):
+            return rffi.cast(rffi.CCHARP, self.buffer)
+
+        def _charbuf_stop(self):
+            keepalive_until_here(self)
 
         def w_getitem(self, space, idx):
             item = self.buffer[idx]
@@ -530,8 +540,10 @@
         self.fromstring(space.str_w(w_s))
 
     def array_tostring__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
+        self._charbuf_stop()
+        return self.space.wrap(s)
 
     def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
         if not isinstance(w_f, W_File):
@@ -613,8 +625,7 @@
     # Misc methods
 
     def buffer__Array(space, self):
-        b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes)
-        return space.wrap(b)
+        return space.wrap(ArrayBuffer(self))
 
     def array_buffer_info__Array(space, self):
         w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer))
@@ -649,7 +660,7 @@
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
-        bytes = self.charbuf()
+        bytes = self._charbuf_start()
         tmp = [bytes[0]] * mytype.bytes
         for start in range(0, self.len * mytype.bytes, mytype.bytes):
             stop = start + mytype.bytes - 1
@@ -657,6 +668,7 @@
                 tmp[i] = bytes[start + i]
             for i in range(mytype.bytes):
                 bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
 
     def repr__Array(space, self):
         if self.len == 0:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -433,7 +433,25 @@
         a = self.array('h', 'Hi')
         buf = buffer(a)
         assert buf[1] == 'i'
-        #raises(TypeError, buf.__setitem__, 1, 'o')
+
+    def test_buffer_write(self):
+        a = self.array('c', 'hello')
+        buf = buffer(a)
+        print repr(buf)
+        try:
+            buf[3] = 'L'
+        except TypeError:
+            skip("buffer(array) returns a read-only buffer on CPython")
+        assert a.tostring() == 'helLo'
+
+    def test_buffer_keepalive(self):
+        buf = buffer(self.array('c', 'text'))
+        assert buf[2] == 'x'
+        #
+        a = self.array('c', 'foobarbaz')
+        buf = buffer(a)
+        a.fromstring('some extra text')
+        assert buf[:] == 'foobarbazsome extra text'
 
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,8 +2,10 @@
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
     PyObjectFields, PyObject)
-from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
+from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
+from pypy.interpreter.error import OperationError
+from pypy.module.array.interp_array import ArrayBuffer
 
 
 PyBufferObjectStruct = lltype.ForwardReference()
@@ -41,26 +43,38 @@
         py_buf.c_b_offset = w_obj.offset
         w_obj = w_obj.buffer
 
+    # If w_obj already allocated a fixed buffer, use it, and keep a
+    # reference to w_obj.
+    # Otherwise, b_base stays NULL, and we own the b_ptr.
+
     if isinstance(w_obj, StringBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.as_str()))
+        py_buf.c_b_base = lltype.nullptr(PyObject.TO)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
+        py_buf.c_b_size = w_obj.getlength()
+    elif isinstance(w_obj, ArrayBuffer):
+        w_base = w_obj.array
+        py_buf.c_b_base = make_ref(space, w_base)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start())
         py_buf.c_b_size = w_obj.getlength()
     else:
-        raise Exception("Fail fail fail fail fail")
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "buffer flavor not supported"))
 
 
 def buffer_realize(space, py_obj):
     """
     Creates the buffer in the PyPy interpreter from a cpyext representation.
     """
-    raise Exception("realize fail fail fail")
-
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "Don't know how to realize a buffer"))
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
-    Py_DecRef(space, py_buf.c_b_base)
-    rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
+    if py_buf.c_b_base:
+        Py_DecRef(space, py_buf.c_b_base)
+    else:
+        rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -315,3 +316,65 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,14 +167,16 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -183,7 +185,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -48,3 +48,17 @@
             ])
         b = module.buffer_new()
         raises(AttributeError, getattr, b, 'x')
+
+    def test_array_buffer(self):
+        module = self.import_extension('foo', [
+            ("roundtrip", "METH_O",
+             """
+                 PyBufferObject *buf = (PyBufferObject *)args;
+                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
+             """),
+            ])
+        import array
+        a = array.array('c', 'text')
+        b = buffer(a)
+        assert module.roundtrip(b) == 'text'
+        
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
@@ -414,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -425,6 +429,36 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
+
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -307,6 +307,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -624,7 +626,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
@@ -99,9 +100,12 @@
         ("exp2", "exp2"),
         ("expm1", "expm1"),
         ("fabs", "fabs"),
+        ("fmax", "fmax"),
+        ("fmin", "fmin"),
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -122,12 +126,16 @@
         ("sinh", "sinh"),
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
+        ('square', 'square'),
         ("tan", "tan"),
         ("tanh", "tanh"),
         ('bitwise_and', 'bitwise_and'),
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
+        ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
         ('isneginf', 'isneginf'),
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -673,6 +673,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -1261,6 +1265,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1386,6 +1415,7 @@
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,9 +3,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype, support, loop
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
+
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
@@ -28,7 +30,7 @@
         return self.identity
 
     def descr_call(self, space, __args__):
-	from interp_numarray import BaseArray
+        from interp_numarray import BaseArray
         args_w, kwds_w = __args__.unpack()
         # it occurs to me that we don't support any datatypes that
         # require casting, change it later when we do
@@ -179,7 +181,7 @@
                 elif out.shape != shape:
                     raise operationerrfmt(space.w_ValueError,
                         'output parameter shape mismatch, expecting [%s]' +
-                        ' , got [%s]', 
+                        ' , got [%s]',
                         ",".join([str(x) for x in shape]),
                         ",".join([str(x) for x in out.shape]),
                         )
@@ -204,7 +206,7 @@
         else:
             arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
             val = loop.compute(arr)
-        return val 
+        return val
 
     def do_axis_reduce(self, obj, dtype, axis, result):
         from pypy.module.micronumpy.interp_numarray import AxisReduce
@@ -253,7 +255,7 @@
         if isinstance(w_obj, Scalar):
             arr = self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))
             if isinstance(out,Scalar):
-                out.value=arr
+                out.value = arr
             elif isinstance(out, BaseArray):
                 out.fill(space, arr)
             else:
@@ -265,7 +267,7 @@
             if not broadcast_shape or broadcast_shape != out.shape:
                 raise operationerrfmt(space.w_ValueError,
                     'output parameter shape mismatch, could not broadcast [%s]' +
-                    ' to [%s]', 
+                    ' to [%s]',
                     ",".join([str(x) for x in w_obj.shape]),
                     ",".join([str(x) for x in out.shape]),
                     )
@@ -292,10 +294,11 @@
         self.func = func
         self.comparison_func = comparison_func
 
+    @jit.unroll_safe
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
             convert_to_array, Scalar, shape_agreement, BaseArray)
-        if len(args_w)>2:
+        if len(args_w) > 2:
             [w_lhs, w_rhs, w_out] = args_w
         else:
             [w_lhs, w_rhs] = args_w
@@ -326,7 +329,7 @@
                 w_rhs.value.convert_to(calc_dtype)
             )
             if isinstance(out,Scalar):
-                out.value=arr
+                out.value = arr
             elif isinstance(out, BaseArray):
                 out.fill(space, arr)
             else:
@@ -337,7 +340,7 @@
         if out and out.shape != shape_agreement(space, new_shape, out.shape):
             raise operationerrfmt(space.w_ValueError,
                 'output parameter shape mismatch, could not broadcast [%s]' +
-                ' to [%s]', 
+                ' to [%s]',
                 ",".join([str(x) for x in new_shape]),
                 ",".join([str(x) for x in out.shape]),
                 )
@@ -347,7 +350,6 @@
         w_lhs.add_invalidates(w_res)
         w_rhs.add_invalidates(w_res)
         if out:
-            #out.add_invalidates(w_res) #causes a recursion loop
             w_res.get_concrete()
         return w_res
 
@@ -539,14 +541,18 @@
             ("reciprocal", "reciprocal", 1),
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
+            ("fmax", "fmax", 2, {"promote_to_float": True}),
+            ("fmin", "fmin", 2, {"promote_to_float": True}),
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
+            ("trunc", "trunc", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
             ("exp2", "exp2", 1, {"promote_to_float": True}),
             ("expm1", "expm1", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
+            ('square', 'square', 1, {'promote_to_float': True}),
 
             ("sin", "sin", 1, {"promote_to_float": True}),
             ("cos", "cos", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -107,6 +107,10 @@
         arr.compute_first_step(self, f)
         return f
 
+    def debug_repr(self):
+        # should be overridden, but in case it isn't, provide a default
+        return str(self)
+
 class ConcreteSignature(Signature):
     _immutable_fields_ = ['dtype']
 
@@ -207,7 +211,7 @@
     def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        transforms = transforms + [ViewTransform(arr.chunks)]
+        transforms = [ViewTransform(arr.chunks)] + transforms 
         self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
@@ -215,6 +219,9 @@
         assert isinstance(arr, VirtualSlice)
         return self.child.eval(frame, arr.child)
 
+    def debug_repr(self):
+        return 'VirtualSlice(%s)' % self.child.debug_repr()
+
 class Call1(Signature):
     _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
 
@@ -270,7 +277,7 @@
         from pypy.module.micronumpy.interp_numarray import Call1
 
         assert isinstance(arr, Call1)
-        vtransforms = transforms + [BroadcastTransform(arr.values.shape)]
+        vtransforms = [BroadcastTransform(arr.values.shape)] + transforms
         self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
         self.res._create_iter(iterlist, arraylist, arr.res, transforms)
 
@@ -348,7 +355,7 @@
         from pypy.module.micronumpy.interp_numarray import ResultArray
 
         assert isinstance(arr, ResultArray)
-        rtransforms = transforms + [BroadcastTransform(arr.left.shape)]
+        rtransforms = [BroadcastTransform(arr.left.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -375,7 +382,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
@@ -388,7 +395,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -401,8 +408,8 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -424,7 +431,7 @@
         frame.cur_value = self.binfunc(self.calc_dtype, frame.cur_value, rval)
 
     def debug_repr(self):
-        return 'ReduceSig(%s)' % (self.name, self.right.debug_repr())
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
 
 class SliceloopSignature(Call2):
     def eval(self, frame, arr):
@@ -448,7 +455,7 @@
         from pypy.module.micronumpy.interp_numarray import SliceArray
 
         assert isinstance(arr, SliceArray)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,6 +1,7 @@
 from pypy.rlib import jit
 from pypy.interpreter.error import OperationError
 
+ at jit.look_inside_iff(lambda chunks: jit.isconstant(len(chunks)))
 def enumerate_chunks(chunks):
     result = []
     i = -1
@@ -85,9 +86,9 @@
         space.isinstance_w(w_item_or_slice, space.w_slice)):
         raise OperationError(space.w_IndexError,
                              space.wrap('unsupported iterator index'))
-            
+
     start, stop, step, lngth = space.decode_index4(w_item_or_slice, size)
-        
+
     coords = [0] * len(shape)
     i = start
     if order == 'C':
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,5 +1,9 @@
+from pypy.rlib import jit
+
+
+ at jit.unroll_safe
 def product(s):
     i = 1
     for x in s:
         i *= x
-    return i
\ No newline at end of file
+    return i
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -395,11 +395,19 @@
         assert a[3] == 0.
 
     def test_newaxis(self):
-        from _numpypy import array
+        import math
+        from _numpypy import array, cos, zeros
         from numpypy.core.numeric import newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
+        a = array(range(3))
+        b = array([1, 3])
+        expected = zeros((3, 2))
+        for x in range(3):
+            for y in range(2):
+                expected[x, y] = math.cos(a[x]) * math.cos(b[y])
+        assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):
         from _numpypy import array
@@ -1338,6 +1346,10 @@
         dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0)
         assert str(dims_disagree.value) == \
             "array dimensions must agree except for axis being concatenated"
+        a = array([1, 2, 3, 4, 5, 6])
+        a = (a + a)[::2]
+        b = concatenate((a[:3], a[-3:]))
+        assert (b == [2, 6, 10, 2, 6, 10]).all()
 
     def test_std(self):
         from _numpypy import array
@@ -1387,6 +1399,16 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat, array
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -135,6 +135,38 @@
         assert fabs(float('-inf')) == float('inf')
         assert isnan(fabs(float('nan')))
 
+    def test_fmax(self):
+        from _numpypy import fmax
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmax(a, [ninf]*5) == a).all()
+        assert (fmax(a, [inf]*5) == [inf]*5).all()
+        assert (fmax(a, [1]*5) == [1, 1, 1, 5, inf]).all()
+        assert math.isnan(fmax(nan, 0))
+        assert math.isnan(fmax(0, nan))
+        assert math.isnan(fmax(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+
+    def test_fmin(self):
+        from _numpypy import fmin
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmin(a, [ninf]*5) == [ninf]*5).all()
+        assert (fmin(a, [inf]*5) == a).all()
+        assert (fmin(a, [1]*5) == [ninf, -5, 0, 1, 1]).all()
+        assert math.isnan(fmin(nan, 0))
+        assert math.isnan(fmin(0, nan))
+        assert math.isnan(fmin(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+
     def test_fmod(self):
         from _numpypy import fmod
         import math
@@ -221,24 +253,17 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floorceil(self):
-        from _numpypy import array, floor, ceil
+    def test_floorceiltrunc(self):
+        from _numpypy import array, floor, ceil, trunc
         import math
-        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = floor(a)
-        for i in range(5):
-            assert b[i] == reference[i]
-        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = ceil(a)
-        assert (reference == b).all()
-        inf = float("inf")
-        data = [1.5, 2.9999, -1.999, inf]
-        results = [math.floor(x) for x in data]
-        assert (floor(data) == results).all()
-        results = [math.ceil(x) for x in data]
-        assert (ceil(data) == results).all()
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == floor(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
+        assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -455,6 +480,19 @@
         assert math.isnan(sqrt(-1))
         assert math.isnan(sqrt(nan))
 
+    def test_square(self):
+        import math
+        from _numpypy import square
+
+        nan, inf, ninf = float("nan"), float("inf"), float("-inf")
+
+        assert math.isnan(square(nan))
+        assert math.isinf(square(inf))
+        assert math.isinf(square(ninf))
+        assert square(ninf) > 0
+        assert [square(x) for x in range(-5, 5)] == [x*x for x in range(-5, 5)]
+        assert math.isinf(square(1e300))
+
     def test_radians(self):
         import math
         from _numpypy import radians, array
@@ -546,10 +584,18 @@
         raises(TypeError, 'array([1.0]) & 1')
 
     def test_unary_bitops(self):
-        from _numpypy import bitwise_not, array
+        from _numpypy import bitwise_not, invert, array
         a = array([1, 2, 3, 4])
         assert (~a == [-2, -3, -4, -5]).all()
         assert (bitwise_not(a) == ~a).all()
+        assert (invert(a) == ~a).all()
+
+    def test_shift(self):
+        from _numpypy import left_shift, right_shift
+        import sys
+
+        assert (left_shift([5, 1], [2, 31]) == [20, 2**31]).all()
+        assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
 
     def test_comparisons(self):
         import operator
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -631,6 +631,22 @@
         return math.fabs(v)
 
     @simple_binary_op
+    def fmax(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return max(v1, v2)
+
+    @simple_binary_op
+    def fmin(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return min(v1, v2)
+
+    @simple_binary_op
     def fmod(self, v1, v2):
         try:
             return math.fmod(v1, v2)
@@ -652,6 +668,13 @@
         return math.ceil(v)
 
     @simple_unary_op
+    def trunc(self, v):
+        if v < 0:
+            return math.ceil(v)
+        else:
+            return math.floor(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -741,6 +764,10 @@
         except ValueError:
             return rfloat.NAN
 
+    @simple_unary_op
+    def square(self, v):
+        return v*v
+
     @raw_unary_op
     def isnan(self, v):
         return rfloat.isnan(v)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -212,7 +212,7 @@
             i19 = int_add(i12, 1)
             setfield_gc(p9, i19, descr=<FieldS .*W_AbstractSeqIterObject.inst_index .*>)
             guard_nonnull_class(p17, 146982464, descr=...)
-            i21 = getfield_gc(p17, descr=<FieldS .*W_ArrayTypei.inst_len .*>)
+            i21 = getfield_gc(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
             i24 = getfield_gc(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)
@@ -351,3 +351,23 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(log.loops_by_id("eval")) == 1
+
+    def test_sys_exc_info(self):
+        def main():
+            i = 1
+            lst = [i]
+            while i < 1000:
+                try:
+                    return lst[i]
+                except:
+                    e = sys.exc_info()[1]    # ID: exc_info
+                    if not isinstance(e, IndexError):
+                        raise
+                i += 1
+            return 42
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("exc_info")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -198,3 +198,37 @@
             i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>)
             guard_value(i49, 1, descr=...)
             ''')
+
+    def test_remove_duplicate_method_calls(self):
+        def main(n):
+            lst = []
+            for i in range(n):
+                s = 'Hello %d' % i
+                t = s.lower()   # ID: callone
+                u = s.lower()   # ID: calltwo
+                lst.append(t)
+                lst.append(u)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '''
+            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr . r EF=3>)
+            guard_no_exception(descr=...)
+            ''')
+        loop.match_by_id('calltwo', '')    # nothing
+
+    def test_move_method_call_out_of_loop(self):
+        def main(n):
+            lst = []
+            s = 'Hello %d' % n
+            for i in range(n):
+                t = s.lower()   # ID: callone
+                lst.append(t)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '')    # nothing
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -595,3 +595,124 @@
         assert len(frames) == 1
         _, other_frame = frames.popitem()
         assert other_frame.f_code.co_name in ('other_thread', '?')
+
+
+class AppTestSysExcInfoDirect:
+
+    def setup_method(self, meth):
+        self.checking = not option.runappdirect
+        if self.checking:
+            self.seen = []
+            from pypy.module.sys import vm
+            def exc_info_with_tb(*args):
+                self.seen.append("n")     # not optimized
+                return self.old[0](*args)
+            def exc_info_without_tb(*args):
+                self.seen.append("y")     # optimized
+                return self.old[1](*args)
+            self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
+            vm.exc_info_with_tb = exc_info_with_tb
+            vm.exc_info_without_tb = exc_info_without_tb
+            #
+            from pypy.rlib import jit
+            self.old2 = [jit.we_are_jitted]
+            jit.we_are_jitted = lambda: True
+
+    def teardown_method(self, meth):
+        if self.checking:
+            from pypy.module.sys import vm
+            from pypy.rlib import jit
+            vm.exc_info_with_tb = self.old[0]
+            vm.exc_info_without_tb = self.old[1]
+            jit.we_are_jitted = self.old2[0]
+            #
+            assert ''.join(self.seen) == meth.expected
+
+    def test_returns_none(self):
+        import sys
+        assert sys.exc_info() == (None, None, None)
+        assert sys.exc_info()[0] is None
+        assert sys.exc_info()[1] is None
+        assert sys.exc_info()[2] is None
+        assert sys.exc_info()[:2] == (None, None)
+        assert sys.exc_info()[:3] == (None, None, None)
+        assert sys.exc_info()[0:2] == (None, None)
+        assert sys.exc_info()[2:4] == (None,)
+    test_returns_none.expected = 'nnnnnnnn'
+
+    def test_returns_subscr(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert sys.exc_info()[0] is KeyError  # y
+            assert sys.exc_info()[1] is e         # y
+            assert sys.exc_info()[2] is not None  # n
+            assert sys.exc_info()[-3] is KeyError # y
+            assert sys.exc_info()[-2] is e        # y
+            assert sys.exc_info()[-1] is not None # n
+    test_returns_subscr.expected = 'yynyyn'
+
+    def test_returns_slice_2(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                  # n
+            assert sys.exc_info()[:0] == ()       # y
+            assert sys.exc_info()[:1] == foo[:1]  # y
+            assert sys.exc_info()[:2] == foo[:2]  # y
+            assert sys.exc_info()[:3] == foo      # n
+            assert sys.exc_info()[:4] == foo      # n
+            assert sys.exc_info()[:-1] == foo[:2] # y
+            assert sys.exc_info()[:-2] == foo[:1] # y
+            assert sys.exc_info()[:-3] == ()      # y
+    test_returns_slice_2.expected = 'nyyynnyyy'
+
+    def test_returns_slice_3(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                   # n
+            assert sys.exc_info()[2:2] == ()       # y
+            assert sys.exc_info()[0:1] == foo[:1]  # y
+            assert sys.exc_info()[1:2] == foo[1:2] # y
+            assert sys.exc_info()[0:3] == foo      # n
+            assert sys.exc_info()[2:4] == foo[2:]  # n
+            assert sys.exc_info()[0:-1] == foo[:2] # y
+            assert sys.exc_info()[0:-2] == foo[:1] # y
+            assert sys.exc_info()[5:-3] == ()      # y
+    test_returns_slice_3.expected = 'nyyynnyyy'
+
+    def test_strange_invocation(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            a = []; k = {}
+            assert sys.exc_info(*a)[:0] == ()
+            assert sys.exc_info(**k)[:0] == ()
+    test_strange_invocation.expected = 'nn'
+
+    def test_call_in_subfunction(self):
+        import sys
+        def g():
+            # this case is not optimized, because we need to search the
+            # frame chain.  it's probably not worth the complications
+            return sys.exc_info()[1]
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert g() is e
+    test_call_in_subfunction.expected = 'n'
+
+
+class AppTestSysExcInfoDirectCallMethod(AppTestSysExcInfoDirect):
+    def setup_class(cls):
+        cls.space = gettestobjspace(**{"objspace.opcodes.CALL_METHOD": True})
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -89,6 +89,9 @@
     """Return the (type, value, traceback) of the most recent exception
 caught by an except clause in the current stack frame or in an older stack
 frame."""
+    return exc_info_with_tb(space)    # indirection for the tests
+
+def exc_info_with_tb(space):
     operror = space.getexecutioncontext().sys_exc_info()
     if operror is None:
         return space.newtuple([space.w_None,space.w_None,space.w_None])
@@ -96,6 +99,59 @@
         return space.newtuple([operror.w_type, operror.get_w_value(space),
                                space.wrap(operror.get_traceback())])
 
+def exc_info_without_tb(space, frame):
+    operror = frame.last_exception
+    return space.newtuple([operror.w_type, operror.get_w_value(space),
+                           space.w_None])
+
+def exc_info_direct(space, frame):
+    from pypy.tool import stdlib_opcode
+    # In order to make the JIT happy, we try to return (exc, val, None)
+    # instead of (exc, val, tb).  We can do that only if we recognize
+    # the following pattern in the bytecode:
+    #       CALL_FUNCTION/CALL_METHOD         <-- invoking me
+    #       LOAD_CONST 0, 1, -2 or -3
+    #       BINARY_SUBSCR
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST <=2
+    #       SLICE_2
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST any integer
+    #       LOAD_CONST <=2
+    #       SLICE_3
+    need_all_three_args = True
+    co = frame.getcode().co_code
+    p = frame.last_instr
+    if (ord(co[p]) == stdlib_opcode.CALL_FUNCTION or
+        ord(co[p]) == stdlib_opcode.CALL_METHOD):
+        if ord(co[p+3]) == stdlib_opcode.LOAD_CONST:
+            lo = ord(co[p+4])
+            hi = ord(co[p+5])
+            w_constant = frame.getconstant_w((hi * 256) | lo)
+            if space.isinstance_w(w_constant, space.w_int):
+                constant = space.int_w(w_constant)
+                if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+                    if -3 <= constant <= 1 and constant != -1:
+                        need_all_three_args = False
+                elif ord(co[p+6]) == stdlib_opcode.SLICE+2:
+                    if constant <= 2:
+                        need_all_three_args = False
+                elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
+                      ord(co[p+9]) == stdlib_opcode.SLICE+3):
+                    lo = ord(co[p+7])
+                    hi = ord(co[p+8])
+                    w_constant = frame.getconstant_w((hi * 256) | lo)
+                    if space.isinstance_w(w_constant, space.w_int):
+                        if space.int_w(w_constant) <= 2:
+                            need_all_three_args = False
+    #
+    if need_all_three_args or frame.last_exception is None or frame.hide():
+        return exc_info_with_tb(space)
+    else:
+        return exc_info_without_tb(space, frame)
+
 def exc_clear(space):
     """Clear global information on the current exception.  Subsequent calls
 to exc_info() will return (None,None,None) until another exception is
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -44,3 +44,9 @@
     assert type(dt.microsecond) is int
 
     copy.copy(dt)
+
+def test_radd():
+    class X(object):
+        def __radd__(self, other):
+            return "radd"
+    assert datetime.date(10, 10, 10) + X() == "radd"
diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py
--- a/pypy/module/thread/test/test_ll_thread.py
+++ b/pypy/module/thread/test/test_ll_thread.py
@@ -66,7 +66,6 @@
     def test_gc_locking(self):
         import time
         from pypy.rlib.objectmodel import invoke_around_extcall
-        from pypy.rlib.objectmodel import we_are_translated
         from pypy.rlib.debug import ll_assert
 
         class State:
@@ -129,8 +128,6 @@
             state.finished = 0
             # the next line installs before_extcall() and after_extcall()
             # to be called automatically around external function calls.
-            # When not translated it does not work around time.sleep(),
-            # so we have to call them manually for this test.
             invoke_around_extcall(before_extcall, after_extcall)
 
             g(10, 1)
@@ -142,13 +139,9 @@
                 willing_to_wait_more -= 1
                 done = len(state.answers) == expected
 
-                if not we_are_translated(): before_extcall()
                 time.sleep(0.01)
-                if not we_are_translated(): after_extcall()
 
-            if not we_are_translated(): before_extcall()
             time.sleep(0.1)
-            if not we_are_translated(): after_extcall()
 
             return len(state.answers)
 
@@ -160,12 +153,11 @@
         answers = fn()
         assert answers == expected
 
-class TestRunDirectly(AbstractThreadTests):
-    def getcompiled(self, f, argtypes):
-        return f
-
-    def test_start_new_thread(self):
-        py.test.skip("deadlocks occasionally -- why???")
+#class TestRunDirectly(AbstractThreadTests):
+#    def getcompiled(self, f, argtypes):
+#        return f
+# These are disabled because they crash occasionally for bad reasons
+# related to the fact that ll2ctypes is not at all thread-safe
 
 class TestUsingBoehm(AbstractThreadTests):
     gcpolicy = 'boehm'
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -43,6 +43,13 @@
     return w_eq
 type_eq._annspecialcase_ = 'specialize:memo'
 
+def list_iter(space):
+    "Utility that returns the app-level descriptor list.__iter__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_list,
+                                               '__iter__')
+    return w_iter
+list_iter._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     w_type = space.type(w_obj)
     typename = w_type.getname(space)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -86,6 +86,7 @@
         return s_None
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -163,7 +163,8 @@
 
 class ModuleDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         dict_w = strategy.unerase(dictimplementation.dstorage)
         self.iterator = dict_w.iteritems()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -247,7 +247,7 @@
         return 0
 
     def iter(self, w_dict):
-        return EmptyIteratorImplementation(self.space, w_dict)
+        return EmptyIteratorImplementation(self.space, self, w_dict)
 
     def clear(self, w_dict):
         return
@@ -263,8 +263,9 @@
 # Iterator Implementation base classes
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.dictimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -280,7 +281,20 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.dictimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the dict.  The (key, value) pair in 'result'
+                # might be out-of-date.  We try to explicitly look up
+                # the key in the dict.
+                w_key = result[0]
+                w_value = self.dictimplementation.getitem(w_key)
+                if w_value is None:
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return (w_key, w_value)
         # no more entries
         self.dictimplementation = None
         return None, None
@@ -489,7 +503,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -503,7 +517,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -20,7 +20,17 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_w(w_lookup_type, space.w_str):
+        if (space.is_w(w_lookup_type, space.w_str) or  # Most common path first
+            space.abstract_issubclass_w(w_lookup_type, space.w_str)):
+            return self.getitem_str(w_dict, space.str_w(w_key))
+        elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode):
+            try:
+                w_key = space.str(w_key)
+            except OperationError, e:
+                if not e.match(space, space.w_UnicodeEncodeError):
+                    raise
+                # non-ascii unicode is never equal to a byte string
+                return None
             return self.getitem_str(w_dict, space.str_w(w_key))
         else:
             return None
@@ -98,7 +108,8 @@
 
 class DictProxyIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_type = strategy.unerase(dictimplementation.dstorage)
         self.iterator = w_type.dict_w.iteritems()
 
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -703,7 +703,8 @@
 
 class MapDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_obj = strategy.unerase(dictimplementation.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -439,6 +439,8 @@
             t = w_obj.getitems()
         elif isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
+        elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            t = w_obj.getitems()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
         if expected_length != -1 and len(t) != expected_length:
@@ -456,6 +458,8 @@
             return w_obj.listview_str()
         if isinstance(w_obj, W_StringObject):
             return w_obj.listview_str()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_str()
         return None
 
     def listview_int(self, w_obj):
@@ -465,8 +469,14 @@
             return w_obj.listview_int()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_int()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_int()
         return None
 
+    def _uses_list_iter(self, w_obj):
+        from pypy.objspace.descroperation import list_iter
+        return self.lookup(w_obj, '__iter__') is list_iter(self)
+
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -359,7 +359,7 @@
         w_set.sstorage = w_other.get_storage_copy()
 
     def iter(self, w_set):
-        return EmptyIteratorImplementation(self.space, w_set)
+        return EmptyIteratorImplementation(self.space, self, w_set)
 
     def popitem(self, w_set):
         raise OperationError(self.space.w_KeyError,
@@ -784,8 +784,9 @@
             d_obj[w_item] = None
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.setimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -801,7 +802,17 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.setimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the set.  The 'result' might be out-of-date.
+                # We try to explicitly look it up in the set.
+                if not self.setimplementation.has_key(result):
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return result
         # no more entries
         self.setimplementation = None
         return None
@@ -823,7 +834,7 @@
 
 class StringIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, w_set):
-        IteratorImplementation.__init__(self, space, w_set)
+        IteratorImplementation.__init__(self, space, strategy, w_set)
         d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
@@ -835,9 +846,9 @@
 
 class IntegerIteratorImplementation(IteratorImplementation):
     #XXX same implementation in dictmultiobject on dictstrategy-branch
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
@@ -848,9 +859,9 @@
             return None
 
 class RDictIteratorImplementation(IteratorImplementation):
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -804,6 +804,33 @@
         assert "IntDictStrategy" in self.get_strategy(d)
         assert d[1L] == "hi"
 
+    def test_iter_dict_length_change(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d[7] = 8
+        # 'd' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_dict_strategy_only_change_1(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert d.get(Foo()) is None    # this changes the strategy of 'd'
+        lst = list(it)  # but iterating still works
+        assert sorted(lst) == [(1, 2), (3, 4), (5, 6)]
+
+    def test_iter_dict_strategy_only_change_2(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d['foo'] = 'bar'
+        del d[1]
+        # 'd' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us (1, 2), but 1 is not in the dict any longer.
+        raises(RuntimeError, list, it)
+
 
 class FakeString(str):
     hash_count = 0
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -25,6 +25,16 @@
         key, value = NotEmpty.__dict__.popitem()
         assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
 
+    def test_dictproxy_getitem(self):
+        class NotEmpty(object):
+            a = 1
+        assert 'a' in NotEmpty.__dict__
+        class substr(str): pass
+        assert substr('a') in NotEmpty.__dict__
+        assert u'a' in NotEmpty.__dict__
+        assert NotEmpty.__dict__[u'a'] == 1
+        assert u'\xe9' not in NotEmpty.__dict__
+
     def test_dictproxyeq(self):
         class a(object):
             pass
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1186,14 +1186,23 @@
         # of dicts, because the OrderedDict in the stdlib relies on this.
         # we extend the use case to lists and sets, i.e. all types that have
         # strategies, to avoid surprizes depending on the strategy.
-        for base, arg in [(list, []), (list, [5]), (list, ['x']),
-                          (set, []),  (set,  [5]), (set,  ['x']),
-                          (dict, []), (dict, [(5,6)]), (dict, [('x',7)])]:
+        class X: pass
+        for base, arg in [
+                (list, []), (list, [5]), (list, ['x']), (list, [X]),
+                (set, []),  (set,  [5]), (set,  ['x']), (set, [X]),
+                (dict, []), (dict, [(5,6)]), (dict, [('x',7)]), (dict, [(X,8)]),
+                ]:
             print base, arg
             class SubClass(base):
                 def __iter__(self):
                     return iter("foobar")
             assert list(SubClass(arg)) == ['f', 'o', 'o', 'b', 'a', 'r']
+            class Sub2(base):
+                pass
+            assert list(Sub2(arg)) == list(base(arg))
+            s = set()
+            s.update(Sub2(arg))
+            assert s == set(base(arg))
 
 class AppTestForRangeLists(AppTestW_ListObject):
 
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -907,3 +907,30 @@
                 return [5, 3, 4][i]
         s = set([10,3,2]).intersection(Obj())
         assert list(s) == [3]
+
+    def test_iter_set_length_change(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add(7)
+        # 's' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_set_strategy_only_change_1(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert Foo() not in s      # this changes the strategy of 'd'
+        lst = list(s)  # but iterating still works
+        assert sorted(lst) == [1, 3, 5]
+
+    def test_iter_set_strategy_only_change_2(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add('foo')
+        s.remove(1)
+        # 's' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us 1, but 1 is not in the set any longer.
+        raises(RuntimeError, list, it)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -382,7 +382,7 @@
         pass
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 vref_None = non_virtual_ref(None)
 
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -22,6 +22,7 @@
                 c_name = hop.inputconst(lltype.Void, 'access_helper')
                 args_v = [hop.inputarg(arg, arg=i)
                           for i, arg in enumerate(hop.args_r)]
+                hop.exception_cannot_occur()
                 return hop.genop('jit_marker', [c_name, c_func] + args_v,
                                  resulttype=hop.r_result)
         return helper
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -21,7 +21,7 @@
 FLOAT_ARRAY_PTR = lltype.Ptr(lltype.Array(rffi.FLOAT))
 
 # these definitions are used only in tests, when not translated
-def longlong2float_emulator(llval):
+def longlong2float(llval):
     with lltype.scoped_alloc(DOUBLE_ARRAY_PTR.TO, 1) as d_array:
         ll_array = rffi.cast(LONGLONG_ARRAY_PTR, d_array)
         ll_array[0] = llval
@@ -51,12 +51,6 @@
 
 eci = ExternalCompilationInfo(includes=['string.h', 'assert.h'],
                               post_include_bits=["""
-static double pypy__longlong2float(long long x) {
-    double dd;
-    assert(sizeof(double) == 8 && sizeof(long long) == 8);
-    memcpy(&dd, &x, 8);
-    return dd;
-}
 static float pypy__uint2singlefloat(unsigned int x) {
     float ff;
     assert(sizeof(float) == 4 && sizeof(unsigned int) == 4);
@@ -71,12 +65,6 @@
 }
 """])
 
-longlong2float = rffi.llexternal(
-    "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
-    _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True,
-    oo_primitive="pypy__longlong2float")
-
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
@@ -99,4 +87,17 @@
 
     def specialize_call(self, hop):
         [v_float] = hop.inputargs(lltype.Float)
-        return hop.genop("convert_float_bytes_to_longlong", [v_float], resulttype=hop.r_result)
+        hop.exception_cannot_occur()
+        return hop.genop("convert_float_bytes_to_longlong", [v_float], resulttype=lltype.SignedLongLong)
+
+class LongLong2FloatEntry(ExtRegistryEntry):
+    _about_ = longlong2float
+
+    def compute_result_annotation(self, s_longlong):
+        assert annmodel.SomeInteger(knowntype=r_int64).contains(s_longlong)
+        return annmodel.SomeFloat()
+
+    def specialize_call(self, hop):
+        [v_longlong] = hop.inputargs(lltype.SignedLongLong)
+        hop.exception_cannot_occur()
+        return hop.genop("convert_longlong_bytes_to_float", [v_longlong], resulttype=lltype.Float)
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -215,6 +215,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
@@ -397,6 +398,7 @@
         r_obj, = hop.args_r
         v_obj, = hop.inputargs(r_obj)
         ll_fn = r_obj.get_ll_hash_function()
+        hop.exception_is_here()
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
@@ -419,6 +421,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_identity_hash() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_identityhash', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -441,6 +444,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_id', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -452,6 +456,7 @@
 
     def specialize_call(self, hop):
         vobj, = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         if hop.rtyper.type_system.name == 'lltypesystem':
             from pypy.rpython.lltypesystem import lltype
             if isinstance(vobj.concretetype, lltype.Ptr):
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -85,7 +85,7 @@
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 
 class rbigint(object):
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -100,6 +100,7 @@
         def specialize_call(self, hop):
             bk = hop.rtyper.annotator.bookkeeper
             s_obj = identity.get_input_annotation(bk)
+            hop.exception_cannot_occur()
             return hop.r_result.rtype_erase(hop, s_obj)
 
     class Entry(ExtRegistryEntry):
@@ -110,6 +111,7 @@
             return identity.leave_tunnel(self.bookkeeper)
 
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
@@ -214,6 +216,7 @@
         return hop.genop('cast_opaque_ptr', [v], resulttype=hop.r_result)
 
     def rtype_unerase_int(self, hop, v):
+        hop.exception_cannot_occur()
         return hop.gendirectcall(ll_unerase_int, v)
 
     def rtype_erase_int(self, hop):
@@ -264,6 +267,7 @@
 
     def rtype_unerase_int(self, hop, v):
         c_one = hop.inputconst(lltype.Signed, 1)
+        hop.exception_cannot_occur()
         v2 = hop.genop('oounbox_int', [v], resulttype=hop.r_result)
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -382,6 +382,7 @@
     def compute_result_annotation(self):
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_roots', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
@@ -392,6 +393,7 @@
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_referents', vlist,
                          resulttype = hop.r_result)
 
@@ -402,6 +404,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_memory_usage', vlist,
                          resulttype = hop.r_result)
 
@@ -412,6 +415,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
@@ -430,6 +434,7 @@
         return annmodel.SomeBool()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_is_rpy_instance', vlist,
                          resulttype = hop.r_result)
 
@@ -449,6 +454,7 @@
         classrepr = getclassrepr(hop.rtyper, classdef)
         vtable = classrepr.getvtable()
         assert lltype.typeOf(vtable) == rclass.CLASSTYPE
+        hop.exception_cannot_occur()
         return Constant(vtable, concretetype=rclass.CLASSTYPE)
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rlib/rsre/rsre_re.py b/pypy/rlib/rsre/rsre_re.py
--- a/pypy/rlib/rsre/rsre_re.py
+++ b/pypy/rlib/rsre/rsre_re.py
@@ -172,8 +172,9 @@
         self._ctx = ctx
 
     def span(self, groupnum=0):
-        if not isinstance(groupnum, (int, long)):
-            groupnum = self.re.groupindex[groupnum]
+#        if not isinstance(groupnum, (int, long)):
+#            groupnum = self.re.groupindex[groupnum]
+ 
         return self._ctx.span(groupnum)
 
     def start(self, groupnum=0):
@@ -182,19 +183,25 @@
     def end(self, groupnum=0):
         return self.span(groupnum)[1]
 
-    def group(self, *groups):
-        groups = groups or (0,)
-        result = []
-        for group in groups:
-            frm, to = self.span(group)
-            if 0 <= frm <= to:
-                result.append(self._ctx._string[frm:to])
-            else:
-                result.append(None)
-        if len(result) > 1:
-            return tuple(result)
+    def group(self, group=0):
+        frm, to = self.span(group)
+        if 0 <= frm <= to:
+            return self._ctx._string[frm:to]
         else:
-            return result[0]
+            return None
+
+#    def group(self, *groups):
+#        groups = groups or (0,)
+#        result = []
+#        for group in groups:
+#            frm, to = self.span(group)
+#            if 0 <= frm <= to:
+#                result.append(self._ctx._string[frm:to])
+#            else:
+#                result.append(None)
+#        if len(result) > 1:
+#            return tuple(result)
+
 
     def groups(self, default=None):
         fmarks = self._ctx.flatten_marks()
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -204,7 +204,7 @@
         assert re.match('(a)', 'a').groups() == ('a',)
         assert re.match(r'(a)', 'a').group(0) == 'a'
         assert re.match(r'(a)', 'a').group(1) == 'a'
-        assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
+        #assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
 
         pat = re.compile('((a)|(b))(c)?')
         assert pat.match('a').groups() == ('a', 'a', None, None)
@@ -218,13 +218,13 @@
         assert m.group(0) == 'a'
         assert m.group(0) == 'a'
         assert m.group(1) == 'a'
-        assert m.group(1, 1) == ('a', 'a')
+        #assert m.group(1, 1) == ('a', 'a')
 
         pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?')
-        assert pat.match('a').group(1, 2, 3) == ('a', None, None)
-        assert pat.match('b').group('a1', 'b2', 'c3') == (
-                         (None, 'b', None))
-        assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
+        #assert pat.match('a').group(1, 2, 3) == ('a', None, None)
+        #assert pat.match('b').group('a1', 'b2', 'c3') == (
+        #                 (None, 'b', None))
+        #assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
     def test_bug_923(self):
         # Issue923: grouping inside optional lookahead problem
diff --git a/pypy/rlib/rsre/test/test_zinterp.py b/pypy/rlib/rsre/test/test_zinterp.py
--- a/pypy/rlib/rsre/test/test_zinterp.py
+++ b/pypy/rlib/rsre/test/test_zinterp.py
@@ -1,7 +1,8 @@
 # minimal test: just checks that (parts of) rsre can be translated
 
-from pypy.rpython.test.test_llinterp import gengraph
+from pypy.rpython.test.test_llinterp import gengraph, interpret
 from pypy.rlib.rsre import rsre_core
+from pypy.rlib.rsre.rsre_re import compile
 
 def main(n):
     assert n >= 0
@@ -19,3 +20,18 @@
 
 def test_gengraph():
     t, typer, graph = gengraph(main, [int])
+
+m = compile("(a|b)aaaaa")
+
+def test_match():
+    def f(i):
+        if i:
+            s = "aaaaaa"
+        else:
+            s = "caaaaa"
+        g = m.match(s)
+        if g is None:
+            return 3
+        return int("aaaaaa" == g.group(0))
+    assert interpret(f, [3]) == 1
+    assert interpret(f, [0]) == 3
diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py
--- a/pypy/rlib/rstring.py
+++ b/pypy/rlib/rstring.py
@@ -245,5 +245,5 @@
             raise ValueError("Value is not no_nul")
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
diff --git a/pypy/rlib/test/test_longlong2float.py b/pypy/rlib/test/test_longlong2float.py
--- a/pypy/rlib/test/test_longlong2float.py
+++ b/pypy/rlib/test/test_longlong2float.py
@@ -2,6 +2,7 @@
 from pypy.rlib.longlong2float import longlong2float, float2longlong
 from pypy.rlib.longlong2float import uint2singlefloat, singlefloat2uint
 from pypy.rlib.rarithmetic import r_singlefloat
+from pypy.rpython.test.test_llinterp import interpret
 
 
 def fn(f1):
@@ -31,6 +32,18 @@
         res = fn2(x)
         assert repr(res) == repr(x)
 
+def test_interpreted():
+    def f(f1):
+        try:
+            ll = float2longlong(f1)
+            return longlong2float(ll)
+        except Exception:
+            return 500
+
+    for x in enum_floats():
+        res = interpret(f, [x])
+        assert repr(res) == repr(x)
+
 # ____________________________________________________________
 
 def fnsingle(f1):
diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py
--- a/pypy/rpython/annlowlevel.py
+++ b/pypy/rpython/annlowlevel.py
@@ -543,11 +543,11 @@
         else:
             assert False
 
+        hop.exception_cannot_occur()
         if isinstance(hop.args_r[1], rpbc.NoneFrozenPBCRepr):
             return hop.inputconst(PTR, null)
         v_arg = hop.inputarg(hop.args_r[1], arg=1)
         assert isinstance(v_arg.concretetype, T)
-        hop.exception_cannot_occur()
         return hop.genop(opname, [v_arg], resulttype = PTR)
 
 
diff --git a/pypy/rpython/controllerentry.py b/pypy/rpython/controllerentry.py
--- a/pypy/rpython/controllerentry.py
+++ b/pypy/rpython/controllerentry.py
@@ -201,6 +201,7 @@
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
         assert hop.s_result.is_constant()
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -130,6 +130,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 def enum_ops_without_sideeffects(raising_is_ok=False):
@@ -350,6 +351,7 @@
     'truncate_longlong_to_int':LLOp(canfold=True),
     'force_cast':           LLOp(sideeffects=False),    # only for rffi.cast()
     'convert_float_bytes_to_longlong': LLOp(canfold=True),
+    'convert_longlong_bytes_to_float': LLOp(canfold=True),
 
     # __________ pointer operations __________
 
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -431,6 +431,10 @@
     from pypy.rlib.longlong2float import float2longlong
     return float2longlong(a)
 
+def op_convert_longlong_bytes_to_float(a):
+    from pypy.rlib.longlong2float import longlong2float
+    return longlong2float(a)
+
 
 def op_unichar_eq(x, y):
     assert isinstance(x, unicode) and len(x) == 1
diff --git a/pypy/rpython/lltypesystem/rbuiltin.py b/pypy/rpython/lltypesystem/rbuiltin.py
--- a/pypy/rpython/lltypesystem/rbuiltin.py
+++ b/pypy/rpython/lltypesystem/rbuiltin.py
@@ -9,6 +9,7 @@
 from pypy.rpython.rbool import bool_repr
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr or hop.args_r[1] == pyobj_repr:
@@ -33,6 +34,7 @@
     return my_instantiate()
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     s_class = hop.args_s[0]
     assert isinstance(s_class, annmodel.SomePBC)
     if len(s_class.descriptions) != 1:
@@ -46,6 +48,7 @@
     return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops)
 
 def rtype_builtin_hasattr(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr:
@@ -56,6 +59,7 @@
     raise TyperError("hasattr is only suported on a constant or on PyObject")
 
 def rtype_builtin___import__(hop):
+    xxx    # should not be used any more
     args_v = hop.inputargs(*[pyobj_repr for ign in hop.args_r])
     c = hop.inputconst(pyobj_repr, __import__)
     return hop.genop('simple_call', [c] + args_v, resulttype = pyobj_repr)
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -746,4 +746,5 @@
         assert isinstance(TYPE, GcStruct)
         assert lltype._castdepth(TYPE, OBJECT) > 0
         hop.rtyper.set_type_for_typeptr(vtable, TYPE)
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -765,7 +765,11 @@
     def _ll_stringslice(s1, start, stop):
         lgt = stop - start
         assert start >= 0
-        assert lgt >= 0
+        # If start > stop, return a empty string. This can happen if the start
+        # is greater than the length of the string. Use < instead of <= to avoid
+        # creating another path for the JIT when start == stop.
+        if lgt < 0:
+            return s1.empty()
         newstr = s1.malloc(lgt)
         s1.copy_contents(s1, newstr, start, 0, lgt)
         return newstr
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -55,6 +55,7 @@
         vtup = hop.inputarg(self, 0)
         LIST = hop.r_result.lowleveltype.TO
         cno = inputconst(Signed, nitems)
+        hop.exception_is_here()
         vlist = hop.gendirectcall(LIST.ll_newlist, cno)
         v_func = hop.inputconst(Void, rlist.dum_nocheck)
         for index in range(nitems):
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1426,23 +1426,25 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
-        # If 'obj' was already forwarded, change it to its forwarding address.
-        if self.is_forwarded(obj):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
+            #
+            # Common case: 'obj' was not already forwarded (otherwise
+            # tid == -42, containing all flags), and it doesn't have the
+            # HAS_SHADOW flag either.  We must move it out of the nursery,
+            # into a new nonmovable location.
+            totalsize = size_gc_header + self.get_size(obj)
+            newhdr = self._malloc_out_of_nursery(totalsize)
+            #
+        elif self.is_forwarded(obj):
+            #
+            # 'obj' was already forwarded.  Change the original reference
+            # to point to its forwarding address, and we're done.
             root.address[0] = self.get_forwarding_address(obj)
             return
-        #
-        # First visit to 'obj': we must move it out of the nursery.
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        size = self.get_size(obj)
-        totalsize = size_gc_header + size
-        #
-        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
-            #
-            # Common case: allocate a new nonmovable location for it.
-            newhdr = self._malloc_out_of_nursery(totalsize)
             #
         else:
-            # The object has already a shadow.
+            # First visit to an object that has already a shadow.
             newobj = self.nursery_objects_shadows.get(obj)
             ll_assert(newobj != NULL, "GCFLAG_HAS_SHADOW but no shadow found")
             newhdr = newobj - size_gc_header
@@ -1450,6 +1452,8 @@
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
             # copied to the shadow itself.
             self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            #
+            totalsize = size_gc_header + self.get_size(obj)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
diff --git a/pypy/rpython/module/r_os_stat.py b/pypy/rpython/module/r_os_stat.py
--- a/pypy/rpython/module/r_os_stat.py
+++ b/pypy/rpython/module/r_os_stat.py
@@ -65,4 +65,5 @@
     r_StatResult = hop.rtyper.getrepr(ll_os_stat.s_StatResult)
     [v_result] = hop.inputargs(r_StatResult.r_tuple)
     # no-op conversion from r_StatResult.r_tuple to r_StatResult
+    hop.exception_cannot_occur()
     return v_result
diff --git a/pypy/rpython/ootypesystem/ooregistry.py b/pypy/rpython/ootypesystem/ooregistry.py
--- a/pypy/rpython/ootypesystem/ooregistry.py
+++ b/pypy/rpython/ootypesystem/ooregistry.py
@@ -22,6 +22,7 @@
                                          annmodel.SomeOOInstance,
                                          annmodel.SomeString))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oostring', vlist, resulttype = ootype.String)
 
 class Entry_oounicode(ExtRegistryEntry):
@@ -38,6 +39,7 @@
         assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint,
                                           annmodel.SomeOOInstance))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oounicode', vlist, resulttype = ootype.Unicode)
     
 
diff --git a/pypy/rpython/ootypesystem/rbuiltin.py b/pypy/rpython/ootypesystem/rbuiltin.py
--- a/pypy/rpython/ootypesystem/rbuiltin.py
+++ b/pypy/rpython/ootypesystem/rbuiltin.py
@@ -7,12 +7,14 @@
 from pypy.rpython.error import TyperError
 
 def rtype_new(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputargs(ootype.Void)
     return hop.genop('new', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_oonewarray(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputarg(ootype.Void, arg=0)
     vlength = hop.inputarg(ootype.Signed, arg=1)
@@ -20,23 +22,27 @@
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_null(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     TYPE = hop.args_s[0].const
     nullvalue = ootype.null(TYPE)
     return hop.inputconst(TYPE, nullvalue)
 
 def rtype_classof(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
     vlist = hop.inputargs(hop.args_r[0])
     return hop.genop('classof', vlist,
                      resulttype = ootype.Class)
 
 def rtype_subclassof(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr, rootype.ooclass_repr)
     return hop.genop('subclassof', vlist,
                      resulttype = ootype.Bool)
 
 def rtype_instanceof(hop):
+    hop.exception_cannot_occur()
     INSTANCE = hop.args_v[1].value
     v_inst = hop.inputarg(hop.args_r[0], arg=0)
     c_cls = hop.inputconst(ootype.Void, INSTANCE)
@@ -44,23 +50,27 @@
                      resulttype=ootype.Bool)
 
 def rtype_runtimenew(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr)
     return hop.genop('runtimenew', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_ooupcast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_oodowncast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_to_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOStaticMeth) or \
            isinstance(hop.args_s[0], annmodel.SomeOOClass) or \
            isinstance(hop.args_s[0].ootype, ootype.OOType)
@@ -68,12 +78,14 @@
     return hop.genop('cast_to_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_from_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.OOType)
     assert isinstance(hop.args_s[1], annmodel.SomeOOObject)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('cast_from_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(ootype.Bool, hop.s_result.const)
 
@@ -99,6 +111,7 @@
     return ootype.subclassof(c1, class_)
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     if hop.args_s[0].is_constant():
 ##        INSTANCE = hop.s_result.rtyper_makerepr(hop.rtyper).lowleveltype
 ##        v_instance = hop.inputconst(ootype.Void, INSTANCE)
diff --git a/pypy/rpython/ootypesystem/rstr.py b/pypy/rpython/ootypesystem/rstr.py
--- a/pypy/rpython/ootypesystem/rstr.py
+++ b/pypy/rpython/ootypesystem/rstr.py
@@ -222,6 +222,10 @@
         length = s.ll_strlen()
         if stop > length:
             stop = length
+        # If start > stop, return a empty string. This can happen if the start
+        # is greater than the length of the string.
+        if start > stop:
+            start = stop
         return s.ll_substring(start, stop-start)
 
     def ll_stringslice_minusone(s):
diff --git a/pypy/rpython/ootypesystem/rtuple.py b/pypy/rpython/ootypesystem/rtuple.py
--- a/pypy/rpython/ootypesystem/rtuple.py
+++ b/pypy/rpython/ootypesystem/rtuple.py
@@ -39,6 +39,7 @@
         RESULT = hop.r_result.lowleveltype
         c_resulttype = inputconst(ootype.Void, RESULT)
         c_length = inputconst(ootype.Signed, len(self.items_r))
+        hop.exception_is_here()
         if isinstance(RESULT, ootype.Array):
             v_list = hop.genop('oonewarray', [c_resulttype, c_length], resulttype=RESULT)
         else:
diff --git a/pypy/rpython/rbool.py b/pypy/rpython/rbool.py
--- a/pypy/rpython/rbool.py
+++ b/pypy/rpython/rbool.py
@@ -34,6 +34,7 @@
 
     def rtype_float(_, hop):
         vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
         return vlist[0]
 
 #
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -111,25 +111,32 @@
         raise TyperError("don't know about built-in function %r" % (
             self.builtinfunc,))
 
+    def _call(self, hop2, **kwds_i):
+        bltintyper = self.findbltintyper(hop2.rtyper)
+        hop2.llops._called_exception_is_here_or_cannot_occur = False
+        v_result = bltintyper(hop2, **kwds_i)
+        if not hop2.llops._called_exception_is_here_or_cannot_occur:
+            raise TyperError("missing hop.exception_cannot_occur() or "
+                             "hop.exception_is_here() in %s" % bltintyper)
+        return v_result
+
     def rtype_simple_call(self, hop):
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
-        return bltintyper(hop2)
+        return self._call(hop2)
 
     def rtype_call_args(self, hop):
         # calling a built-in function with keyword arguments:
         # mostly for rpython.objectmodel.hint()
         hop, kwds_i = call_args_expand(hop)
 
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
         hop2.r_s_popfirstarg()
         # the RPython-level keyword args are passed with an 'i_' prefix and
         # the corresponding value is an *index* in the hop2 arguments,
         # to be used with hop.inputarg(arg=..)
-        return bltintyper(hop2, **kwds_i)
+        return self._call(hop2, **kwds_i)
 
 
 class BuiltinMethodRepr(Repr):
@@ -198,6 +205,7 @@
 # ____________________________________________________________
 
 def rtype_builtin_bool(hop):
+    # not called any more?
     assert hop.nb_args == 1
     return hop.args_r[0].rtype_is_true(hop)
 
@@ -241,6 +249,7 @@
 
 def rtype_builtin_min(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_min, v1, v2)
 
 def ll_min(i1, i2):
@@ -250,6 +259,7 @@
 
 def rtype_builtin_max(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_max, v1, v2)
 
 def ll_max(i1, i2):
@@ -264,6 +274,7 @@
     pass
 
 def rtype_OSError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("OSError() should not be called with "
                          "a single argument")
@@ -274,6 +285,7 @@
         r_self.setfield(v_self, 'errno', v_errno, hop.llops)
 
 def rtype_WindowsError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("WindowsError() should not be called with "
                          "a single argument")
@@ -442,6 +454,7 @@
     assert hop.args_s[0].is_constant()
     TGT = hop.args_s[0].const
     v_type, v_value = hop.inputargs(lltype.Void, hop.args_r[1])
+    hop.exception_cannot_occur()
     return gen_cast(hop.llops, TGT, v_value)
 
 _cast_to_Signed = {
@@ -523,11 +536,13 @@
 
 def rtype_identity_hash(hop):
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('gc_identityhash', vlist, resulttype=lltype.Signed)
 
 def rtype_runtime_type_info(hop):
     assert isinstance(hop.args_r[0], rptr.PtrRepr)
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('runtime_type_info', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
@@ -558,6 +573,7 @@
 
 def rtype_raw_malloc(hop):
     v_size, = hop.inputargs(lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_malloc', [v_size], resulttype=llmemory.Address)
 
 def rtype_raw_malloc_usage(hop):
@@ -586,6 +602,7 @@
     if s_addr.is_null_address():
         raise TyperError("raw_memclear(x, n) where x is the constant NULL")
     v_list = hop.inputargs(llmemory.Address, lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_memclear', v_list)
 
 BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc
@@ -596,6 +613,7 @@
 
 def rtype_offsetof(hop):
     TYPE, field = hop.inputargs(lltype.Void, lltype.Void)
+    hop.exception_cannot_occur()
     return hop.inputconst(lltype.Signed,
                           llmemory.offsetof(TYPE.value, field.value))
 
@@ -605,6 +623,7 @@
 # non-gc objects
 
 def rtype_free_non_gc_object(hop):
+    hop.exception_cannot_occur()
     vinst, = hop.inputargs(hop.args_r[0])
     flavor = hop.args_r[0].gcflavor
     assert flavor != 'gc'
@@ -617,6 +636,7 @@
 # keepalive_until_here
 
 def rtype_keepalive_until_here(hop):
+    hop.exception_cannot_occur()
     for v in hop.args_v:
         hop.genop('keepalive', [v], resulttype=lltype.Void)
     return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/rfloat.py b/pypy/rpython/rfloat.py
--- a/pypy/rpython/rfloat.py
+++ b/pypy/rpython/rfloat.py
@@ -136,7 +136,10 @@
         hop.exception_cannot_occur()
         return hop.genop('cast_float_to_int', vlist, resulttype=Signed)
 
-    rtype_float = rtype_pos
+    def rtype_float(_, hop):
+        vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
+        return vlist[0]
 
     # version picked by specialisation based on which
     # type system rtyping is using, from <type_system>.ll_str module
diff --git a/pypy/rpython/rint.py b/pypy/rpython/rint.py
--- a/pypy/rpython/rint.py
+++ b/pypy/rpython/rint.py
@@ -310,6 +310,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_chr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_char', vlist, resulttype=Char)
 
     def rtype_unichr(_, hop):
@@ -317,6 +319,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_unichr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_unichar', vlist, resulttype=UniChar)
 
     def rtype_is_true(self, hop):
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -115,6 +115,7 @@
     def rtype_bltn_list(self, hop):
         v_lst = hop.inputarg(self, 0)
         cRESLIST = hop.inputconst(Void, hop.r_result.LIST)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_copy, cRESLIST, v_lst)
 
     def rtype_len(self, hop):
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -107,8 +107,10 @@
     if isinstance(hop.r_result, AbstractRangeRepr):
         if hop.r_result.step != 0:
             c_rng = hop.inputconst(Void, hop.r_result.RANGE)
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrange, c_rng, vstart, vstop)
         else:
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrangest, vstart, vstop, vstep)
     else:
         # cannot build a RANGE object, needs a real list
@@ -117,6 +119,7 @@
         if isinstance(ITEMTYPE, Ptr):
             ITEMTYPE = ITEMTYPE.TO
         cLIST = hop.inputconst(Void, ITEMTYPE)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_range2list, cLIST, vstart, vstop, vstep)
 
 rtype_builtin_xrange = rtype_builtin_range
@@ -212,4 +215,5 @@
                                      [v_index, v_item])
 
 def rtype_builtin_enumerate(hop):
+    hop.exception_cannot_occur()
     return hop.r_result.r_baseiter.newiter(hop)
diff --git a/pypy/rpython/rstr.py b/pypy/rpython/rstr.py
--- a/pypy/rpython/rstr.py
+++ b/pypy/rpython/rstr.py
@@ -288,6 +288,8 @@
 
     def rtype_unicode(self, hop):
         if hop.args_s[0].is_constant():
+            # convertion errors occur during annotation, so cannot any more:
+            hop.exception_cannot_occur()
             return hop.inputconst(hop.r_result, hop.s_result.const)
         repr = hop.args_r[0].repr
         v_str = hop.inputarg(repr, 0)
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -846,6 +846,7 @@
         return result
 
     def exception_is_here(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
@@ -861,6 +862,7 @@
         self.llops.llop_raising_exceptions = len(self.llops)
 
     def exception_cannot_occur(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
diff --git a/pypy/rpython/test/test_extregistry.py b/pypy/rpython/test/test_extregistry.py
--- a/pypy/rpython/test/test_extregistry.py
+++ b/pypy/rpython/test/test_extregistry.py
@@ -114,6 +114,7 @@
         _about_ = dummy_func
         s_result_annotation = annmodel.SomeInteger()
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             return hop.inputconst(lltype.Signed, 42)
 
     def func():
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1085,6 +1085,7 @@
                 return annmodel.SomeInteger()
             def specialize_call(self, hop):
                 [v_instance] = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.gendirectcall(ll_my_gethash, v_instance)
 
         def f(n):
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -477,7 +477,11 @@
             s1 = s[:3]
             s2 = s[3:]
             s3 = s[3:10]
-            return s1+s2 == s and s2+s1 == const('lohel') and s1+s3 == s
+            s4 = s[42:44]
+            return (s1+s2 == s and
+                    s2+s1 == const('lohel') and
+                    s1+s3 == s and
+                    s4 == const(''))
         res = self.interpret(fn, [0])
         assert res
 
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -38,7 +38,7 @@
 closed_heads.reverse()
 
 for head, branch in closed_heads:
-    print '\t', branch
+    print '\t', head, '\t', branch
 print
 print 'The branches listed above will be merged to "closed-branches".'
 print 'You need to run this script in a clean working copy where you'
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -7,10 +7,17 @@
 from pypy import conftest
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.translator.platform import platform as compiler
+from pypy.rlib.rarithmetic import is_emulated_long
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib.entrypoint import entrypoint, secondary_entrypoints
 from pypy.rpython.lltypesystem.lloperation import llop
 
+_MSVC = compiler.name == "msvc"
+_MINGW = compiler.name == "mingw32"
+_WIN32 = _MSVC or _MINGW
+_WIN64 = _WIN32 and is_emulated_long
+# XXX get rid of 'is_emulated_long' and have a real config here.
+
 class AbstractTestAsmGCRoot:
     # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved
     # instructions:
@@ -18,8 +25,8 @@
 
     @classmethod
     def make_config(cls):
-        if compiler.name == "msvc":
-            py.test.skip("all asmgcroot tests disabled for MSVC")
+        if _MSVC and _WIN64:
+            py.test.skip("all asmgcroot tests disabled for MSVC X64")
         from pypy.config.pypyoption import get_pypy_config
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
diff --git a/pypy/translator/c/src/float.h b/pypy/translator/c/src/float.h
--- a/pypy/translator/c/src/float.h
+++ b/pypy/translator/c/src/float.h
@@ -43,5 +43,6 @@
 #define OP_CAST_FLOAT_TO_LONGLONG(x,r) r = (long long)(x)
 #define OP_CAST_FLOAT_TO_ULONGLONG(x,r) r = (unsigned long long)(x)
 #define OP_CONVERT_FLOAT_BYTES_TO_LONGLONG(x,r) memcpy(&r, &x, sizeof(double))
+#define OP_CONVERT_LONGLONG_BYTES_TO_FLOAT(x,r) memcpy(&r, &x, sizeof(long long))
 #endif
 
diff --git a/pypy/translator/c/test/test_extfunc.py b/pypy/translator/c/test/test_extfunc.py
--- a/pypy/translator/c/test/test_extfunc.py
+++ b/pypy/translator/c/test/test_extfunc.py
@@ -919,4 +919,5 @@
             t, cbuilder = self.compile(does_stuff)
             data = cbuilder.cmdexec('')
             res = os.nice(0) + 3
+            if res > 19: res = 19    # xxx Linux specific, probably
             assert data.startswith('os.nice returned %d\n' % res)
diff --git a/pypy/translator/cli/dotnet.py b/pypy/translator/cli/dotnet.py
--- a/pypy/translator/cli/dotnet.py
+++ b/pypy/translator/cli/dotnet.py
@@ -459,6 +459,7 @@
             
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         assert hop.args_s[1].is_constant()
         TYPE = hop.args_s[1].const
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
@@ -507,6 +508,7 @@
 
     def specialize_call(self, hop):
         v_obj, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('same_as', [v_obj], hop.r_result.lowleveltype)
 
 def new_array(type, length):
@@ -608,6 +610,7 @@
 
     def specialize_call(self, hop):
         v_type, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('cli_typeof', [v_type], hop.r_result.lowleveltype)
 
 
@@ -626,6 +629,7 @@
         v_obj, = hop.inputargs(*hop.args_r)
         methodname = hop.args_r[0].methodname
         c_methodname = hop.inputconst(ootype.Void, methodname)
+        hop.exception_cannot_occur()
         return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype)
 
 
@@ -647,6 +651,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -668,6 +673,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -701,6 +707,7 @@
 
     def specialize_call(self, hop):
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype)
 
 
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -243,4 +243,5 @@
     'force_cast':               [PushAllArgs, CastPrimitive, StoreResult],
 
     'convert_float_bytes_to_longlong': jvm.PYPYDOUBLEBYTESTOLONG,
+    'convert_longlong_bytes_to_float': jvm.PYPYLONGBYTESTODOUBLE,
 })
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -942,6 +942,7 @@
 PYPYULONGTODOUBLE =     Method.s(jPyPy, 'ulong_to_double', (jLong,), jDouble)
 PYPYLONGBITWISENEGATE = Method.v(jPyPy, 'long_bitwise_negate', (jLong,), jLong)
 PYPYDOUBLEBYTESTOLONG = Method.v(jPyPy, 'pypy__float2longlong', (jDouble,), jLong)
+PYPYLONGBYTESTODOUBLE = Method.v(jPyPy, 'pypy__longlong2float', (jLong,), jDouble)
 PYPYSTRTOINT =          Method.v(jPyPy, 'str_to_int', (jString,), jInt)
 PYPYSTRTOUINT =         Method.v(jPyPy, 'str_to_uint', (jString,), jInt)
 PYPYSTRTOLONG =         Method.v(jPyPy, 'str_to_long', (jString,), jLong)

From noreply at buildbot.pypy.org  Fri Apr  6 09:02:43 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  6 Apr 2012 09:02:43 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: translation fixes
Message-ID: <20120406070243.8A6308208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54213:8664eeb9165d
Date: 2012-04-06 07:56 +0300
http://bitbucket.org/pypy/pypy/changeset/8664eeb9165d/

Log:	translation fixes

diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -20,8 +20,9 @@
         assert index == 0
         ll2ctypes.TLS.errno = value
 if os.name == 'nt':
-    post_include_bits =['''
+    separate_module_sources =['''
         /* Lifted completely from CPython 3.3 Modules/posix_module.c */
+        #include <malloc.h> /* for _msize */
         typedef struct {
             intptr_t osfhnd;
             char osfile;
@@ -72,12 +73,15 @@
             errno = EBADF;
             return 0;
         }
-    ''']
+    ''',]
+    export_symbols = ['_PyVerify_fd']
 else:
-    post_include_bits = []
+    separate_module_sources = []
+    export_symbols = []
 eci = ExternalCompilationInfo(
     includes=['errno.h','stdio.h'],
-    post_include_bits = post_include_bits,
+    separate_module_sources = separate_module_sources,
+    export_symbols = export_symbols,
 )
 
 _get_errno, _set_errno = CExternVariable(INT, 'errno', eci,
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -20,7 +20,7 @@
 #endif
 
 #ifdef MS_WINDOWS
-/*#include "src/winstuff.c"*/
+#include "src/winstuff.c"
 #endif
 
 #ifdef __GNUC__
diff --git a/pypy/translator/c/src/winstuff.c b/pypy/translator/c/src/winstuff.c
--- a/pypy/translator/c/src/winstuff.c
+++ b/pypy/translator/c/src/winstuff.c
@@ -27,7 +27,7 @@
 
 void pypy_Windows_startup(void)
 {
-#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
+#if 0 && defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
     /* Set CRT argument error handler */
     _set_invalid_parameter_handler(InvalidParameterHandler);
     /* turn off assertions within CRT in debug mode;

From noreply at buildbot.pypy.org  Fri Apr  6 09:02:44 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  6 Apr 2012 09:02:44 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix for clock_gettime
Message-ID: <20120406070244.BCB8B8208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54214:3b96efd18ff6
Date: 2012-04-06 10:00 +0300
http://bitbucket.org/pypy/pypy/changeset/3b96efd18ff6/

Log:	fix for clock_gettime

diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
--- a/pypy/module/__pypy__/interp_time.py
+++ b/pypy/module/__pypy__/interp_time.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import sys
+import os
 
 from pypy.interpreter.error import exception_from_errno
 from pypy.interpreter.gateway import unwrap_spec
@@ -7,11 +7,15 @@
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
+if os.name == 'nt':
+    libraries = []
+else:
+    libraries = ["rt"]
 
 class CConfig:
     _compilation_info_ = ExternalCompilationInfo(
         includes=["time.h"],
-        libraries=["rt"],
+        libraries=libraries,
     )
 
     HAS_CLOCK_GETTIME = rffi_platform.Has('clock_gettime')
@@ -22,11 +26,6 @@
     CLOCK_PROCESS_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_PROCESS_CPUTIME_ID")
     CLOCK_THREAD_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_THREAD_CPUTIME_ID")
 
-    TIMESPEC = rffi_platform.Struct("struct timespec", [
-        ("tv_sec", rffi.TIME_T),
-        ("tv_nsec", rffi.LONG),
-    ])
-
 cconfig = rffi_platform.configure(CConfig)
 
 HAS_CLOCK_GETTIME = cconfig["HAS_CLOCK_GETTIME"]
@@ -37,29 +36,33 @@
 CLOCK_PROCESS_CPUTIME_ID = cconfig["CLOCK_PROCESS_CPUTIME_ID"]
 CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"]
 
-TIMESPEC = cconfig["TIMESPEC"]
+if HAS_CLOCK_GETTIME:
+    TIMESPEC = lltype.Struct("struct timespec", 
+        ("tv_sec", rffi.TIME_T),
+        ("tv_nsec", rffi.LONG),
+    )
 
-c_clock_gettime = rffi.llexternal("clock_gettime",
-    [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
-    compilation_info=CConfig._compilation_info_, threadsafe=False
-)
-c_clock_getres = rffi.llexternal("clock_getres",
-    [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
-    compilation_info=CConfig._compilation_info_, threadsafe=False
-)
+    c_clock_gettime = rffi.llexternal("clock_gettime",
+        [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
+        compilation_info=CConfig._compilation_info_, threadsafe=False
+    )
+    c_clock_getres = rffi.llexternal("clock_getres",
+        [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
+        compilation_info=CConfig._compilation_info_, threadsafe=False
+    )
 
- at unwrap_spec(clk_id="c_int")
-def clock_gettime(space, clk_id):
-    with lltype.scoped_alloc(TIMESPEC) as tp:
-        ret = c_clock_gettime(clk_id, tp)
-        if ret != 0:
-            raise exception_from_errno(space, space.w_IOError)
-        return space.wrap(tp.c_tv_sec + tp.c_tv_nsec * 1e-9)
+    @unwrap_spec(clk_id="c_int")
+    def clock_gettime(space, clk_id):
+        with lltype.scoped_alloc(TIMESPEC) as tp:
+            ret = c_clock_gettime(clk_id, tp)
+            if ret != 0:
+                raise exception_from_errno(space, space.w_IOError)
+            return space.wrap(tp.c_tv_sec + tp.c_tv_nsec * 1e-9)
 
- at unwrap_spec(clk_id="c_int")
-def clock_getres(space, clk_id):
-    with lltype.scoped_alloc(TIMESPEC) as tp:
-        ret = c_clock_getres(clk_id, tp)
-        if ret != 0:
-            raise exception_from_errno(space, space.w_IOError)
-        return space.wrap(tp.c_tv_sec + tp.c_tv_nsec * 1e-9)
+    @unwrap_spec(clk_id="c_int")
+    def clock_getres(space, clk_id):
+        with lltype.scoped_alloc(TIMESPEC) as tp:
+            ret = c_clock_getres(clk_id, tp)
+            if ret != 0:
+                raise exception_from_errno(space, space.w_IOError)
+            return space.wrap(tp.c_tv_sec + tp.c_tv_nsec * 1e-9)

From pullrequests-noreply at bitbucket.org  Fri Apr  6 11:35:32 2012
From: pullrequests-noreply at bitbucket.org (Roberto De Ioris)
Date: Fri, 06 Apr 2012 09:35:32 -0000
Subject: [pypy-commit] [pypy/pypy] PyFile_FromFile,
 PySys_SetArgv and PyThreadState_New implementation (pull request
 #68)
In-Reply-To: <819aec1e6e3998acaa7b35b136186f8a@bitbucket.org>
References: <819aec1e6e3998acaa7b35b136186f8a@bitbucket.org>
Message-ID: <20120406093532.2950.65072@bitbucket05.managed.contegix.com>

Pull request #68 has been updated by Roberto De Ioris to include new changes.

https://bitbucket.org/pypy/pypy/pull-request/68/pyfile_fromfile-pysys_setargv-and

Title: PyFile_FromFile, PySys_SetArgv and PyThreadState_New implementation

Creator: Roberto De Ioris

PyFile_FromFile() is used by c apps to map a libc FILE object to a Python file object. The implementation make use of the pypy-specific file.fdopen function.  PySys_SetArgv is used to set sys.argv. Both Tested with latest uWSGI tip. PyThreadState_New is only a prototype, i suspect the different internal threading usage in PyPy will require a bigger effort.

Updated list of changes:

5b28c054a739 by Roberto De Ioris: "added PySys_SetArgv and PyThreadState_New"
7bd1b6f657a8 by Roberto De Ioris: "merge with official pypy"
d356d169e770 by Roberto De Ioris: "add fflush to cpyext pyfile implementation"
1f3baa6ce8b1 by rob... at mrsurr: "merge"
2afeef0e350b by Roberto De Ioris: "implemed PyFile_FromFile as cpyext"
7eb44aa590a2 by Roberto De Ioris: "added Py_Initialize,Py_Finalize,Py_SetPythonHome,Py_SetProgramName as cpyext"
4c1e64afbe9a by Roberto De Ioris: "added implementation of Py_Initialize"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Fri Apr  6 12:06:33 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri,  6 Apr 2012 12:06:33 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix for non-windows
Message-ID: <20120406100633.F34728208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54215:a6e9220f391d
Date: 2012-04-06 13:05 +0300
http://bitbucket.org/pypy/pypy/changeset/a6e9220f391d/

Log:	fix for non-windows

diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
--- a/pypy/module/__pypy__/interp_time.py
+++ b/pypy/module/__pypy__/interp_time.py
@@ -37,10 +37,13 @@
 CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"]
 
 if HAS_CLOCK_GETTIME:
-    TIMESPEC = lltype.Struct("struct timespec", 
+    #redo it for timespec
+    CConfig.TIMESPEC = rffi_platform.Struct("struct timespec", [
         ("tv_sec", rffi.TIME_T),
         ("tv_nsec", rffi.LONG),
-    )
+    ])
+    cconfig = rffi_platform.configure(CConfig)
+    TIMESPEC = cconfig['TIMESPEC']
 
     c_clock_gettime = rffi.llexternal("clock_gettime",
         [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,

From noreply at buildbot.pypy.org  Fri Apr  6 16:41:57 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Fri,  6 Apr 2012 16:41:57 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: draft for another
	py3k-status-update blog post
Message-ID: <20120406144157.D366446E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: extradoc
Changeset: r4178:9899005dd47d
Date: 2012-04-06 16:41 +0200
http://bitbucket.org/pypy/extradoc/changeset/9899005dd47d/

Log:	draft for another py3k-status-update blog post

diff --git a/blog/draft/py3k-activity-march.png b/blog/draft/py3k-activity-march.png
new file mode 100644
index 0000000000000000000000000000000000000000..23b5b2a2aa5a7284a06c01328c6c1b4561d2a063
GIT binary patch

[cut]

diff --git a/blog/draft/py3k-status-update-3.rst b/blog/draft/py3k-status-update-3.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/py3k-status-update-3.rst
@@ -0,0 +1,61 @@
+Py3k status update #3
+---------------------
+
+This is the third status update about my work on the `py3k branch`_, which I
+can work on thanks to all of the people who donated_ to the `py3k proposal`_.
+
+A lot of work has been done during the last month: as usual, the list of
+changes is too big to be reported in a detalied way, so this is just a summary
+of what happened.
+
+One of the most active areas was killing old and deprecated features. In
+particular, we killed support for the ``__cmp__`` special method and its
+counsins, the ``cmp`` builtin function and keyword argument for
+``list.sort()`` and ``sorted()``.  Killing is easy, but then you have to fix
+all the places which breaks because of this, including all the types which
+relied on ``__cmp__`` to be comparable,, fixing all the tests which tried to
+order objects which are no longer ordeable now, or implementing new behavior
+like forbidding calling ``hash()`` on objects which implement ``__eq__`` but
+not ``__hash__``.
+
+Among the other features, we killed lots of now-gone functions in the
+``operator`` module, the builtins ``apply()``, ``reduce()`` and ``buffer``,
+and the ``os.*`` functions to deal with temporary files, which has been
+deprecated in favour of the new ``tempfile`` module.
+
+The other topic which can't miss in a py3k status update is, as usual,
+string-vs-unicode. At this round, we fixed bugs in string formatting (in
+particular to teach ``format()`` to always use unicode strings) and various
+corner cases about when calling the (possibly overridden) ``__str__`` method
+on subclasses of ``str``. Believe me, you don't want to know the precise rules
+:-).
+
+Other features which we worked on and fixed tests include, but are not limited
+to, ``marshal``, ``hashlib``, ``zipimport``, ``_socket`` and ``itertools``,
+plus the habitual endless lists of tests which fail for shallow reasons such
+as the syntactic differences, ``int`` vs ``long``, ``range()`` vs
+``list(range())`` etc. As a result, the number of failing tests dropped_ from
+650 to 235: we are beginning to see the light at the end of the tunnel :-)
+
+Finally, we did some RPython fixes, so that it is possible again to translate
+PyPy in the py3k branch. However, the resuling binary is a strange beast which
+mixes python 2 and python 3 semantics, so it is unusable for anything but
+`showing friends how cool it's that`_.
+
+.. image:: py3k-activity-march.png
+
+I would like to underline that I was not alone in doing all this work. In
+particular, a lot of people joined the PyPy sprint at Pycon and worked on the
+branch, as you can clearly see in this activity graph. I would like to thank
+all who helped!
+
+XXX: benjamin, could you write a quick summary of what you did?
+
+cheers,
+Antonio
+
+.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html
+.. _`py3k proposal`: http://pypy.org/py3donate.html
+.. _`py3k branch`: https://bitbucket.org/pypy/pypy/src/py3k
+.. _`showing friends how cool it's that`: http://paste.pocoo.org/show/577006/
+.. _dropped: http://buildbot.pypy.org/summary?category=linux32&branch=py3k&recentrev=53956:3c8ac35c653a

From noreply at buildbot.pypy.org  Fri Apr  6 19:15:24 2012
From: noreply at buildbot.pypy.org (gutworth)
Date: Fri,  6 Apr 2012 19:15:24 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: mention what I did with syntax
Message-ID: <20120406171524.3F51346E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: extradoc
Changeset: r4179:976290076952
Date: 2012-04-06 13:15 -0400
http://bitbucket.org/pypy/extradoc/changeset/976290076952/

Log:	mention what I did with syntax

diff --git a/blog/draft/py3k-status-update-3.rst b/blog/draft/py3k-status-update-3.rst
--- a/blog/draft/py3k-status-update-3.rst
+++ b/blog/draft/py3k-status-update-3.rst
@@ -37,6 +37,12 @@
 ``list(range())`` etc. As a result, the number of failing tests dropped_ from
 650 to 235: we are beginning to see the light at the end of the tunnel :-)
 
+Benjamin finished implementing Python 3 syntax. Most of it was small cleanups
+and tweaks to be compatible with CPython such as making ``True`` and ``False``
+keywords and preventing ``. . .`` from being parsed as ``Ellipsis``. Larger
+features syntax additions included keyword only arguments and function
+annotations.
+
 Finally, we did some RPython fixes, so that it is possible again to translate
 PyPy in the py3k branch. However, the resuling binary is a strange beast which
 mixes python 2 and python 3 semantics, so it is unusable for anything but
@@ -49,10 +55,8 @@
 branch, as you can clearly see in this activity graph. I would like to thank
 all who helped!
 
-XXX: benjamin, could you write a quick summary of what you did?
-
 cheers,
-Antonio
+Antonio and Benjamin
 
 .. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html
 .. _`py3k proposal`: http://pypy.org/py3donate.html

From noreply at buildbot.pypy.org  Fri Apr  6 19:15:52 2012
From: noreply at buildbot.pypy.org (gutworth)
Date: Fri,  6 Apr 2012 19:15:52 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: kill extra word
Message-ID: <20120406171552.79CF346E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: extradoc
Changeset: r4180:ade4b357fc44
Date: 2012-04-06 13:15 -0400
http://bitbucket.org/pypy/extradoc/changeset/ade4b357fc44/

Log:	kill extra word

diff --git a/blog/draft/py3k-status-update-3.rst b/blog/draft/py3k-status-update-3.rst
--- a/blog/draft/py3k-status-update-3.rst
+++ b/blog/draft/py3k-status-update-3.rst
@@ -40,8 +40,7 @@
 Benjamin finished implementing Python 3 syntax. Most of it was small cleanups
 and tweaks to be compatible with CPython such as making ``True`` and ``False``
 keywords and preventing ``. . .`` from being parsed as ``Ellipsis``. Larger
-features syntax additions included keyword only arguments and function
-annotations.
+syntax additions included keyword only arguments and function annotations.
 
 Finally, we did some RPython fixes, so that it is possible again to translate
 PyPy in the py3k branch. However, the resuling binary is a strange beast which

From noreply at buildbot.pypy.org  Sat Apr  7 15:01:44 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat,  7 Apr 2012 15:01:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: bump the version number;
 this checkin will be cherry-picked in the release branch
Message-ID: <20120407130144.C1AEC8208A@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54216:8fafaa971ea2
Date: 2012-04-07 14:40 +0200
http://bitbucket.org/pypy/pypy/changeset/8fafaa971ea2/

Log:	bump the version number; this checkin will be cherry-picked in the
	release branch

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 2, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 9, 0, "final", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)

From noreply at buildbot.pypy.org  Sat Apr  7 15:01:46 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat,  7 Apr 2012 15:01:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: bump the version number to continue the
 development on default
Message-ID: <20120407130146.30B148208B@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54217:571bc0fbf7d1
Date: 2012-04-07 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/571bc0fbf7d1/

Log:	bump the version number to continue the development on default

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.9.0"
+#define PYPY_VERSION "1.9.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 2, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 9, 0, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 9, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)

From noreply at buildbot.pypy.org  Sat Apr  7 15:01:47 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat,  7 Apr 2012 15:01:47 +0200 (CEST)
Subject: [pypy-commit] pypy release-1.9.x: create a release branch
Message-ID: <20120407130147.BD005820C7@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: release-1.9.x
Changeset: r54218:77dd5ae278aa
Date: 2012-04-07 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/77dd5ae278aa/

Log:	create a release branch


From noreply at buildbot.pypy.org  Sat Apr  7 15:01:49 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat,  7 Apr 2012 15:01:49 +0200 (CEST)
Subject: [pypy-commit] pypy release-1.9.x: bump the version number;
 this checkin will be cherry-picked in the release branch
Message-ID: <20120407130149.1301D820D9@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: release-1.9.x
Changeset: r54219:82161531c3d5
Date: 2012-04-07 14:40 +0200
http://bitbucket.org/pypy/pypy/changeset/82161531c3d5/

Log:	bump the version number; this checkin will be cherry-picked in the
	release branch

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 2, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 9, 0, "final", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:09 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: typo
Message-ID: <20120407130409.428008208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54220:ab5974a8f615
Date: 2012-04-05 22:05 +0200
http://bitbucket.org/pypy/pypy/changeset/ab5974a8f615/

Log:	typo

diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -1,5 +1,5 @@
 ## ----------------------------------------------------------------------------
-## dict strategy (see dict_multiobject.py)
+## dict strategy (see dictmultiobject.py)
 
 from pypy.rlib import rerased
 from pypy.rlib.debug import mark_dict_non_null

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:10 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: remove inefficient implementation of
 IntDictStrategy.w_keys and rely on default
Message-ID: <20120407130410.915548208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54221:057f65a41c8e
Date: 2012-04-05 22:10 +0200
http://bitbucket.org/pypy/pypy/changeset/057f65a41c8e/

Log:	remove inefficient implementation of IntDictStrategy.w_keys and rely
	on default implementation

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -563,10 +563,7 @@
     def listview_int(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
 
-    def w_keys(self, w_dict):
-        # XXX there is no space.newlist_int yet
-        space = self.space
-        return space.call_function(space.w_list, w_dict)
+    # XXX there is no space.newlist_int yet to implement w_keys more efficiently
 
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:11 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: remvoe unused argument that was only
 passed in mostly disabled code path
Message-ID: <20120407130411.EFF008208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54222:4fa7e4646fd4
Date: 2012-04-05 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/4fa7e4646fd4/

Log:	remvoe unused argument that was only passed in mostly disabled code
	path

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,8 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, classofinstance=None,
-                                   strdict=False):
+                                   instance=False, strdict=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,10 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                classofinstance=classofinstance,
                 strdict=strdict)
 
     def newset(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -885,10 +885,9 @@
     def newtuple(self, l):
         return tuple(l)
 
-    def newdict(self, module=False, instance=False, classofinstance=None):
+    def newdict(self, module=False, instance=False):
         return W_DictMultiObject.allocate_and_init_instance(
-                self, module=module, instance=instance,
-                classofinstance=classofinstance)
+                self, module=module, instance=instance)
 
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:13 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: the keys method no longer exists
Message-ID: <20120407130413.48E3C8208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54223:02d0ae2e0c67
Date: 2012-04-05 22:45 +0200
http://bitbucket.org/pypy/pypy/changeset/02d0ae2e0c67/

Log:	the keys method no longer exists

diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -80,8 +80,8 @@
     def iter(self, w_dict):
         return IdentityDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class IdentityDictIteratorImplementation(_UnwrappedIteratorMixin, IteratorImplementation):

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:14 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: add a generic popitem test to the fake
 space test class as well
Message-ID: <20120407130414.AA1EB8208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54224:69d52bf3335c
Date: 2012-04-05 23:07 +0200
http://bitbucket.org/pypy/pypy/changeset/69d52bf3335c/

Log:	add a generic popitem test to the fake space test class as well

diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -967,6 +967,20 @@
         assert type(self.impl.strategy) is self.StrategyClass
         #assert self.impl.r_dict_content is None
 
+    def test_popitem(self):
+        self.fill_impl()
+        assert self.impl.length() == 2
+        a, b = self.impl.popitem()
+        assert self.impl.length() == 1
+        if a == self.string:
+            assert b == 1000
+            assert self.impl.getitem(self.string2) == 2000
+        else:
+            assert a == self.string2
+            assert b == 2000
+            assert self.impl.getitem_str(self.string) == 1000
+        self.check_not_devolved()
+
     def test_setitem(self):
         self.impl.setitem(self.string, 1000)
         assert self.impl.length() == 1

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:15 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:15 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: a branch to try to write a
 special strategy for kwarg-dicts
Message-ID: <20120407130415.EA8128208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54225:6fe24eaa5861
Date: 2012-04-05 23:10 +0200
http://bitbucket.org/pypy/pypy/changeset/6fe24eaa5861/

Log:	a branch to try to write a special strategy for kwarg-dicts


From noreply at buildbot.pypy.org  Sat Apr  7 15:04:17 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:17 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: add a kwargs dict strategy
 based on two lists
Message-ID: <20120407130417.459F08208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54226:3275a7dffd64
Date: 2012-04-05 23:20 +0200
http://bitbucket.org/pypy/pypy/changeset/3275a7dffd64/

Log:	add a kwargs dict strategy based on two lists

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,136 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.is_constant(self.length(w_dict)) and jit.is_constant(key))
+    def setitem_str(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            keys.append(key)
+            values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.is_constant(self.length(w_dict)) and jit.is_constant(key))
+    def getitem_str(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = [self.wrap(key) for key in self.unerase(w_dict.dstorage)[0]]
+        return self.space.newlist(l)
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        self.iterator = iter(range(len(strategy.unerase(dictimplementation.dstorage)[0])))
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        keys, values_w = self.strategy.unerase(self.dictimplementation.dstorage)
+        for i in self.iterator:
+            return self.space.wrap(keys[i]), values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,84 @@
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:18 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:18 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: make a kwargs dict when
 calling a function that takes **args
Message-ID: <20120407130418.926F88208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54227:0cfe5878d642
Date: 2012-04-05 23:22 +0200
http://bitbucket.org/pypy/pypy/changeset/0cfe5878d642/

Log:	make a kwargs dict when calling a function that takes **args

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -385,7 +385,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -82,3 +82,20 @@
     get_impl = get_impl
     StrategyClass = KwargsDictStrategy
 
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:19 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:19 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: limit size of the
	kwargsdict to (for now) 16
Message-ID: <20120407130419.ED7C88208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54228:dfac680a26f9
Date: 2012-04-06 10:43 +0200
http://bitbucket.org/pypy/pypy/changeset/dfac680a26f9/

Log:	limit size of the kwargsdict to (for now) 16

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -4,7 +4,8 @@
 from pypy.rlib import rerased, jit
 from pypy.objspace.std.dictmultiobject import (DictStrategy,
                                                IteratorImplementation,
-                                               ObjectDictStrategy)
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
 
 
 class KwargsDictStrategy(DictStrategy):
@@ -54,8 +55,13 @@
                 values_w[i] = w_value
                 break
         else:
-            keys.append(key)
-            values_w.append(w_value)
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
 
     def setdefault(self, w_dict, w_key, w_default):
         # XXX could do better, but is it worth it?
@@ -122,6 +128,17 @@
         w_dict.strategy = strategy
         w_dict.dstorage = strategy.erase(d_new)
 
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+
 class KwargsDictIterator(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
         IteratorImplementation.__init__(self, space, strategy, dictimplementation)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -67,6 +67,13 @@
     assert keys == ["a", "b", "c", "d"]
     assert values == [1, 2, 3, 4]
 
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
 
 from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
 def get_impl(self):

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:21 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:21 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: add a new space method
 view_as_kwargs which efficiently unwraps a KwargsDict and use it in the
 argument parser
Message-ID: <20120407130421.5182C8208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54229:166c64714bac
Date: 2012-04-06 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/166c64714bac/

Log:	add a new space method view_as_kwargs which efficiently unwraps a
	KwargsDict and use it in the argument parser

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -169,6 +169,10 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None:
+            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            return jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
             if not space.is_true(w_starstararg):
                 return False # don't call unpackiterable - it's jit-opaque
@@ -227,6 +231,26 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
+        if self.keywords is None:
+            self.keywords = keywords
+            self.keywords_w = keywords_w
+        else:
+            # looks quadratic, but the JIT should remove all of it nicely.
+            # Also, all the lists should be small
+            for key in keywords:
+                for otherkey in self.keywords:
+                    if otherkey == key:
+                        raise operationerrfmt(self.space.w_TypeError,
+                                              "got multiple values "
+                                              "for keyword argument "
+                                              "'%s'", key)
+            self.keywords = self.keywords + keywords
+            self.keywords_w = self.keywords_w + keywords_w
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -95,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -169,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -138,6 +138,9 @@
         w_dict.strategy = strategy
         w_dict.dstorage = storage
 
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
 
 class KwargsDictIterator(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -472,6 +472,12 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+        if type(w_obj) is W_DictMultiObject:
+            return w_obj.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:22 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:22 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: make empty dicts use this
	path as well
Message-ID: <20120407130422.9C3078208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54230:5bad845d3944
Date: 2012-04-06 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/5bad845d3944/

Log:	make empty dicts use this path as well

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -170,12 +170,10 @@
         # unpack the ** arguments
         space = self.space
         keywords, values_w = space.view_as_kwargs(w_starstararg)
-        if keywords is not None:
+        if keywords is not None: # this path also taken for empty dicts
             self._add_keywordargs_no_unwrapping(keywords, values_w)
             return jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -190,11 +188,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -262,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:23 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:23 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: translation fixes
Message-ID: <20120407130423.E19C88208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54231:065d2eb0c85e
Date: 2012-04-06 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/065d2eb0c85e/

Log:	translation fixes

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -46,7 +46,7 @@
             w_dict.setitem(w_key, w_value)
 
     @jit.look_inside_iff(lambda self, w_dict, key, w_value:
-            jit.is_constant(self.length(w_dict)) and jit.is_constant(key))
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
     def setitem_str(self, w_dict, key, w_value):
         keys, values_w = self.unerase(w_dict.dstorage)
         result = []
@@ -76,7 +76,7 @@
     def length(self, w_dict):
         return len(self.unerase(w_dict.dstorage)[0])
 
-    @jit.look_inside_iff(lambda self, w_dict, key: jit.is_constant(self.length(w_dict)) and jit.is_constant(key))
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
     def getitem_str(self, w_dict, key):
         keys, values_w = self.unerase(w_dict.dstorage)
         result = []
@@ -145,12 +145,15 @@
 class KwargsDictIterator(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
         IteratorImplementation.__init__(self, space, strategy, dictimplementation)
-        self.iterator = iter(range(len(strategy.unerase(dictimplementation.dstorage)[0])))
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
 
     def next_entry(self):
         # note that this 'for' loop only runs once, at most
-        keys, values_w = self.strategy.unerase(self.dictimplementation.dstorage)
         for i in self.iterator:
-            return self.space.wrap(keys[i]), values_w[i]
+            return self.space.wrap(self.keys[i]), self.values_w[i]
         else:
             return None, None

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:25 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:25 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: more translation fixes
Message-ID: <20120407130425.384128208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54232:6fce95c0e36e
Date: 2012-04-06 16:40 +0200
http://bitbucket.org/pypy/pypy/changeset/6fce95c0e36e/

Log:	more translation fixes

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -45,9 +45,12 @@
             self.switch_to_object_strategy(w_dict)
             w_dict.setitem(w_key, w_value)
 
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
     @jit.look_inside_iff(lambda self, w_dict, key, w_value:
             jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
-    def setitem_str(self, w_dict, key, w_value):
+    def _setitem_str_indirection(self, w_dict, key, w_value):
         keys, values_w = self.unerase(w_dict.dstorage)
         result = []
         for i in range(len(keys)):
@@ -76,8 +79,11 @@
     def length(self, w_dict):
         return len(self.unerase(w_dict.dstorage)[0])
 
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
     @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
-    def getitem_str(self, w_dict, key):
+    def _getitem_str_indirection(self, w_dict, key):
         keys, values_w = self.unerase(w_dict.dstorage)
         result = []
         for i in range(len(keys)):

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:26 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:26 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: - gah,
	actually enable view_as_kwargs
Message-ID: <20120407130426.821A48208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54233:1ce73e4161df
Date: 2012-04-07 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/1ce73e4161df/

Log:	- gah, actually enable view_as_kwargs
	- also, fix a resizing problem
	- fix logic when to look into argument matching

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -172,7 +172,7 @@
         keywords, values_w = space.view_as_kwargs(w_starstararg)
         if keywords is not None: # this path also taken for empty dicts
             self._add_keywordargs_no_unwrapping(keywords, values_w)
-            return jit.isconstant(len(self.keywords))
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
             keys_w = space.unpackiterable(w_starstararg)
         else:
@@ -231,8 +231,8 @@
             jit.isconstant(self.keywords)))
     def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
         if self.keywords is None:
-            self.keywords = keywords
-            self.keywords_w = keywords_w
+            self.keywords = keywords[:] # copy to make non-resizable
+            self.keywords_w = keywords_w[:]
         else:
             # looks quadratic, but the JIT should remove all of it nicely.
             # Also, all the lists should be small
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -473,9 +473,8 @@
         return None
 
     def view_as_kwargs(self, w_dict):
-        return (None, None)
-        if type(w_obj) is W_DictMultiObject:
-            return w_obj.view_as_kwargs()
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
         return (None, None)
 
     def _uses_list_iter(self, w_obj):

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:28 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:28 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: test that kwargs virtualize
	nicely
Message-ID: <20120407130428.3708A8208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54234:0019961d9e80
Date: 2012-04-07 14:58 +0200
http://bitbucket.org/pypy/pypy/changeset/0019961d9e80/

Log:	test that kwargs virtualize nicely

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+

From noreply at buildbot.pypy.org  Sat Apr  7 15:04:29 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sat,  7 Apr 2012 15:04:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120407130429.88E0C8208A@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54235:f016b819e011
Date: 2012-04-07 15:03 +0200
http://bitbucket.org/pypy/pypy/changeset/f016b819e011/

Log:	merge

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,8 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, classofinstance=None,
-                                   strdict=False):
+                                   instance=False, strdict=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -563,10 +562,7 @@
     def listview_int(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
 
-    def w_keys(self, w_dict):
-        # XXX there is no space.newlist_int yet
-        space = self.space
-        return space.call_function(space.w_list, w_dict)
+    # XXX there is no space.newlist_int yet to implement w_keys more efficiently
 
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -1,5 +1,5 @@
 ## ----------------------------------------------------------------------------
-## dict strategy (see dict_multiobject.py)
+## dict strategy (see dictmultiobject.py)
 
 from pypy.rlib import rerased
 from pypy.rlib.debug import mark_dict_non_null
@@ -80,8 +80,8 @@
     def iter(self, w_dict):
         return IdentityDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class IdentityDictIteratorImplementation(_UnwrappedIteratorMixin, IteratorImplementation):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,10 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                classofinstance=classofinstance,
                 strdict=strdict)
 
     def newset(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -885,10 +885,9 @@
     def newtuple(self, l):
         return tuple(l)
 
-    def newdict(self, module=False, instance=False, classofinstance=None):
+    def newdict(self, module=False, instance=False):
         return W_DictMultiObject.allocate_and_init_instance(
-                self, module=module, instance=instance,
-                classofinstance=classofinstance)
+                self, module=module, instance=instance)
 
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict
@@ -968,6 +967,20 @@
         assert type(self.impl.strategy) is self.StrategyClass
         #assert self.impl.r_dict_content is None
 
+    def test_popitem(self):
+        self.fill_impl()
+        assert self.impl.length() == 2
+        a, b = self.impl.popitem()
+        assert self.impl.length() == 1
+        if a == self.string:
+            assert b == 1000
+            assert self.impl.getitem(self.string2) == 2000
+        else:
+            assert a == self.string2
+            assert b == 2000
+            assert self.impl.getitem_str(self.string) == 1000
+        self.check_not_devolved()
+
     def test_setitem(self):
         self.impl.setitem(self.string, 1000)
         assert self.impl.length() == 1

From noreply at buildbot.pypy.org  Sat Apr  7 20:47:47 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat,  7 Apr 2012 20:47:47 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: Merged default in.
Message-ID: <20120407184747.2FD398208A@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54236:48aba87044b4
Date: 2012-04-07 14:47 -0400
http://bitbucket.org/pypy/pypy/changeset/48aba87044b4/

Log:	Merged default in.

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -968,8 +968,7 @@
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
+        return NotImplemented    # note that this doesn't work on CPython 2.2
 
     __radd__ = __add__
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -149,6 +149,22 @@
 exported.  This would give us a one-size-fits-all generic .so file to be
 imported by any application that wants to load .so files :-)
 
+Optimising cpyext (CPython C-API compatibility layer)
+-----------------------------------------------------
+
+A lot of work has gone into PyPy's implementation of CPython's C-API over
+the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However,
+there are still many edges and corner cases where it misbehaves, and it has
+not received any substantial optimisation so far.
+
+The objective of this project is to fix bugs in cpyext and to optimise
+several performance critical parts of it, such as the reference counting
+support and other heavily used C-API functions. The net result would be to
+have CPython extensions run much faster on PyPy than they currently do, or
+to make them work at all if they currently don't. A part of this work would
+be to get cpyext into a shape where it supports running Cython generated
+extensions.
 
 .. _`issue tracker`: http://bugs.pypy.org
 .. _`mailing list`: http://mail.python.org/mailman/listinfo/pypy-dev
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -199,17 +199,11 @@
 The following features (present in some past Stackless version of PyPy)
 are for the time being not supported any more:
 
-* Tasklets and channels (currently ``stackless.py`` seems to import,
-  but you have tasklets on top of coroutines on top of greenlets on
-  top of continulets on top of stacklets, and it's probably not too
-  hard to cut two of these levels by adapting ``stackless.py`` to
-  use directly continulets)
-
 * Coroutines (could be rewritten at app-level)
 
-* Pickling and unpickling continulets (*)
-
-* Continuing execution of a continulet in a different thread (*)
+* Continuing execution of a continulet in a different thread
+  (but if it is "simple enough", you can pickle it and unpickle it
+  in the other thread).
 
 * Automatic unlimited stack (must be emulated__ so far)
 
@@ -217,15 +211,6 @@
 
 .. __: `recursion depth limit`_
 
-(*) Pickling, as well as changing threads, could be implemented by using
-a "soft" stack switching mode again.  We would get either "hard" or
-"soft" switches, similarly to Stackless Python 3rd version: you get a
-"hard" switch (like now) when the C stack contains non-trivial C frames
-to save, and a "soft" switch (like previously) when it contains only
-simple calls from Python to Python.  Soft-switched continulets would
-also consume a bit less RAM, and the switch might be a bit faster too
-(unsure about that; what is the Stackless Python experience?).
-
 
 Recursion depth limit
 +++++++++++++++++++++
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -296,6 +296,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self.frame_trace_action = FrameTraceAction(self)
+        self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
         self.our_magic = default_magic
@@ -467,9 +468,9 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: time2 or rctime take precedence over time
+        # a bit of custom logic: rctime take precedence over time
         # XXX this could probably be done as a "requires" in the config
-        if ('time2' in modules or 'rctime' in modules) and 'time' in modules:
+        if 'rctime' in modules and 'time' in modules:
             modules.remove('time')
 
         if not self.config.objspace.nofaking:
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -154,6 +154,7 @@
         #operationerr.print_detailed_traceback(self.space)
 
     def _convert_exc(self, operr):
+        # Only for the flow object space
         return operr
 
     def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
@@ -166,6 +167,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -113,6 +113,12 @@
         from pypy.interpreter.pycode import PyCode
 
         code = self.getcode() # hook for the jit
+        #
+        if (jit.we_are_jitted() and code is self.space._code_of_sys_exc_info
+                                and nargs == 0):
+            from pypy.module.sys.vm import exc_info_direct
+            return exc_info_direct(self.space, frame)
+        #
         fast_natural_arity = code.fast_natural_arity
         if nargs == fast_natural_arity:
             if nargs == 0:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -874,6 +874,12 @@
             fn.add_to_table()
         if gateway.as_classmethod:
             fn = ClassMethod(space.wrap(fn))
+        #
+        from pypy.module.sys.vm import exc_info
+        if code._bltin is exc_info:
+            assert space._code_of_sys_exc_info is None
+            space._code_of_sys_exc_info = code
+        #
         return fn
 
 
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -2,34 +2,39 @@
 from pypy.interpreter import unicodehelper
 from pypy.rlib.rstring import StringBuilder
 
-def parsestr(space, encoding, s, unicode_literals=False):
-    # compiler.transformer.Transformer.decode_literal depends on what 
-    # might seem like minor details of this function -- changes here 
-    # must be reflected there.
+def parsestr(space, encoding, s, unicode_literal=False):
+    """Parses a string or unicode literal, and return a wrapped value.
+
+    If encoding=iso8859-1, the source string is also in this encoding.
+    If encoding=None, the source string is ascii only.
+    In other cases, the source string is in utf-8 encoding.
+
+    When a bytes string is returned, it will be encoded with the
+    original encoding.
+
+    Yes, it's very inefficient.
+    Yes, CPython has very similar code.
+    """
 
     # we use ps as "pointer to s"
     # q is the virtual last char index of the string
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = unicode_literals
 
     # string decoration handling
-    o = ord(quote)
-    isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
-    if isalpha or quote == '_':
-        if quote == 'b' or quote == 'B':
-            ps += 1
-            quote = s[ps]
-            unicode = False
-        elif quote == 'u' or quote == 'U':
-            ps += 1
-            quote = s[ps]
-            unicode = True
-        if quote == 'r' or quote == 'R':
-            ps += 1
-            quote = s[ps]
-            rawmode = True
+    if quote == 'b' or quote == 'B':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = False
+    elif quote == 'u' or quote == 'U':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = True
+    if quote == 'r' or quote == 'R':
+        ps += 1
+        quote = s[ps]
+        rawmode = True
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
                              'Internal error: parser passed unquoted literal')
@@ -46,21 +51,28 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
-    if unicode: # XXX Py_UnicodeFlag is ignored for now
+    if unicode_literal: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None or encoding == "iso-8859-1":
+            # 'unicode_escape' expects latin-1 bytes, string is ready.
             buf = s
             bufp = ps
             bufq = q
             u = None
         else:
-            # "\XX" may become "\u005c\uHHLL" (12 bytes)
+            # String is utf8-encoded, but 'unicode_escape' expects
+            # latin-1; So multibyte sequences must be escaped.
             lis = [] # using a list to assemble the value
             end = q
+            # Worst case: "\XX" may become "\u005c\uHHLL" (12 bytes)
             while ps < end:
                 if s[ps] == '\\':
                     lis.append(s[ps])
                     ps += 1
                     if ord(s[ps]) & 0x80:
+                        # A multibyte sequence will follow, it will be
+                        # escaped like \u1234. To avoid confusion with
+                        # the backslash we just wrote, we emit "\u005c"
+                        # instead.
                         lis.append("u005c")
                 if ord(s[ps]) & 0x80: # XXX inefficient
                     w, ps = decode_utf8(space, s, ps, end, "utf-16-be")
@@ -86,13 +98,11 @@
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
-    # XXX add strchr like interface to rtyper
     assert 0 <= ps <= q
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
             w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
-            #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1796,6 +1796,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1812,6 +1813,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -432,7 +432,8 @@
                     raise _ssl_seterror(self.space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return self.space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return self.space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                              length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -11,7 +11,7 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.objectmodel import specialize
+from pypy.rlib.objectmodel import specialize, keepalive_until_here
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
@@ -145,18 +145,24 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(RWBuffer):
-    def __init__(self, data, bytes):
-        self.data = data
-        self.len = bytes
+    def __init__(self, array):
+        self.array = array
 
     def getlength(self):
-        return self.len
+        return self.array.len * self.array.itemsize
 
     def getitem(self, index):
-        return self.data[index]
+        array = self.array
+        data = array._charbuf_start()
+        char = data[index]
+        array._charbuf_stop()
+        return char
 
     def setitem(self, index, char):
-        self.data[index] = char
+        array = self.array
+        data = array._charbuf_start()
+        data[index] = char
+        array._charbuf_stop()
 
 
 def make_array(mytype):
@@ -278,9 +284,10 @@
             oldlen = self.len
             new = len(s) / mytype.bytes
             self.setlen(oldlen + new)
-            cbuf = self.charbuf()
+            cbuf = self._charbuf_start()
             for i in range(len(s)):
                 cbuf[oldlen * mytype.bytes + i] = s[i]
+            self._charbuf_stop()
 
         def fromlist(self, w_lst):
             s = self.len
@@ -310,8 +317,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def charbuf(self):
-            return  rffi.cast(rffi.CCHARP, self.buffer)
+        def _charbuf_start(self):
+            return rffi.cast(rffi.CCHARP, self.buffer)
+
+        def _charbuf_stop(self):
+            keepalive_until_here(self)
 
         def w_getitem(self, space, idx):
             item = self.buffer[idx]
@@ -530,8 +540,10 @@
         self.fromstring(space.str_w(w_s))
 
     def array_tostring__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
+        self._charbuf_stop()
+        return self.space.wrap(s)
 
     def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
         if not isinstance(w_f, W_File):
@@ -613,8 +625,7 @@
     # Misc methods
 
     def buffer__Array(space, self):
-        b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes)
-        return space.wrap(b)
+        return space.wrap(ArrayBuffer(self))
 
     def array_buffer_info__Array(space, self):
         w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer))
@@ -649,7 +660,7 @@
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
-        bytes = self.charbuf()
+        bytes = self._charbuf_start()
         tmp = [bytes[0]] * mytype.bytes
         for start in range(0, self.len * mytype.bytes, mytype.bytes):
             stop = start + mytype.bytes - 1
@@ -657,6 +668,7 @@
                 tmp[i] = bytes[start + i]
             for i in range(mytype.bytes):
                 bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
 
     def repr__Array(space, self):
         if self.len == 0:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -433,7 +433,25 @@
         a = self.array('h', 'Hi')
         buf = buffer(a)
         assert buf[1] == 'i'
-        #raises(TypeError, buf.__setitem__, 1, 'o')
+
+    def test_buffer_write(self):
+        a = self.array('c', 'hello')
+        buf = buffer(a)
+        print repr(buf)
+        try:
+            buf[3] = 'L'
+        except TypeError:
+            skip("buffer(array) returns a read-only buffer on CPython")
+        assert a.tostring() == 'helLo'
+
+    def test_buffer_keepalive(self):
+        buf = buffer(self.array('c', 'text'))
+        assert buf[2] == 'x'
+        #
+        a = self.array('c', 'foobarbaz')
+        buf = buffer(a)
+        a.fromstring('some extra text')
+        assert buf[:] == 'foobarbazsome extra text'
 
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,8 +2,10 @@
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
     PyObjectFields, PyObject)
-from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
+from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
+from pypy.interpreter.error import OperationError
+from pypy.module.array.interp_array import ArrayBuffer
 
 
 PyBufferObjectStruct = lltype.ForwardReference()
@@ -41,26 +43,38 @@
         py_buf.c_b_offset = w_obj.offset
         w_obj = w_obj.buffer
 
+    # If w_obj already allocated a fixed buffer, use it, and keep a
+    # reference to w_obj.
+    # Otherwise, b_base stays NULL, and we own the b_ptr.
+
     if isinstance(w_obj, StringBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.as_str()))
+        py_buf.c_b_base = lltype.nullptr(PyObject.TO)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
+        py_buf.c_b_size = w_obj.getlength()
+    elif isinstance(w_obj, ArrayBuffer):
+        w_base = w_obj.array
+        py_buf.c_b_base = make_ref(space, w_base)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start())
         py_buf.c_b_size = w_obj.getlength()
     else:
-        raise Exception("Fail fail fail fail fail")
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "buffer flavor not supported"))
 
 
 def buffer_realize(space, py_obj):
     """
     Creates the buffer in the PyPy interpreter from a cpyext representation.
     """
-    raise Exception("realize fail fail fail")
-
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "Don't know how to realize a buffer"))
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
-    Py_DecRef(space, py_buf.c_b_base)
-    rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
+    if py_buf.c_b_base:
+        Py_DecRef(space, py_buf.c_b_base)
+    else:
+        rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -315,3 +316,65 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,14 +167,16 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -183,7 +185,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -48,3 +48,17 @@
             ])
         b = module.buffer_new()
         raises(AttributeError, getattr, b, 'x')
+
+    def test_array_buffer(self):
+        module = self.import_extension('foo', [
+            ("roundtrip", "METH_O",
+             """
+                 PyBufferObject *buf = (PyBufferObject *)args;
+                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
+             """),
+            ])
+        import array
+        a = array.array('c', 'text')
+        b = buffer(a)
+        assert module.roundtrip(b) == 'text'
+        
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
@@ -414,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -425,6 +429,36 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
+
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -307,6 +307,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -624,7 +626,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
@@ -99,9 +100,12 @@
         ("exp2", "exp2"),
         ("expm1", "expm1"),
         ("fabs", "fabs"),
+        ("fmax", "fmax"),
+        ("fmin", "fmin"),
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -122,12 +126,16 @@
         ("sinh", "sinh"),
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
+        ('square', 'square'),
         ("tan", "tan"),
         ("tanh", "tanh"),
         ('bitwise_and', 'bitwise_and'),
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
+        ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
         ('isneginf', 'isneginf'),
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -673,6 +673,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -1261,6 +1265,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1386,6 +1415,7 @@
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,9 +3,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype, support, loop
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
+
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
@@ -28,7 +30,7 @@
         return self.identity
 
     def descr_call(self, space, __args__):
-	from interp_numarray import BaseArray
+        from interp_numarray import BaseArray
         args_w, kwds_w = __args__.unpack()
         # it occurs to me that we don't support any datatypes that
         # require casting, change it later when we do
@@ -179,7 +181,7 @@
                 elif out.shape != shape:
                     raise operationerrfmt(space.w_ValueError,
                         'output parameter shape mismatch, expecting [%s]' +
-                        ' , got [%s]', 
+                        ' , got [%s]',
                         ",".join([str(x) for x in shape]),
                         ",".join([str(x) for x in out.shape]),
                         )
@@ -204,7 +206,7 @@
         else:
             arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
             val = loop.compute(arr)
-        return val 
+        return val
 
     def do_axis_reduce(self, obj, dtype, axis, result):
         from pypy.module.micronumpy.interp_numarray import AxisReduce
@@ -253,7 +255,7 @@
         if isinstance(w_obj, Scalar):
             arr = self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))
             if isinstance(out,Scalar):
-                out.value=arr
+                out.value = arr
             elif isinstance(out, BaseArray):
                 out.fill(space, arr)
             else:
@@ -265,7 +267,7 @@
             if not broadcast_shape or broadcast_shape != out.shape:
                 raise operationerrfmt(space.w_ValueError,
                     'output parameter shape mismatch, could not broadcast [%s]' +
-                    ' to [%s]', 
+                    ' to [%s]',
                     ",".join([str(x) for x in w_obj.shape]),
                     ",".join([str(x) for x in out.shape]),
                     )
@@ -292,10 +294,11 @@
         self.func = func
         self.comparison_func = comparison_func
 
+    @jit.unroll_safe
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
             convert_to_array, Scalar, shape_agreement, BaseArray)
-        if len(args_w)>2:
+        if len(args_w) > 2:
             [w_lhs, w_rhs, w_out] = args_w
         else:
             [w_lhs, w_rhs] = args_w
@@ -326,7 +329,7 @@
                 w_rhs.value.convert_to(calc_dtype)
             )
             if isinstance(out,Scalar):
-                out.value=arr
+                out.value = arr
             elif isinstance(out, BaseArray):
                 out.fill(space, arr)
             else:
@@ -337,7 +340,7 @@
         if out and out.shape != shape_agreement(space, new_shape, out.shape):
             raise operationerrfmt(space.w_ValueError,
                 'output parameter shape mismatch, could not broadcast [%s]' +
-                ' to [%s]', 
+                ' to [%s]',
                 ",".join([str(x) for x in new_shape]),
                 ",".join([str(x) for x in out.shape]),
                 )
@@ -347,7 +350,6 @@
         w_lhs.add_invalidates(w_res)
         w_rhs.add_invalidates(w_res)
         if out:
-            #out.add_invalidates(w_res) #causes a recursion loop
             w_res.get_concrete()
         return w_res
 
@@ -539,14 +541,18 @@
             ("reciprocal", "reciprocal", 1),
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
+            ("fmax", "fmax", 2, {"promote_to_float": True}),
+            ("fmin", "fmin", 2, {"promote_to_float": True}),
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
+            ("trunc", "trunc", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
             ("exp2", "exp2", 1, {"promote_to_float": True}),
             ("expm1", "expm1", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
+            ('square', 'square', 1, {'promote_to_float': True}),
 
             ("sin", "sin", 1, {"promote_to_float": True}),
             ("cos", "cos", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -107,6 +107,10 @@
         arr.compute_first_step(self, f)
         return f
 
+    def debug_repr(self):
+        # should be overridden, but in case it isn't, provide a default
+        return str(self)
+
 class ConcreteSignature(Signature):
     _immutable_fields_ = ['dtype']
 
@@ -207,7 +211,7 @@
     def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        transforms = transforms + [ViewTransform(arr.chunks)]
+        transforms = [ViewTransform(arr.chunks)] + transforms 
         self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
@@ -215,6 +219,9 @@
         assert isinstance(arr, VirtualSlice)
         return self.child.eval(frame, arr.child)
 
+    def debug_repr(self):
+        return 'VirtualSlice(%s)' % self.child.debug_repr()
+
 class Call1(Signature):
     _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
 
@@ -270,7 +277,7 @@
         from pypy.module.micronumpy.interp_numarray import Call1
 
         assert isinstance(arr, Call1)
-        vtransforms = transforms + [BroadcastTransform(arr.values.shape)]
+        vtransforms = [BroadcastTransform(arr.values.shape)] + transforms
         self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
         self.res._create_iter(iterlist, arraylist, arr.res, transforms)
 
@@ -348,7 +355,7 @@
         from pypy.module.micronumpy.interp_numarray import ResultArray
 
         assert isinstance(arr, ResultArray)
-        rtransforms = transforms + [BroadcastTransform(arr.left.shape)]
+        rtransforms = [BroadcastTransform(arr.left.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -375,7 +382,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
@@ -388,7 +395,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -401,8 +408,8 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -424,7 +431,7 @@
         frame.cur_value = self.binfunc(self.calc_dtype, frame.cur_value, rval)
 
     def debug_repr(self):
-        return 'ReduceSig(%s)' % (self.name, self.right.debug_repr())
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
 
 class SliceloopSignature(Call2):
     def eval(self, frame, arr):
@@ -448,7 +455,7 @@
         from pypy.module.micronumpy.interp_numarray import SliceArray
 
         assert isinstance(arr, SliceArray)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,6 +1,7 @@
 from pypy.rlib import jit
 from pypy.interpreter.error import OperationError
 
+ at jit.look_inside_iff(lambda chunks: jit.isconstant(len(chunks)))
 def enumerate_chunks(chunks):
     result = []
     i = -1
@@ -85,9 +86,9 @@
         space.isinstance_w(w_item_or_slice, space.w_slice)):
         raise OperationError(space.w_IndexError,
                              space.wrap('unsupported iterator index'))
-            
+
     start, stop, step, lngth = space.decode_index4(w_item_or_slice, size)
-        
+
     coords = [0] * len(shape)
     i = start
     if order == 'C':
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,5 +1,9 @@
+from pypy.rlib import jit
+
+
+ at jit.unroll_safe
 def product(s):
     i = 1
     for x in s:
         i *= x
-    return i
\ No newline at end of file
+    return i
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -395,11 +395,19 @@
         assert a[3] == 0.
 
     def test_newaxis(self):
-        from _numpypy import array
+        import math
+        from _numpypy import array, cos, zeros
         from numpypy.core.numeric import newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
+        a = array(range(3))
+        b = array([1, 3])
+        expected = zeros((3, 2))
+        for x in range(3):
+            for y in range(2):
+                expected[x, y] = math.cos(a[x]) * math.cos(b[y])
+        assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):
         from _numpypy import array
@@ -1338,6 +1346,10 @@
         dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0)
         assert str(dims_disagree.value) == \
             "array dimensions must agree except for axis being concatenated"
+        a = array([1, 2, 3, 4, 5, 6])
+        a = (a + a)[::2]
+        b = concatenate((a[:3], a[-3:]))
+        assert (b == [2, 6, 10, 2, 6, 10]).all()
 
     def test_std(self):
         from _numpypy import array
@@ -1387,6 +1399,16 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat, array
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -135,6 +135,38 @@
         assert fabs(float('-inf')) == float('inf')
         assert isnan(fabs(float('nan')))
 
+    def test_fmax(self):
+        from _numpypy import fmax
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmax(a, [ninf]*5) == a).all()
+        assert (fmax(a, [inf]*5) == [inf]*5).all()
+        assert (fmax(a, [1]*5) == [1, 1, 1, 5, inf]).all()
+        assert math.isnan(fmax(nan, 0))
+        assert math.isnan(fmax(0, nan))
+        assert math.isnan(fmax(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+
+    def test_fmin(self):
+        from _numpypy import fmin
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmin(a, [ninf]*5) == [ninf]*5).all()
+        assert (fmin(a, [inf]*5) == a).all()
+        assert (fmin(a, [1]*5) == [ninf, -5, 0, 1, 1]).all()
+        assert math.isnan(fmin(nan, 0))
+        assert math.isnan(fmin(0, nan))
+        assert math.isnan(fmin(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+
     def test_fmod(self):
         from _numpypy import fmod
         import math
@@ -221,24 +253,17 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floorceil(self):
-        from _numpypy import array, floor, ceil
+    def test_floorceiltrunc(self):
+        from _numpypy import array, floor, ceil, trunc
         import math
-        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = floor(a)
-        for i in range(5):
-            assert b[i] == reference[i]
-        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = ceil(a)
-        assert (reference == b).all()
-        inf = float("inf")
-        data = [1.5, 2.9999, -1.999, inf]
-        results = [math.floor(x) for x in data]
-        assert (floor(data) == results).all()
-        results = [math.ceil(x) for x in data]
-        assert (ceil(data) == results).all()
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == floor(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
+        assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -455,6 +480,19 @@
         assert math.isnan(sqrt(-1))
         assert math.isnan(sqrt(nan))
 
+    def test_square(self):
+        import math
+        from _numpypy import square
+
+        nan, inf, ninf = float("nan"), float("inf"), float("-inf")
+
+        assert math.isnan(square(nan))
+        assert math.isinf(square(inf))
+        assert math.isinf(square(ninf))
+        assert square(ninf) > 0
+        assert [square(x) for x in range(-5, 5)] == [x*x for x in range(-5, 5)]
+        assert math.isinf(square(1e300))
+
     def test_radians(self):
         import math
         from _numpypy import radians, array
@@ -546,10 +584,18 @@
         raises(TypeError, 'array([1.0]) & 1')
 
     def test_unary_bitops(self):
-        from _numpypy import bitwise_not, array
+        from _numpypy import bitwise_not, invert, array
         a = array([1, 2, 3, 4])
         assert (~a == [-2, -3, -4, -5]).all()
         assert (bitwise_not(a) == ~a).all()
+        assert (invert(a) == ~a).all()
+
+    def test_shift(self):
+        from _numpypy import left_shift, right_shift
+        import sys
+
+        assert (left_shift([5, 1], [2, 31]) == [20, 2**31]).all()
+        assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
 
     def test_comparisons(self):
         import operator
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -631,6 +631,22 @@
         return math.fabs(v)
 
     @simple_binary_op
+    def fmax(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return max(v1, v2)
+
+    @simple_binary_op
+    def fmin(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return min(v1, v2)
+
+    @simple_binary_op
     def fmod(self, v1, v2):
         try:
             return math.fmod(v1, v2)
@@ -652,6 +668,13 @@
         return math.ceil(v)
 
     @simple_unary_op
+    def trunc(self, v):
+        if v < 0:
+            return math.ceil(v)
+        else:
+            return math.floor(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -741,6 +764,10 @@
         except ValueError:
             return rfloat.NAN
 
+    @simple_unary_op
+    def square(self, v):
+        return v*v
+
     @raw_unary_op
     def isnan(self, v):
         return rfloat.isnan(v)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -212,7 +212,7 @@
             i19 = int_add(i12, 1)
             setfield_gc(p9, i19, descr=<FieldS .*W_AbstractSeqIterObject.inst_index .*>)
             guard_nonnull_class(p17, 146982464, descr=...)
-            i21 = getfield_gc(p17, descr=<FieldS .*W_ArrayTypei.inst_len .*>)
+            i21 = getfield_gc(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
             i24 = getfield_gc(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)
@@ -351,3 +351,23 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(log.loops_by_id("eval")) == 1
+
+    def test_sys_exc_info(self):
+        def main():
+            i = 1
+            lst = [i]
+            while i < 1000:
+                try:
+                    return lst[i]
+                except:
+                    e = sys.exc_info()[1]    # ID: exc_info
+                    if not isinstance(e, IndexError):
+                        raise
+                i += 1
+            return 42
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("exc_info")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -198,3 +198,37 @@
             i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>)
             guard_value(i49, 1, descr=...)
             ''')
+
+    def test_remove_duplicate_method_calls(self):
+        def main(n):
+            lst = []
+            for i in range(n):
+                s = 'Hello %d' % i
+                t = s.lower()   # ID: callone
+                u = s.lower()   # ID: calltwo
+                lst.append(t)
+                lst.append(u)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '''
+            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr . r EF=3>)
+            guard_no_exception(descr=...)
+            ''')
+        loop.match_by_id('calltwo', '')    # nothing
+
+    def test_move_method_call_out_of_loop(self):
+        def main(n):
+            lst = []
+            s = 'Hello %d' % n
+            for i in range(n):
+                t = s.lower()   # ID: callone
+                lst.append(t)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '')    # nothing
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -595,3 +595,124 @@
         assert len(frames) == 1
         _, other_frame = frames.popitem()
         assert other_frame.f_code.co_name in ('other_thread', '?')
+
+
+class AppTestSysExcInfoDirect:
+
+    def setup_method(self, meth):
+        self.checking = not option.runappdirect
+        if self.checking:
+            self.seen = []
+            from pypy.module.sys import vm
+            def exc_info_with_tb(*args):
+                self.seen.append("n")     # not optimized
+                return self.old[0](*args)
+            def exc_info_without_tb(*args):
+                self.seen.append("y")     # optimized
+                return self.old[1](*args)
+            self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
+            vm.exc_info_with_tb = exc_info_with_tb
+            vm.exc_info_without_tb = exc_info_without_tb
+            #
+            from pypy.rlib import jit
+            self.old2 = [jit.we_are_jitted]
+            jit.we_are_jitted = lambda: True
+
+    def teardown_method(self, meth):
+        if self.checking:
+            from pypy.module.sys import vm
+            from pypy.rlib import jit
+            vm.exc_info_with_tb = self.old[0]
+            vm.exc_info_without_tb = self.old[1]
+            jit.we_are_jitted = self.old2[0]
+            #
+            assert ''.join(self.seen) == meth.expected
+
+    def test_returns_none(self):
+        import sys
+        assert sys.exc_info() == (None, None, None)
+        assert sys.exc_info()[0] is None
+        assert sys.exc_info()[1] is None
+        assert sys.exc_info()[2] is None
+        assert sys.exc_info()[:2] == (None, None)
+        assert sys.exc_info()[:3] == (None, None, None)
+        assert sys.exc_info()[0:2] == (None, None)
+        assert sys.exc_info()[2:4] == (None,)
+    test_returns_none.expected = 'nnnnnnnn'
+
+    def test_returns_subscr(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert sys.exc_info()[0] is KeyError  # y
+            assert sys.exc_info()[1] is e         # y
+            assert sys.exc_info()[2] is not None  # n
+            assert sys.exc_info()[-3] is KeyError # y
+            assert sys.exc_info()[-2] is e        # y
+            assert sys.exc_info()[-1] is not None # n
+    test_returns_subscr.expected = 'yynyyn'
+
+    def test_returns_slice_2(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                  # n
+            assert sys.exc_info()[:0] == ()       # y
+            assert sys.exc_info()[:1] == foo[:1]  # y
+            assert sys.exc_info()[:2] == foo[:2]  # y
+            assert sys.exc_info()[:3] == foo      # n
+            assert sys.exc_info()[:4] == foo      # n
+            assert sys.exc_info()[:-1] == foo[:2] # y
+            assert sys.exc_info()[:-2] == foo[:1] # y
+            assert sys.exc_info()[:-3] == ()      # y
+    test_returns_slice_2.expected = 'nyyynnyyy'
+
+    def test_returns_slice_3(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                   # n
+            assert sys.exc_info()[2:2] == ()       # y
+            assert sys.exc_info()[0:1] == foo[:1]  # y
+            assert sys.exc_info()[1:2] == foo[1:2] # y
+            assert sys.exc_info()[0:3] == foo      # n
+            assert sys.exc_info()[2:4] == foo[2:]  # n
+            assert sys.exc_info()[0:-1] == foo[:2] # y
+            assert sys.exc_info()[0:-2] == foo[:1] # y
+            assert sys.exc_info()[5:-3] == ()      # y
+    test_returns_slice_3.expected = 'nyyynnyyy'
+
+    def test_strange_invocation(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            a = []; k = {}
+            assert sys.exc_info(*a)[:0] == ()
+            assert sys.exc_info(**k)[:0] == ()
+    test_strange_invocation.expected = 'nn'
+
+    def test_call_in_subfunction(self):
+        import sys
+        def g():
+            # this case is not optimized, because we need to search the
+            # frame chain.  it's probably not worth the complications
+            return sys.exc_info()[1]
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert g() is e
+    test_call_in_subfunction.expected = 'n'
+
+
+class AppTestSysExcInfoDirectCallMethod(AppTestSysExcInfoDirect):
+    def setup_class(cls):
+        cls.space = gettestobjspace(**{"objspace.opcodes.CALL_METHOD": True})
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 2, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 9, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -89,6 +89,9 @@
     """Return the (type, value, traceback) of the most recent exception
 caught by an except clause in the current stack frame or in an older stack
 frame."""
+    return exc_info_with_tb(space)    # indirection for the tests
+
+def exc_info_with_tb(space):
     operror = space.getexecutioncontext().sys_exc_info()
     if operror is None:
         return space.newtuple([space.w_None,space.w_None,space.w_None])
@@ -96,6 +99,59 @@
         return space.newtuple([operror.w_type, operror.get_w_value(space),
                                space.wrap(operror.get_traceback())])
 
+def exc_info_without_tb(space, frame):
+    operror = frame.last_exception
+    return space.newtuple([operror.w_type, operror.get_w_value(space),
+                           space.w_None])
+
+def exc_info_direct(space, frame):
+    from pypy.tool import stdlib_opcode
+    # In order to make the JIT happy, we try to return (exc, val, None)
+    # instead of (exc, val, tb).  We can do that only if we recognize
+    # the following pattern in the bytecode:
+    #       CALL_FUNCTION/CALL_METHOD         <-- invoking me
+    #       LOAD_CONST 0, 1, -2 or -3
+    #       BINARY_SUBSCR
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST <=2
+    #       SLICE_2
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST any integer
+    #       LOAD_CONST <=2
+    #       SLICE_3
+    need_all_three_args = True
+    co = frame.getcode().co_code
+    p = frame.last_instr
+    if (ord(co[p]) == stdlib_opcode.CALL_FUNCTION or
+        ord(co[p]) == stdlib_opcode.CALL_METHOD):
+        if ord(co[p+3]) == stdlib_opcode.LOAD_CONST:
+            lo = ord(co[p+4])
+            hi = ord(co[p+5])
+            w_constant = frame.getconstant_w((hi * 256) | lo)
+            if space.isinstance_w(w_constant, space.w_int):
+                constant = space.int_w(w_constant)
+                if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+                    if -3 <= constant <= 1 and constant != -1:
+                        need_all_three_args = False
+                elif ord(co[p+6]) == stdlib_opcode.SLICE+2:
+                    if constant <= 2:
+                        need_all_three_args = False
+                elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
+                      ord(co[p+9]) == stdlib_opcode.SLICE+3):
+                    lo = ord(co[p+7])
+                    hi = ord(co[p+8])
+                    w_constant = frame.getconstant_w((hi * 256) | lo)
+                    if space.isinstance_w(w_constant, space.w_int):
+                        if space.int_w(w_constant) <= 2:
+                            need_all_three_args = False
+    #
+    if need_all_three_args or frame.last_exception is None or frame.hide():
+        return exc_info_with_tb(space)
+    else:
+        return exc_info_without_tb(space, frame)
+
 def exc_clear(space):
     """Clear global information on the current exception.  Subsequent calls
 to exc_info() will return (None,None,None) until another exception is
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -44,3 +44,9 @@
     assert type(dt.microsecond) is int
 
     copy.copy(dt)
+
+def test_radd():
+    class X(object):
+        def __radd__(self, other):
+            return "radd"
+    assert datetime.date(10, 10, 10) + X() == "radd"
diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py
--- a/pypy/module/thread/test/test_ll_thread.py
+++ b/pypy/module/thread/test/test_ll_thread.py
@@ -66,7 +66,6 @@
     def test_gc_locking(self):
         import time
         from pypy.rlib.objectmodel import invoke_around_extcall
-        from pypy.rlib.objectmodel import we_are_translated
         from pypy.rlib.debug import ll_assert
 
         class State:
@@ -129,8 +128,6 @@
             state.finished = 0
             # the next line installs before_extcall() and after_extcall()
             # to be called automatically around external function calls.
-            # When not translated it does not work around time.sleep(),
-            # so we have to call them manually for this test.
             invoke_around_extcall(before_extcall, after_extcall)
 
             g(10, 1)
@@ -142,13 +139,9 @@
                 willing_to_wait_more -= 1
                 done = len(state.answers) == expected
 
-                if not we_are_translated(): before_extcall()
                 time.sleep(0.01)
-                if not we_are_translated(): after_extcall()
 
-            if not we_are_translated(): before_extcall()
             time.sleep(0.1)
-            if not we_are_translated(): after_extcall()
 
             return len(state.answers)
 
@@ -160,12 +153,11 @@
         answers = fn()
         assert answers == expected
 
-class TestRunDirectly(AbstractThreadTests):
-    def getcompiled(self, f, argtypes):
-        return f
-
-    def test_start_new_thread(self):
-        py.test.skip("deadlocks occasionally -- why???")
+#class TestRunDirectly(AbstractThreadTests):
+#    def getcompiled(self, f, argtypes):
+#        return f
+# These are disabled because they crash occasionally for bad reasons
+# related to the fact that ll2ctypes is not at all thread-safe
 
 class TestUsingBoehm(AbstractThreadTests):
     gcpolicy = 'boehm'
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -43,6 +43,13 @@
     return w_eq
 type_eq._annspecialcase_ = 'specialize:memo'
 
+def list_iter(space):
+    "Utility that returns the app-level descriptor list.__iter__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_list,
+                                               '__iter__')
+    return w_iter
+list_iter._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     w_type = space.type(w_obj)
     typename = w_type.getname(space)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -86,6 +86,7 @@
         return s_None
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 # ____________________________________________________________
@@ -109,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -163,7 +163,8 @@
 
 class ModuleDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         dict_w = strategy.unerase(dictimplementation.dstorage)
         self.iterator = dict_w.iteritems()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,8 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, classofinstance=None,
-                                   strdict=False):
+                                   instance=False, strdict=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -247,7 +246,7 @@
         return 0
 
     def iter(self, w_dict):
-        return EmptyIteratorImplementation(self.space, w_dict)
+        return EmptyIteratorImplementation(self.space, self, w_dict)
 
     def clear(self, w_dict):
         return
@@ -263,8 +262,9 @@
 # Iterator Implementation base classes
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.dictimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -280,7 +280,20 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.dictimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the dict.  The (key, value) pair in 'result'
+                # might be out-of-date.  We try to explicitly look up
+                # the key in the dict.
+                w_key = result[0]
+                w_value = self.dictimplementation.getitem(w_key)
+                if w_value is None:
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return (w_key, w_value)
         # no more entries
         self.dictimplementation = None
         return None, None
@@ -489,7 +502,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -503,7 +516,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -549,10 +562,7 @@
     def listview_int(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
 
-    def w_keys(self, w_dict):
-        # XXX there is no space.newlist_int yet
-        space = self.space
-        return space.call_function(space.w_list, w_dict)
+    # XXX there is no space.newlist_int yet to implement w_keys more efficiently
 
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -20,7 +20,17 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_w(w_lookup_type, space.w_str):
+        if (space.is_w(w_lookup_type, space.w_str) or  # Most common path first
+            space.abstract_issubclass_w(w_lookup_type, space.w_str)):
+            return self.getitem_str(w_dict, space.str_w(w_key))
+        elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode):
+            try:
+                w_key = space.str(w_key)
+            except OperationError, e:
+                if not e.match(space, space.w_UnicodeEncodeError):
+                    raise
+                # non-ascii unicode is never equal to a byte string
+                return None
             return self.getitem_str(w_dict, space.str_w(w_key))
         else:
             return None
@@ -98,7 +108,8 @@
 
 class DictProxyIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_type = strategy.unerase(dictimplementation.dstorage)
         self.iterator = w_type.dict_w.iteritems()
 
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -1,5 +1,5 @@
 ## ----------------------------------------------------------------------------
-## dict strategy (see dict_multiobject.py)
+## dict strategy (see dictmultiobject.py)
 
 from pypy.rlib import rerased
 from pypy.rlib.debug import mark_dict_non_null
@@ -80,8 +80,8 @@
     def iter(self, w_dict):
         return IdentityDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class IdentityDictIteratorImplementation(_UnwrappedIteratorMixin, IteratorImplementation):
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -703,7 +703,8 @@
 
 class MapDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_obj = strategy.unerase(dictimplementation.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,10 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                classofinstance=classofinstance,
                 strdict=strdict)
 
     def newset(self):
@@ -439,6 +438,8 @@
             t = w_obj.getitems()
         elif isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy(self)
+        elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            t = w_obj.getitems()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
         if expected_length != -1 and len(t) != expected_length:
@@ -456,6 +457,8 @@
             return w_obj.listview_str()
         if isinstance(w_obj, W_StringObject):
             return w_obj.listview_str()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_str()
         return None
 
     def listview_int(self, w_obj):
@@ -465,8 +468,14 @@
             return w_obj.listview_int()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_int()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_int()
         return None
 
+    def _uses_list_iter(self, w_obj):
+        from pypy.objspace.descroperation import list_iter
+        return self.lookup(w_obj, '__iter__') is list_iter(self)
+
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -359,7 +359,7 @@
         w_set.sstorage = w_other.get_storage_copy()
 
     def iter(self, w_set):
-        return EmptyIteratorImplementation(self.space, w_set)
+        return EmptyIteratorImplementation(self.space, self, w_set)
 
     def popitem(self, w_set):
         raise OperationError(self.space.w_KeyError,
@@ -784,8 +784,9 @@
             d_obj[w_item] = None
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.setimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -801,7 +802,17 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.setimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the set.  The 'result' might be out-of-date.
+                # We try to explicitly look it up in the set.
+                if not self.setimplementation.has_key(result):
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return result
         # no more entries
         self.setimplementation = None
         return None
@@ -823,7 +834,7 @@
 
 class StringIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, w_set):
-        IteratorImplementation.__init__(self, space, w_set)
+        IteratorImplementation.__init__(self, space, strategy, w_set)
         d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
@@ -835,9 +846,9 @@
 
 class IntegerIteratorImplementation(IteratorImplementation):
     #XXX same implementation in dictmultiobject on dictstrategy-branch
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
@@ -848,9 +859,9 @@
             return None
 
 class RDictIteratorImplementation(IteratorImplementation):
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -804,6 +804,33 @@
         assert "IntDictStrategy" in self.get_strategy(d)
         assert d[1L] == "hi"
 
+    def test_iter_dict_length_change(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d[7] = 8
+        # 'd' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_dict_strategy_only_change_1(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert d.get(Foo()) is None    # this changes the strategy of 'd'
+        lst = list(it)  # but iterating still works
+        assert sorted(lst) == [(1, 2), (3, 4), (5, 6)]
+
+    def test_iter_dict_strategy_only_change_2(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d['foo'] = 'bar'
+        del d[1]
+        # 'd' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us (1, 2), but 1 is not in the dict any longer.
+        raises(RuntimeError, list, it)
+
 
 class FakeString(str):
     hash_count = 0
@@ -858,10 +885,9 @@
     def newtuple(self, l):
         return tuple(l)
 
-    def newdict(self, module=False, instance=False, classofinstance=None):
+    def newdict(self, module=False, instance=False):
         return W_DictMultiObject.allocate_and_init_instance(
-                self, module=module, instance=instance,
-                classofinstance=classofinstance)
+                self, module=module, instance=instance)
 
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict
@@ -941,6 +967,20 @@
         assert type(self.impl.strategy) is self.StrategyClass
         #assert self.impl.r_dict_content is None
 
+    def test_popitem(self):
+        self.fill_impl()
+        assert self.impl.length() == 2
+        a, b = self.impl.popitem()
+        assert self.impl.length() == 1
+        if a == self.string:
+            assert b == 1000
+            assert self.impl.getitem(self.string2) == 2000
+        else:
+            assert a == self.string2
+            assert b == 2000
+            assert self.impl.getitem_str(self.string) == 1000
+        self.check_not_devolved()
+
     def test_setitem(self):
         self.impl.setitem(self.string, 1000)
         assert self.impl.length() == 1
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -25,6 +25,16 @@
         key, value = NotEmpty.__dict__.popitem()
         assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
 
+    def test_dictproxy_getitem(self):
+        class NotEmpty(object):
+            a = 1
+        assert 'a' in NotEmpty.__dict__
+        class substr(str): pass
+        assert substr('a') in NotEmpty.__dict__
+        assert u'a' in NotEmpty.__dict__
+        assert NotEmpty.__dict__[u'a'] == 1
+        assert u'\xe9' not in NotEmpty.__dict__
+
     def test_dictproxyeq(self):
         class a(object):
             pass
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1186,14 +1186,23 @@
         # of dicts, because the OrderedDict in the stdlib relies on this.
         # we extend the use case to lists and sets, i.e. all types that have
         # strategies, to avoid surprizes depending on the strategy.
-        for base, arg in [(list, []), (list, [5]), (list, ['x']),
-                          (set, []),  (set,  [5]), (set,  ['x']),
-                          (dict, []), (dict, [(5,6)]), (dict, [('x',7)])]:
+        class X: pass
+        for base, arg in [
+                (list, []), (list, [5]), (list, ['x']), (list, [X]),
+                (set, []),  (set,  [5]), (set,  ['x']), (set, [X]),
+                (dict, []), (dict, [(5,6)]), (dict, [('x',7)]), (dict, [(X,8)]),
+                ]:
             print base, arg
             class SubClass(base):
                 def __iter__(self):
                     return iter("foobar")
             assert list(SubClass(arg)) == ['f', 'o', 'o', 'b', 'a', 'r']
+            class Sub2(base):
+                pass
+            assert list(Sub2(arg)) == list(base(arg))
+            s = set()
+            s.update(Sub2(arg))
+            assert s == set(base(arg))
 
 class AppTestForRangeLists(AppTestW_ListObject):
 
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -907,3 +907,30 @@
                 return [5, 3, 4][i]
         s = set([10,3,2]).intersection(Obj())
         assert list(s) == [3]
+
+    def test_iter_set_length_change(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add(7)
+        # 's' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_set_strategy_only_change_1(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert Foo() not in s      # this changes the strategy of 'd'
+        lst = list(s)  # but iterating still works
+        assert sorted(lst) == [1, 3, 5]
+
+    def test_iter_set_strategy_only_change_2(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add('foo')
+        s.remove(1)
+        # 's' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us 1, but 1 is not in the set any longer.
+        raises(RuntimeError, list, it)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -382,7 +382,7 @@
         pass
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 vref_None = non_virtual_ref(None)
 
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -22,6 +22,7 @@
                 c_name = hop.inputconst(lltype.Void, 'access_helper')
                 args_v = [hop.inputarg(arg, arg=i)
                           for i, arg in enumerate(hop.args_r)]
+                hop.exception_cannot_occur()
                 return hop.genop('jit_marker', [c_name, c_func] + args_v,
                                  resulttype=hop.r_result)
         return helper
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -215,6 +215,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
@@ -397,6 +398,7 @@
         r_obj, = hop.args_r
         v_obj, = hop.inputargs(r_obj)
         ll_fn = r_obj.get_ll_hash_function()
+        hop.exception_is_here()
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
@@ -419,6 +421,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_identity_hash() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_identityhash', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -441,6 +444,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_id', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -452,6 +456,7 @@
 
     def specialize_call(self, hop):
         vobj, = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         if hop.rtyper.type_system.name == 'lltypesystem':
             from pypy.rpython.lltypesystem import lltype
             if isinstance(vobj.concretetype, lltype.Ptr):
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -85,7 +85,7 @@
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 
 class rbigint(object):
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -100,6 +100,7 @@
         def specialize_call(self, hop):
             bk = hop.rtyper.annotator.bookkeeper
             s_obj = identity.get_input_annotation(bk)
+            hop.exception_cannot_occur()
             return hop.r_result.rtype_erase(hop, s_obj)
 
     class Entry(ExtRegistryEntry):
@@ -110,6 +111,7 @@
             return identity.leave_tunnel(self.bookkeeper)
 
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
@@ -214,6 +216,7 @@
         return hop.genop('cast_opaque_ptr', [v], resulttype=hop.r_result)
 
     def rtype_unerase_int(self, hop, v):
+        hop.exception_cannot_occur()
         return hop.gendirectcall(ll_unerase_int, v)
 
     def rtype_erase_int(self, hop):
@@ -264,6 +267,7 @@
 
     def rtype_unerase_int(self, hop, v):
         c_one = hop.inputconst(lltype.Signed, 1)
+        hop.exception_cannot_occur()
         v2 = hop.genop('oounbox_int', [v], resulttype=hop.r_result)
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -382,6 +382,7 @@
     def compute_result_annotation(self):
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_roots', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
@@ -392,6 +393,7 @@
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_referents', vlist,
                          resulttype = hop.r_result)
 
@@ -402,6 +404,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_memory_usage', vlist,
                          resulttype = hop.r_result)
 
@@ -412,6 +415,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
@@ -430,6 +434,7 @@
         return annmodel.SomeBool()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_is_rpy_instance', vlist,
                          resulttype = hop.r_result)
 
@@ -449,6 +454,7 @@
         classrepr = getclassrepr(hop.rtyper, classdef)
         vtable = classrepr.getvtable()
         assert lltype.typeOf(vtable) == rclass.CLASSTYPE
+        hop.exception_cannot_occur()
         return Constant(vtable, concretetype=rclass.CLASSTYPE)
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rlib/rsre/rsre_re.py b/pypy/rlib/rsre/rsre_re.py
--- a/pypy/rlib/rsre/rsre_re.py
+++ b/pypy/rlib/rsre/rsre_re.py
@@ -172,8 +172,9 @@
         self._ctx = ctx
 
     def span(self, groupnum=0):
-        if not isinstance(groupnum, (int, long)):
-            groupnum = self.re.groupindex[groupnum]
+#        if not isinstance(groupnum, (int, long)):
+#            groupnum = self.re.groupindex[groupnum]
+ 
         return self._ctx.span(groupnum)
 
     def start(self, groupnum=0):
@@ -182,19 +183,25 @@
     def end(self, groupnum=0):
         return self.span(groupnum)[1]
 
-    def group(self, *groups):
-        groups = groups or (0,)
-        result = []
-        for group in groups:
-            frm, to = self.span(group)
-            if 0 <= frm <= to:
-                result.append(self._ctx._string[frm:to])
-            else:
-                result.append(None)
-        if len(result) > 1:
-            return tuple(result)
+    def group(self, group=0):
+        frm, to = self.span(group)
+        if 0 <= frm <= to:
+            return self._ctx._string[frm:to]
         else:
-            return result[0]
+            return None
+
+#    def group(self, *groups):
+#        groups = groups or (0,)
+#        result = []
+#        for group in groups:
+#            frm, to = self.span(group)
+#            if 0 <= frm <= to:
+#                result.append(self._ctx._string[frm:to])
+#            else:
+#                result.append(None)
+#        if len(result) > 1:
+#            return tuple(result)
+
 
     def groups(self, default=None):
         fmarks = self._ctx.flatten_marks()
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -204,7 +204,7 @@
         assert re.match('(a)', 'a').groups() == ('a',)
         assert re.match(r'(a)', 'a').group(0) == 'a'
         assert re.match(r'(a)', 'a').group(1) == 'a'
-        assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
+        #assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
 
         pat = re.compile('((a)|(b))(c)?')
         assert pat.match('a').groups() == ('a', 'a', None, None)
@@ -218,13 +218,13 @@
         assert m.group(0) == 'a'
         assert m.group(0) == 'a'
         assert m.group(1) == 'a'
-        assert m.group(1, 1) == ('a', 'a')
+        #assert m.group(1, 1) == ('a', 'a')
 
         pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?')
-        assert pat.match('a').group(1, 2, 3) == ('a', None, None)
-        assert pat.match('b').group('a1', 'b2', 'c3') == (
-                         (None, 'b', None))
-        assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
+        #assert pat.match('a').group(1, 2, 3) == ('a', None, None)
+        #assert pat.match('b').group('a1', 'b2', 'c3') == (
+        #                 (None, 'b', None))
+        #assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
     def test_bug_923(self):
         # Issue923: grouping inside optional lookahead problem
diff --git a/pypy/rlib/rsre/test/test_zinterp.py b/pypy/rlib/rsre/test/test_zinterp.py
--- a/pypy/rlib/rsre/test/test_zinterp.py
+++ b/pypy/rlib/rsre/test/test_zinterp.py
@@ -1,7 +1,8 @@
 # minimal test: just checks that (parts of) rsre can be translated
 
-from pypy.rpython.test.test_llinterp import gengraph
+from pypy.rpython.test.test_llinterp import gengraph, interpret
 from pypy.rlib.rsre import rsre_core
+from pypy.rlib.rsre.rsre_re import compile
 
 def main(n):
     assert n >= 0
@@ -19,3 +20,18 @@
 
 def test_gengraph():
     t, typer, graph = gengraph(main, [int])
+
+m = compile("(a|b)aaaaa")
+
+def test_match():
+    def f(i):
+        if i:
+            s = "aaaaaa"
+        else:
+            s = "caaaaa"
+        g = m.match(s)
+        if g is None:
+            return 3
+        return int("aaaaaa" == g.group(0))
+    assert interpret(f, [3]) == 1
+    assert interpret(f, [0]) == 3
diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py
--- a/pypy/rlib/rstring.py
+++ b/pypy/rlib/rstring.py
@@ -245,5 +245,5 @@
             raise ValueError("Value is not no_nul")
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py
--- a/pypy/rpython/annlowlevel.py
+++ b/pypy/rpython/annlowlevel.py
@@ -543,11 +543,11 @@
         else:
             assert False
 
+        hop.exception_cannot_occur()
         if isinstance(hop.args_r[1], rpbc.NoneFrozenPBCRepr):
             return hop.inputconst(PTR, null)
         v_arg = hop.inputarg(hop.args_r[1], arg=1)
         assert isinstance(v_arg.concretetype, T)
-        hop.exception_cannot_occur()
         return hop.genop(opname, [v_arg], resulttype = PTR)
 
 
diff --git a/pypy/rpython/controllerentry.py b/pypy/rpython/controllerentry.py
--- a/pypy/rpython/controllerentry.py
+++ b/pypy/rpython/controllerentry.py
@@ -201,6 +201,7 @@
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
         assert hop.s_result.is_constant()
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -130,6 +130,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 def enum_ops_without_sideeffects(raising_is_ok=False):
diff --git a/pypy/rpython/lltypesystem/rbuiltin.py b/pypy/rpython/lltypesystem/rbuiltin.py
--- a/pypy/rpython/lltypesystem/rbuiltin.py
+++ b/pypy/rpython/lltypesystem/rbuiltin.py
@@ -9,6 +9,7 @@
 from pypy.rpython.rbool import bool_repr
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr or hop.args_r[1] == pyobj_repr:
@@ -33,6 +34,7 @@
     return my_instantiate()
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     s_class = hop.args_s[0]
     assert isinstance(s_class, annmodel.SomePBC)
     if len(s_class.descriptions) != 1:
@@ -46,6 +48,7 @@
     return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops)
 
 def rtype_builtin_hasattr(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr:
@@ -56,6 +59,7 @@
     raise TyperError("hasattr is only suported on a constant or on PyObject")
 
 def rtype_builtin___import__(hop):
+    xxx    # should not be used any more
     args_v = hop.inputargs(*[pyobj_repr for ign in hop.args_r])
     c = hop.inputconst(pyobj_repr, __import__)
     return hop.genop('simple_call', [c] + args_v, resulttype = pyobj_repr)
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -746,4 +746,5 @@
         assert isinstance(TYPE, GcStruct)
         assert lltype._castdepth(TYPE, OBJECT) > 0
         hop.rtyper.set_type_for_typeptr(vtable, TYPE)
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -765,7 +765,11 @@
     def _ll_stringslice(s1, start, stop):
         lgt = stop - start
         assert start >= 0
-        assert lgt >= 0
+        # If start > stop, return a empty string. This can happen if the start
+        # is greater than the length of the string. Use < instead of <= to avoid
+        # creating another path for the JIT when start == stop.
+        if lgt < 0:
+            return s1.empty()
         newstr = s1.malloc(lgt)
         s1.copy_contents(s1, newstr, start, 0, lgt)
         return newstr
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -55,6 +55,7 @@
         vtup = hop.inputarg(self, 0)
         LIST = hop.r_result.lowleveltype.TO
         cno = inputconst(Signed, nitems)
+        hop.exception_is_here()
         vlist = hop.gendirectcall(LIST.ll_newlist, cno)
         v_func = hop.inputconst(Void, rlist.dum_nocheck)
         for index in range(nitems):
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1426,23 +1426,25 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
-        # If 'obj' was already forwarded, change it to its forwarding address.
-        if self.is_forwarded(obj):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
+            #
+            # Common case: 'obj' was not already forwarded (otherwise
+            # tid == -42, containing all flags), and it doesn't have the
+            # HAS_SHADOW flag either.  We must move it out of the nursery,
+            # into a new nonmovable location.
+            totalsize = size_gc_header + self.get_size(obj)
+            newhdr = self._malloc_out_of_nursery(totalsize)
+            #
+        elif self.is_forwarded(obj):
+            #
+            # 'obj' was already forwarded.  Change the original reference
+            # to point to its forwarding address, and we're done.
             root.address[0] = self.get_forwarding_address(obj)
             return
-        #
-        # First visit to 'obj': we must move it out of the nursery.
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        size = self.get_size(obj)
-        totalsize = size_gc_header + size
-        #
-        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
-            #
-            # Common case: allocate a new nonmovable location for it.
-            newhdr = self._malloc_out_of_nursery(totalsize)
             #
         else:
-            # The object has already a shadow.
+            # First visit to an object that has already a shadow.
             newobj = self.nursery_objects_shadows.get(obj)
             ll_assert(newobj != NULL, "GCFLAG_HAS_SHADOW but no shadow found")
             newhdr = newobj - size_gc_header
@@ -1450,6 +1452,8 @@
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
             # copied to the shadow itself.
             self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            #
+            totalsize = size_gc_header + self.get_size(obj)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -455,6 +455,6 @@
     return intmask(time), intmask(nsec)
 
 def time_t_to_FILE_TIME(time, filetime):
-    ft = (rffi.r_longlong(time) + secs_between_epochs) * 10000000
+    ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
     filetime.c_dwLowDateTime = rffi.r_uint(ft)    # masking off high bits
diff --git a/pypy/rpython/module/r_os_stat.py b/pypy/rpython/module/r_os_stat.py
--- a/pypy/rpython/module/r_os_stat.py
+++ b/pypy/rpython/module/r_os_stat.py
@@ -65,4 +65,5 @@
     r_StatResult = hop.rtyper.getrepr(ll_os_stat.s_StatResult)
     [v_result] = hop.inputargs(r_StatResult.r_tuple)
     # no-op conversion from r_StatResult.r_tuple to r_StatResult
+    hop.exception_cannot_occur()
     return v_result
diff --git a/pypy/rpython/ootypesystem/ooregistry.py b/pypy/rpython/ootypesystem/ooregistry.py
--- a/pypy/rpython/ootypesystem/ooregistry.py
+++ b/pypy/rpython/ootypesystem/ooregistry.py
@@ -22,6 +22,7 @@
                                          annmodel.SomeOOInstance,
                                          annmodel.SomeString))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oostring', vlist, resulttype = ootype.String)
 
 class Entry_oounicode(ExtRegistryEntry):
@@ -38,6 +39,7 @@
         assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint,
                                           annmodel.SomeOOInstance))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oounicode', vlist, resulttype = ootype.Unicode)
     
 
diff --git a/pypy/rpython/ootypesystem/rbuiltin.py b/pypy/rpython/ootypesystem/rbuiltin.py
--- a/pypy/rpython/ootypesystem/rbuiltin.py
+++ b/pypy/rpython/ootypesystem/rbuiltin.py
@@ -7,12 +7,14 @@
 from pypy.rpython.error import TyperError
 
 def rtype_new(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputargs(ootype.Void)
     return hop.genop('new', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_oonewarray(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputarg(ootype.Void, arg=0)
     vlength = hop.inputarg(ootype.Signed, arg=1)
@@ -20,23 +22,27 @@
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_null(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     TYPE = hop.args_s[0].const
     nullvalue = ootype.null(TYPE)
     return hop.inputconst(TYPE, nullvalue)
 
 def rtype_classof(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
     vlist = hop.inputargs(hop.args_r[0])
     return hop.genop('classof', vlist,
                      resulttype = ootype.Class)
 
 def rtype_subclassof(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr, rootype.ooclass_repr)
     return hop.genop('subclassof', vlist,
                      resulttype = ootype.Bool)
 
 def rtype_instanceof(hop):
+    hop.exception_cannot_occur()
     INSTANCE = hop.args_v[1].value
     v_inst = hop.inputarg(hop.args_r[0], arg=0)
     c_cls = hop.inputconst(ootype.Void, INSTANCE)
@@ -44,23 +50,27 @@
                      resulttype=ootype.Bool)
 
 def rtype_runtimenew(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr)
     return hop.genop('runtimenew', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_ooupcast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_oodowncast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_to_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOStaticMeth) or \
            isinstance(hop.args_s[0], annmodel.SomeOOClass) or \
            isinstance(hop.args_s[0].ootype, ootype.OOType)
@@ -68,12 +78,14 @@
     return hop.genop('cast_to_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_from_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.OOType)
     assert isinstance(hop.args_s[1], annmodel.SomeOOObject)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('cast_from_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(ootype.Bool, hop.s_result.const)
 
@@ -99,6 +111,7 @@
     return ootype.subclassof(c1, class_)
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     if hop.args_s[0].is_constant():
 ##        INSTANCE = hop.s_result.rtyper_makerepr(hop.rtyper).lowleveltype
 ##        v_instance = hop.inputconst(ootype.Void, INSTANCE)
diff --git a/pypy/rpython/ootypesystem/rstr.py b/pypy/rpython/ootypesystem/rstr.py
--- a/pypy/rpython/ootypesystem/rstr.py
+++ b/pypy/rpython/ootypesystem/rstr.py
@@ -222,6 +222,10 @@
         length = s.ll_strlen()
         if stop > length:
             stop = length
+        # If start > stop, return a empty string. This can happen if the start
+        # is greater than the length of the string.
+        if start > stop:
+            start = stop
         return s.ll_substring(start, stop-start)
 
     def ll_stringslice_minusone(s):
diff --git a/pypy/rpython/ootypesystem/rtuple.py b/pypy/rpython/ootypesystem/rtuple.py
--- a/pypy/rpython/ootypesystem/rtuple.py
+++ b/pypy/rpython/ootypesystem/rtuple.py
@@ -39,6 +39,7 @@
         RESULT = hop.r_result.lowleveltype
         c_resulttype = inputconst(ootype.Void, RESULT)
         c_length = inputconst(ootype.Signed, len(self.items_r))
+        hop.exception_is_here()
         if isinstance(RESULT, ootype.Array):
             v_list = hop.genop('oonewarray', [c_resulttype, c_length], resulttype=RESULT)
         else:
diff --git a/pypy/rpython/rbool.py b/pypy/rpython/rbool.py
--- a/pypy/rpython/rbool.py
+++ b/pypy/rpython/rbool.py
@@ -34,6 +34,7 @@
 
     def rtype_float(_, hop):
         vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
         return vlist[0]
 
 #
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -111,25 +111,32 @@
         raise TyperError("don't know about built-in function %r" % (
             self.builtinfunc,))
 
+    def _call(self, hop2, **kwds_i):
+        bltintyper = self.findbltintyper(hop2.rtyper)
+        hop2.llops._called_exception_is_here_or_cannot_occur = False
+        v_result = bltintyper(hop2, **kwds_i)
+        if not hop2.llops._called_exception_is_here_or_cannot_occur:
+            raise TyperError("missing hop.exception_cannot_occur() or "
+                             "hop.exception_is_here() in %s" % bltintyper)
+        return v_result
+
     def rtype_simple_call(self, hop):
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
-        return bltintyper(hop2)
+        return self._call(hop2)
 
     def rtype_call_args(self, hop):
         # calling a built-in function with keyword arguments:
         # mostly for rpython.objectmodel.hint()
         hop, kwds_i = call_args_expand(hop)
 
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
         hop2.r_s_popfirstarg()
         # the RPython-level keyword args are passed with an 'i_' prefix and
         # the corresponding value is an *index* in the hop2 arguments,
         # to be used with hop.inputarg(arg=..)
-        return bltintyper(hop2, **kwds_i)
+        return self._call(hop2, **kwds_i)
 
 
 class BuiltinMethodRepr(Repr):
@@ -198,6 +205,7 @@
 # ____________________________________________________________
 
 def rtype_builtin_bool(hop):
+    # not called any more?
     assert hop.nb_args == 1
     return hop.args_r[0].rtype_is_true(hop)
 
@@ -241,6 +249,7 @@
 
 def rtype_builtin_min(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_min, v1, v2)
 
 def ll_min(i1, i2):
@@ -250,6 +259,7 @@
 
 def rtype_builtin_max(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_max, v1, v2)
 
 def ll_max(i1, i2):
@@ -264,6 +274,7 @@
     pass
 
 def rtype_OSError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("OSError() should not be called with "
                          "a single argument")
@@ -274,6 +285,7 @@
         r_self.setfield(v_self, 'errno', v_errno, hop.llops)
 
 def rtype_WindowsError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("WindowsError() should not be called with "
                          "a single argument")
@@ -442,6 +454,7 @@
     assert hop.args_s[0].is_constant()
     TGT = hop.args_s[0].const
     v_type, v_value = hop.inputargs(lltype.Void, hop.args_r[1])
+    hop.exception_cannot_occur()
     return gen_cast(hop.llops, TGT, v_value)
 
 _cast_to_Signed = {
@@ -523,11 +536,13 @@
 
 def rtype_identity_hash(hop):
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('gc_identityhash', vlist, resulttype=lltype.Signed)
 
 def rtype_runtime_type_info(hop):
     assert isinstance(hop.args_r[0], rptr.PtrRepr)
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('runtime_type_info', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
@@ -558,6 +573,7 @@
 
 def rtype_raw_malloc(hop):
     v_size, = hop.inputargs(lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_malloc', [v_size], resulttype=llmemory.Address)
 
 def rtype_raw_malloc_usage(hop):
@@ -586,6 +602,7 @@
     if s_addr.is_null_address():
         raise TyperError("raw_memclear(x, n) where x is the constant NULL")
     v_list = hop.inputargs(llmemory.Address, lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_memclear', v_list)
 
 BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc
@@ -596,6 +613,7 @@
 
 def rtype_offsetof(hop):
     TYPE, field = hop.inputargs(lltype.Void, lltype.Void)
+    hop.exception_cannot_occur()
     return hop.inputconst(lltype.Signed,
                           llmemory.offsetof(TYPE.value, field.value))
 
@@ -605,6 +623,7 @@
 # non-gc objects
 
 def rtype_free_non_gc_object(hop):
+    hop.exception_cannot_occur()
     vinst, = hop.inputargs(hop.args_r[0])
     flavor = hop.args_r[0].gcflavor
     assert flavor != 'gc'
@@ -617,6 +636,7 @@
 # keepalive_until_here
 
 def rtype_keepalive_until_here(hop):
+    hop.exception_cannot_occur()
     for v in hop.args_v:
         hop.genop('keepalive', [v], resulttype=lltype.Void)
     return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/rfloat.py b/pypy/rpython/rfloat.py
--- a/pypy/rpython/rfloat.py
+++ b/pypy/rpython/rfloat.py
@@ -136,7 +136,10 @@
         hop.exception_cannot_occur()
         return hop.genop('cast_float_to_int', vlist, resulttype=Signed)
 
-    rtype_float = rtype_pos
+    def rtype_float(_, hop):
+        vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
+        return vlist[0]
 
     # version picked by specialisation based on which
     # type system rtyping is using, from <type_system>.ll_str module
diff --git a/pypy/rpython/rint.py b/pypy/rpython/rint.py
--- a/pypy/rpython/rint.py
+++ b/pypy/rpython/rint.py
@@ -310,6 +310,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_chr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_char', vlist, resulttype=Char)
 
     def rtype_unichr(_, hop):
@@ -317,6 +319,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_unichr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_unichar', vlist, resulttype=UniChar)
 
     def rtype_is_true(self, hop):
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -115,6 +115,7 @@
     def rtype_bltn_list(self, hop):
         v_lst = hop.inputarg(self, 0)
         cRESLIST = hop.inputconst(Void, hop.r_result.LIST)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_copy, cRESLIST, v_lst)
 
     def rtype_len(self, hop):
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -107,8 +107,10 @@
     if isinstance(hop.r_result, AbstractRangeRepr):
         if hop.r_result.step != 0:
             c_rng = hop.inputconst(Void, hop.r_result.RANGE)
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrange, c_rng, vstart, vstop)
         else:
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrangest, vstart, vstop, vstep)
     else:
         # cannot build a RANGE object, needs a real list
@@ -117,6 +119,7 @@
         if isinstance(ITEMTYPE, Ptr):
             ITEMTYPE = ITEMTYPE.TO
         cLIST = hop.inputconst(Void, ITEMTYPE)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_range2list, cLIST, vstart, vstop, vstep)
 
 rtype_builtin_xrange = rtype_builtin_range
@@ -212,4 +215,5 @@
                                      [v_index, v_item])
 
 def rtype_builtin_enumerate(hop):
+    hop.exception_cannot_occur()
     return hop.r_result.r_baseiter.newiter(hop)
diff --git a/pypy/rpython/rstr.py b/pypy/rpython/rstr.py
--- a/pypy/rpython/rstr.py
+++ b/pypy/rpython/rstr.py
@@ -288,6 +288,8 @@
 
     def rtype_unicode(self, hop):
         if hop.args_s[0].is_constant():
+            # convertion errors occur during annotation, so cannot any more:
+            hop.exception_cannot_occur()
             return hop.inputconst(hop.r_result, hop.s_result.const)
         repr = hop.args_r[0].repr
         v_str = hop.inputarg(repr, 0)
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -846,6 +846,7 @@
         return result
 
     def exception_is_here(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
@@ -861,6 +862,7 @@
         self.llops.llop_raising_exceptions = len(self.llops)
 
     def exception_cannot_occur(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
diff --git a/pypy/rpython/test/test_extregistry.py b/pypy/rpython/test/test_extregistry.py
--- a/pypy/rpython/test/test_extregistry.py
+++ b/pypy/rpython/test/test_extregistry.py
@@ -114,6 +114,7 @@
         _about_ = dummy_func
         s_result_annotation = annmodel.SomeInteger()
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             return hop.inputconst(lltype.Signed, 42)
 
     def func():
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1085,6 +1085,7 @@
                 return annmodel.SomeInteger()
             def specialize_call(self, hop):
                 [v_instance] = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.gendirectcall(ll_my_gethash, v_instance)
 
         def f(n):
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -477,7 +477,11 @@
             s1 = s[:3]
             s2 = s[3:]
             s3 = s[3:10]
-            return s1+s2 == s and s2+s1 == const('lohel') and s1+s3 == s
+            s4 = s[42:44]
+            return (s1+s2 == s and
+                    s2+s1 == const('lohel') and
+                    s1+s3 == s and
+                    s4 == const(''))
         res = self.interpret(fn, [0])
         assert res
 
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -38,7 +38,7 @@
 closed_heads.reverse()
 
 for head, branch in closed_heads:
-    print '\t', branch
+    print '\t', head, '\t', branch
 print
 print 'The branches listed above will be merged to "closed-branches".'
 print 'You need to run this script in a clean working copy where you'
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -847,6 +847,10 @@
                 if sources:
                     target, = sources
 
+        if target.endswith('@PLT'):
+            # In -fPIC mode, all functions calls have this suffix
+            target = target[:-4]
+
         if target in self.FUNCTIONS_NOT_RETURNING:
             return [InsnStop(target)]
         if self.format == 'mingw32' and target == '__alloca':
@@ -1137,7 +1141,7 @@
     r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$")
 
     r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$")
-    r_jmptable_item = re.compile(r"\t.quad\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$")
+    r_jmptable_item = re.compile(r"\t.(?:quad|long)\t"+LABEL+"(-\"[A-Za-z0-9$]+\"|-"+LABEL+")?\s*$")
     r_jmptable_end  = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL)
 
     r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/")
diff --git a/pypy/translator/c/test/test_extfunc.py b/pypy/translator/c/test/test_extfunc.py
--- a/pypy/translator/c/test/test_extfunc.py
+++ b/pypy/translator/c/test/test_extfunc.py
@@ -919,4 +919,5 @@
             t, cbuilder = self.compile(does_stuff)
             data = cbuilder.cmdexec('')
             res = os.nice(0) + 3
+            if res > 19: res = 19    # xxx Linux specific, probably
             assert data.startswith('os.nice returned %d\n' % res)
diff --git a/pypy/translator/cli/dotnet.py b/pypy/translator/cli/dotnet.py
--- a/pypy/translator/cli/dotnet.py
+++ b/pypy/translator/cli/dotnet.py
@@ -459,6 +459,7 @@
             
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         assert hop.args_s[1].is_constant()
         TYPE = hop.args_s[1].const
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
@@ -507,6 +508,7 @@
 
     def specialize_call(self, hop):
         v_obj, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('same_as', [v_obj], hop.r_result.lowleveltype)
 
 def new_array(type, length):
@@ -608,6 +610,7 @@
 
     def specialize_call(self, hop):
         v_type, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('cli_typeof', [v_type], hop.r_result.lowleveltype)
 
 
@@ -626,6 +629,7 @@
         v_obj, = hop.inputargs(*hop.args_r)
         methodname = hop.args_r[0].methodname
         c_methodname = hop.inputconst(ootype.Void, methodname)
+        hop.exception_cannot_occur()
         return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype)
 
 
@@ -647,6 +651,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -668,6 +673,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -701,6 +707,7 @@
 
     def specialize_call(self, hop):
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype)
 
 

From noreply at buildbot.pypy.org  Sat Apr  7 21:01:57 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat,  7 Apr 2012 21:01:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: a script to update the contributors list
Message-ID: <20120407190157.E34B58208A@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54237:df57ca3b7687
Date: 2012-04-07 19:56 +0200
http://bitbucket.org/pypy/pypy/changeset/df57ca3b7687/

Log:	a script to update the contributors list

diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
new file mode 100644
--- /dev/null
+++ b/pypy/doc/tool/makecontributor.py
@@ -0,0 +1,64 @@
+import py
+import sys
+from collections import defaultdict
+import operator
+import re
+import mercurial.localrepo
+import mercurial.ui
+
+ROOT = py.path.local(__file__).join('..', '..', '..', '..')
+author_re = re.compile('(.*) <.*>')
+
+excluded = set(["pypy" "convert-repo"])
+
+alias = {
+    'arigo': 'Armin Rigo',
+    'lac': 'Laura Creighton',
+    'fijal': 'Maciej Fijalkowski',
+    'tismer at christia-wjtqxl.localdomain': 'Christian Tismer',
+    'holger krekel': 'Holger Krekel',
+    'hager': 'Sven Hager',
+    'mattip': 'Matti Picus',
+    'mattip>': 'Matti Picus',
+    'matthp': 'Matti Picus',
+    'Matti Picus matti.picus at gmail.com': 'Matti Picus',
+    'edelsohn': 'David Edelsohn',
+    'edelsoh': 'David Edelsohn',
+    'l.diekmann': 'Lukas Diekmann',
+    'ldiekmann': 'Lukas Diekmann',
+    'aliles': 'Aaron Iles',
+    'mikefc': 'Michael Cheng',
+    'cocoatomo': 'Tomo Cocoa',
+    'roberto at goyle': 'Roberto De Ioris',
+    'roberto at mrspurr': 'Roberto De Ioris',
+    'landtuna at gmail.com': 'Jim Hunziker',
+    'kristjan at kristjan-lp.ccp.ad.local': 'Kristjan Valur Jonsson',
+    }
+
+def get_canonical_author(name):
+    match = author_re.match(name)
+    if match:
+        name = match.group(1)
+    return alias.get(name, name)
+
+def main(show_numbers):
+    ui = mercurial.ui.ui()
+    repo = mercurial.localrepo.localrepository(ui, str(ROOT))
+    authors = defaultdict(int)
+    for i in repo:
+        ctx = repo[i]
+        author = get_canonical_author(ctx.user())
+        if author not in excluded:
+            authors[author] += 1
+    #
+    items = authors.items()
+    items.sort(key=operator.itemgetter(1), reverse=True)
+    for name, n in items:
+        if show_numbers:
+            print '%5d %s' % (n, name)
+        else:
+            print name
+
+if __name__ == '__main__':
+    show_numbers = '-n' in sys.argv
+    main(show_numbers)

From noreply at buildbot.pypy.org  Sat Apr  7 21:01:59 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat,  7 Apr 2012 21:01:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: be advanced: try hard to parse the pair
 programming markers in commit logs, and count them in the commit count :-)
Message-ID: <20120407190159.768828208A@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54238:97275a06775e
Date: 2012-04-07 21:00 +0200
http://bitbucket.org/pypy/pypy/changeset/97275a06775e/

Log:	be advanced: try hard to parse the pair programming markers in
	commit logs, and count them in the commit count :-)

diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -8,50 +8,119 @@
 
 ROOT = py.path.local(__file__).join('..', '..', '..', '..')
 author_re = re.compile('(.*) <.*>')
-
-excluded = set(["pypy" "convert-repo"])
+pair_programming_re = re.compile(r'^\((.*?)\)')
+excluded = set(["pypy", "convert-repo"])
 
 alias = {
-    'arigo': 'Armin Rigo',
-    'lac': 'Laura Creighton',
-    'fijal': 'Maciej Fijalkowski',
-    'tismer at christia-wjtqxl.localdomain': 'Christian Tismer',
-    'holger krekel': 'Holger Krekel',
-    'hager': 'Sven Hager',
-    'mattip': 'Matti Picus',
-    'mattip>': 'Matti Picus',
-    'matthp': 'Matti Picus',
-    'Matti Picus matti.picus at gmail.com': 'Matti Picus',
-    'edelsohn': 'David Edelsohn',
-    'edelsoh': 'David Edelsohn',
-    'l.diekmann': 'Lukas Diekmann',
-    'ldiekmann': 'Lukas Diekmann',
-    'aliles': 'Aaron Iles',
-    'mikefc': 'Michael Cheng',
-    'cocoatomo': 'Tomo Cocoa',
-    'roberto at goyle': 'Roberto De Ioris',
-    'roberto at mrspurr': 'Roberto De Ioris',
-    'landtuna at gmail.com': 'Jim Hunziker',
-    'kristjan at kristjan-lp.ccp.ad.local': 'Kristjan Valur Jonsson',
+    'Anders Chrigstrom': ['arre'],
+    'Antonio Cuni': ['antocuni', 'anto'],
+    'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
+    'Maciej Fijalkowski': ['fijal'],
+    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Michael Hudson': ['mwh'],
+    'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
+    "Amaury Forgeot d'Arc": ['afa'],
+    'Alex Gaynor': ['alex', 'agaynor'],
+    'David Schneider': ['bivab', 'david'],
+    'Christian Tismer': ['chris', 'christian', 'tismer',
+                         'tismer at christia-wjtqxl.localdomain'],
+    'Benjamin Peterson': ['benjamin'],
+    'Hakan Ardo': ['hakan', 'hakanardo'],
+    'Niklaus Haldimann': ['nik'],
+    'Alexander Schremmer': ['xoraxax'],
+    'Anders Hammarquist': ['iko'],
+    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'Niko Matsakis': ['niko'],
+    'Jakub Gustak': ['jlg'],
+    'Guido Wesdorp': ['guido'],
+    'Michael Foord': ['mfoord'],
+    'Mark Pearse': ['mwp'],
+    'Toon Verwaest': ['tverwaes'],
+    'Eric van Riet Paap': ['ericvrp'],
+    'Jacob Hallen': ['jacob', 'jakob'],
+    'Anders Lehmann': ['ale', 'anders'],
+    'Bert Freudenberg': ['bert'],
+    'Boris Feigin': ['boris', 'boria'],
+    'Valentino Volonghi': ['valentino', 'dialtone'],
+    'Aurelien Campeas': ['aurelien', 'aureliene'],
+    'Adrien Di Mascio': ['adim'],
+    'Jacek Generowicz': ['Jacek', 'jacek'],
+    'Jim Hunziker': ['landtuna at gmail.com'],
+    'Kristjan Valur Jonsson': ['kristjan at kristjan-lp.ccp.ad.local'],
+    'Laura Creighton': ['lac'],
+    'Aaron Iles': ['aliles'],
+    'Ludovic Aubry': ['ludal', 'ludovic'],
+    'Lukas Diekmann': ['l.diekmann', 'ldiekmann'],
+    'Matti Picus': ['Matti Picus matti.picus at gmail.com',
+                    'matthp', 'mattip', 'mattip>'],
+    'Michael Cheng': ['mikefc'],
+    'Richard Emslie': ['rxe'],
+    'Roberto De Ioris': ['roberto at goyle'],
+    'Roberto De Ioris': ['roberto at mrspurr'],
+    'Sven Hager': ['hager'],
+    'Tomo Cocoa': ['cocoatomo'],
     }
 
+alias_map = {}
+for name, nicks in alias.iteritems():
+    for nick in nicks:
+        alias_map[nick] = name
+
 def get_canonical_author(name):
     match = author_re.match(name)
     if match:
         name = match.group(1)
-    return alias.get(name, name)
+    return alias_map.get(name, name)
+
+ignored_nicknames = defaultdict(int)
+
+def get_more_authors(log):
+    match = pair_programming_re.match(log)
+    if not match:
+        return set()
+    ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
+                    'in-progress', 'bits of', 'even a little', 'floating',]
+    sep_words = ['and', ';', '+', '/', 'with special  by']
+    nicknames = match.group(1)
+    for word in ignore_words:
+        nicknames = nicknames.replace(word, '')
+    for word in sep_words:
+        nicknames = nicknames.replace(word, ',')
+    nicknames = [nick.strip().lower() for nick in nicknames.split(',')]
+    authors = set()
+    for nickname in nicknames:
+        author = alias_map.get(nickname)
+        if not author:
+            ignored_nicknames[nickname] += 1
+        else:
+            authors.add(author)
+    return authors
 
 def main(show_numbers):
     ui = mercurial.ui.ui()
     repo = mercurial.localrepo.localrepository(ui, str(ROOT))
-    authors = defaultdict(int)
+    authors_count = defaultdict(int)
     for i in repo:
         ctx = repo[i]
-        author = get_canonical_author(ctx.user())
-        if author not in excluded:
-            authors[author] += 1
-    #
-    items = authors.items()
+        authors = set()
+        authors.add(get_canonical_author(ctx.user()))
+        authors.update(get_more_authors(ctx.description()))
+        for author in authors:
+            if author not in excluded:
+                authors_count[author] += 1
+
+    # uncomment the next lines to get the list of nicknamed which could not be
+    # parsed from commit logs
+    ## items = ignored_nicknames.items()
+    ## items.sort(key=operator.itemgetter(1), reverse=True)
+    ## for name, n in items:
+    ##     if show_numbers:
+    ##         print '%5d %s' % (n, name)
+    ##     else:
+    ##         print name
+                
+    items = authors_count.items()
     items.sort(key=operator.itemgetter(1), reverse=True)
     for name, n in items:
         if show_numbers:

From noreply at buildbot.pypy.org  Sat Apr  7 21:02:00 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Sat,  7 Apr 2012 21:02:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120407190200.B804C8208A@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54239:df6b4266c2e9
Date: 2012-04-07 21:01 +0200
http://bitbucket.org/pypy/pypy/changeset/df6b4266c2e9/

Log:	merge

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,8 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, classofinstance=None,
-                                   strdict=False):
+                                   instance=False, strdict=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -563,10 +562,7 @@
     def listview_int(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
 
-    def w_keys(self, w_dict):
-        # XXX there is no space.newlist_int yet
-        space = self.space
-        return space.call_function(space.w_list, w_dict)
+    # XXX there is no space.newlist_int yet to implement w_keys more efficiently
 
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -1,5 +1,5 @@
 ## ----------------------------------------------------------------------------
-## dict strategy (see dict_multiobject.py)
+## dict strategy (see dictmultiobject.py)
 
 from pypy.rlib import rerased
 from pypy.rlib.debug import mark_dict_non_null
@@ -80,8 +80,8 @@
     def iter(self, w_dict):
         return IdentityDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class IdentityDictIteratorImplementation(_UnwrappedIteratorMixin, IteratorImplementation):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,10 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                classofinstance=classofinstance,
                 strdict=strdict)
 
     def newset(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -885,10 +885,9 @@
     def newtuple(self, l):
         return tuple(l)
 
-    def newdict(self, module=False, instance=False, classofinstance=None):
+    def newdict(self, module=False, instance=False):
         return W_DictMultiObject.allocate_and_init_instance(
-                self, module=module, instance=instance,
-                classofinstance=classofinstance)
+                self, module=module, instance=instance)
 
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict
@@ -968,6 +967,20 @@
         assert type(self.impl.strategy) is self.StrategyClass
         #assert self.impl.r_dict_content is None
 
+    def test_popitem(self):
+        self.fill_impl()
+        assert self.impl.length() == 2
+        a, b = self.impl.popitem()
+        assert self.impl.length() == 1
+        if a == self.string:
+            assert b == 1000
+            assert self.impl.getitem(self.string2) == 2000
+        else:
+            assert a == self.string2
+            assert b == 2000
+            assert self.impl.getitem_str(self.string) == 1000
+        self.check_not_devolved()
+
     def test_setitem(self):
         self.impl.setitem(self.string, 1000)
         assert self.impl.length() == 1

From noreply at buildbot.pypy.org  Sun Apr  8 00:46:00 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 00:46:00 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: windows exception on get
 linenumber, linecolumn after error
Message-ID: <20120407224600.4536E8208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54240:0e4763b18703
Date: 2012-04-06 17:22 +0300
http://bitbucket.org/pypy/pypy/changeset/0e4763b18703/

Log:	windows exception on get linenumber, linecolumn after error

diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -10,6 +10,7 @@
 from pypy.translator.platform import platform
 
 import sys
+import os
 import weakref
 import py
 
@@ -677,8 +678,12 @@
 
     def set_error(self, space, code):
         err = rffi.charp2strn(XML_ErrorString(code), 200)
-        lineno = XML_GetCurrentLineNumber(self.itself)
-        colno = XML_GetCurrentColumnNumber(self.itself)
+        if os.name == 'nt':
+            lineno = -1
+            colno = -1
+        else:
+            lineno = XML_GetCurrentLineNumber(self.itself)
+            colno = XML_GetCurrentColumnNumber(self.itself)
         msg = "%s: line %d, column %d" % (err, lineno, colno)
         w_errorcls = space.fromcache(Cache).w_error
         w_error = space.call_function(w_errorcls, space.wrap(msg))

From noreply at buildbot.pypy.org  Sun Apr  8 00:46:01 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 00:46:01 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix for float('nan') returning
 -NAN, Microsoft should try to adhere to standards
Message-ID: <20120407224601.7B03D8208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54241:da56cef5777d
Date: 2012-04-08 01:44 +0300
http://bitbucket.org/pypy/pypy/changeset/da56cef5777d/

Log:	fix for float('nan') returning -NAN, Microsoft should try to adhere
	to standards

diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -11,6 +11,7 @@
      GCHeaderOffset, GCREF, AddressAsInt
 from pypy.rpython.lltypesystem.llarena import RoundedUpForAllocation
 from pypy.translator.c.support import cdecl, barebonearray
+from pypy.translator.platform import platform as target_platform
 
 # ____________________________________________________________
 #
@@ -98,11 +99,19 @@
     else:
         return '%dLL' % value
 
-def is_positive_nan(value):
-    # bah.  we don't have math.copysign() if we're running Python 2.5
-    import struct
-    c = struct.pack("!d", value)[0]
-    return {'\x7f': True, '\xff': False}[c]
+if target_platform.name == 'msvc':
+    def is_positive_nan(value):
+        # Microsoft decided that NAN should have the sign bit set,
+        # so the values are reversed!
+        import struct
+        c = struct.pack("!d", value)[0]
+        return {'\x7f': False, '\xff': True}[c]
+else:
+    def is_positive_nan(value):
+        # bah.  we don't have math.copysign() if we're running Python 2.5
+        import struct
+        c = struct.pack("!d", value)[0]
+        return {'\x7f': True, '\xff': False}[c]
 
 def name_float(value, db):
     if isinf(value):

From noreply at buildbot.pypy.org  Sun Apr  8 00:46:02 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 00:46:02 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: revert console handling to run
 tests on this version
Message-ID: <20120407224602.AF7958208A@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54242:05ab3be91dfd
Date: 2012-04-08 01:45 +0300
http://bitbucket.org/pypy/pypy/changeset/05ab3be91dfd/

Log:	revert console handling to run tests on this version

diff --git a/pypy/translator/c/src/winstuff.c b/pypy/translator/c/src/winstuff.c
--- a/pypy/translator/c/src/winstuff.c
+++ b/pypy/translator/c/src/winstuff.c
@@ -27,7 +27,7 @@
 
 void pypy_Windows_startup(void)
 {
-#if 0 && defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
+#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
     /* Set CRT argument error handler */
     _set_invalid_parameter_handler(InvalidParameterHandler);
     /* turn off assertions within CRT in debug mode;

From pullrequests-noreply at bitbucket.org  Sun Apr  8 06:57:25 2012
From: pullrequests-noreply at bitbucket.org (Roberto De Ioris)
Date: Sun, 08 Apr 2012 04:57:25 -0000
Subject: [pypy-commit] [pypy/pypy] PyFile_FromFile,
 PySys_SetArgv and PyThreadState_New implementation (pull request
 #68)
Message-ID: <819aec1e6e3998acaa7b35b136186f8a@bitbucket.org>

A new pull request has been opened by Roberto De Ioris.

unbit/pypy/default has changes to be pulled into pypy/pypy/embedded-pypy.

https://bitbucket.org/pypy/pypy/pull-request/68/pyfile_fromfile-pysys_setargv-and

Title: PyFile_FromFile, PySys_SetArgv and PyThreadState_New implementation

PyFile_FromFile() is used by c apps to map a libc FILE object to a Python file object. The implementation make use of the pypy-specific file.fdopen function.  PySys_SetArgv is used to set sys.argv. Both Tested with latest uWSGI tip. PyThreadState_New is only a prototype, i suspect the different internal threading usage in PyPy will require a bigger effort.

Changes to be pulled:

5b28c054a739 by Roberto De Ioris: "added PySys_SetArgv and PyThreadState_New"
7bd1b6f657a8 by Roberto De Ioris: "merge with official pypy"
a7b45b1ec178 by Matti Picus: "fix float-to-time conversion in windows"
057219807d73 by arigo: "merge heads"
01a73d86d3c7 by arigo: "minimark: Tweaks to reduce by one the number of checks in the common
case."
d356d169e770 by Roberto De Ioris: "add fflush to cpyext pyfile implementation"
1f3baa6ce8b1 by rob... at mrsurr: "merge"
2afeef0e350b by Roberto De Ioris: "implemed PyFile_FromFile as cpyext"
7eb44aa590a2 by Roberto De Ioris: "added Py_Initialize,Py_Finalize,Py_SetPythonHome,Py_SetProgramName as cpyext"
4c1e64afbe9a by Roberto De Ioris: "added implementation of Py_Initialize"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Sun Apr  8 13:08:55 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 13:08:55 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: Backed out changeset:
 da56cef5777d, pypy is OK problem is with underlying cpython
Message-ID: <20120408110855.09E8C82111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54243:0d96f6934f8f
Date: 2012-04-08 10:15 +0300
http://bitbucket.org/pypy/pypy/changeset/0d96f6934f8f/

Log:	Backed out changeset: da56cef5777d, pypy is OK problem is with
	underlying cpython

diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -11,7 +11,6 @@
      GCHeaderOffset, GCREF, AddressAsInt
 from pypy.rpython.lltypesystem.llarena import RoundedUpForAllocation
 from pypy.translator.c.support import cdecl, barebonearray
-from pypy.translator.platform import platform as target_platform
 
 # ____________________________________________________________
 #
@@ -99,19 +98,11 @@
     else:
         return '%dLL' % value
 
-if target_platform.name == 'msvc':
-    def is_positive_nan(value):
-        # Microsoft decided that NAN should have the sign bit set,
-        # so the values are reversed!
-        import struct
-        c = struct.pack("!d", value)[0]
-        return {'\x7f': False, '\xff': True}[c]
-else:
-    def is_positive_nan(value):
-        # bah.  we don't have math.copysign() if we're running Python 2.5
-        import struct
-        c = struct.pack("!d", value)[0]
-        return {'\x7f': True, '\xff': False}[c]
+def is_positive_nan(value):
+    # bah.  we don't have math.copysign() if we're running Python 2.5
+    import struct
+    c = struct.pack("!d", value)[0]
+    return {'\x7f': True, '\xff': False}[c]
 
 def name_float(value, db):
     if isinf(value):

From noreply at buildbot.pypy.org  Sun Apr  8 13:08:56 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 13:08:56 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: add more validate_fd
Message-ID: <20120408110856.55A7D82111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54244:10ccfa6466f5
Date: 2012-04-08 13:02 +0300
http://bitbucket.org/pypy/pypy/changeset/10ccfa6466f5/

Log:	add more validate_fd

diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -942,6 +942,8 @@
                                    rffi.INT, threadsafe=False)
         
         def close_llimpl(fd):
+            if not rposix.validate_fd(fd):
+                raise OSError(rposix.get_errno(), 'Bad file descriptor')
             error = rffi.cast(lltype.Signed, os_close(rffi.cast(rffi.INT, fd)))
             if error == -1:
                 raise OSError(rposix.get_errno(), "close failed")
@@ -977,6 +979,8 @@
                                    rffi.LONGLONG)
 
         def lseek_llimpl(fd, pos, how):
+            if not rposix.validate_fd(fd):
+                raise OSError(rposix.get_errno(), 'Bad file descriptor')
             how = fix_seek_arg(how)
             res = os_lseek(rffi.cast(rffi.INT,      fd),
                            rffi.cast(rffi.LONGLONG, pos),
@@ -1002,6 +1006,8 @@
                                        [rffi.INT, rffi.LONGLONG], rffi.INT)
 
         def ftruncate_llimpl(fd, length):
+            if not rposix.validate_fd(fd):
+                raise OSError(rposix.get_errno(), 'Bad file descriptor')
             res = rffi.cast(rffi.LONG,
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
@@ -1020,6 +1026,8 @@
             os_fsync = self.llexternal('_commit', [rffi.INT], rffi.INT)
 
         def fsync_llimpl(fd):
+            if not rposix.validate_fd(fd):
+                raise OSError(rposix.get_errno(), 'Bad file descriptor')
             res = rffi.cast(rffi.SIGNED, os_fsync(rffi.cast(rffi.INT, fd)))
             if res < 0:
                 raise OSError(rposix.get_errno(), "fsync failed")
@@ -1032,6 +1040,8 @@
         os_fdatasync = self.llexternal('fdatasync', [rffi.INT], rffi.INT)
 
         def fdatasync_llimpl(fd):
+            if not rposix.validate_fd(fd):
+                raise OSError(rposix.get_errno(), 'Bad file descriptor')
             res = rffi.cast(rffi.SIGNED, os_fdatasync(rffi.cast(rffi.INT, fd)))
             if res < 0:
                 raise OSError(rposix.get_errno(), "fdatasync failed")
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -191,7 +191,7 @@
 
 def test_os_write():
     #Same as test in rpython/test/test_rbuiltin
-    fname = str(udir.join('os_write_test.txt'))
+    fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
@@ -202,6 +202,46 @@
     os.close(fd)
     raises(OSError, os.write, fd, 'Hello world')
 
+def test_os_close():
+    fname = str(udir.join('os_test.txt'))
+    fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
+    assert fd >= 0
+    os.write(fd, 'Hello world')
+    os.close(fd)
+    raises(OSError, os.close, fd)
+
+def test_os_lseek():
+    fname = str(udir.join('os_test.txt'))
+    fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
+    assert fd >= 0
+    os.write(fd, 'Hello world')
+    os.lseek(fd,0,0)
+    assert os.tell(fd) == 0
+    os.close(fd)
+    raises(OSError, os.lseek, fd, 0, 0)
+
+def test_os_fsync():
+    fname = str(udir.join('os_test.txt'))
+    fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
+    assert fd >= 0
+    os.write(fd, 'Hello world')
+    os.fsync(fd)
+    fid = open(fname)
+    assert fid.read() == 'Hello world'
+    os.close(fd)
+    raises(OSError, os.fsync, fd)
+
+def test_os_fdatasync():
+    fname = str(udir.join('os_test.txt'))
+    fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
+    assert fd >= 0
+    os.write(fd, 'Hello world')
+    os.fdatasync(fd)
+    fid = open(fname)
+    assert fid.read() == 'Hello world'
+    os.close(fd)
+    raises(OSError, os.fdatasync, fd)
+
 class ExpectTestOs:
     def setup_class(cls):
         if not hasattr(os, 'ttyname'):

From noreply at buildbot.pypy.org  Sun Apr  8 13:08:57 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 13:08:57 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: skip test if unable to run
Message-ID: <20120408110857.909A382111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54245:49cc32603e11
Date: 2012-04-08 13:21 +0300
http://bitbucket.org/pypy/pypy/changeset/49cc32603e11/

Log:	skip test if unable to run

diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -157,13 +157,15 @@
         raises(IOError, _multiprocessing.Connection, -15)
 
     def test_byte_order(self):
+        import socket
+        if not 'fromfd' in dir(socket):
+            skip('No fromfd in socket')
         # The exact format of net strings (length in network byte
         # order) is important for interoperation with others
         # implementations.
         rhandle, whandle = self.make_pair()
         whandle.send_bytes("abc")
         whandle.send_bytes("defg")
-        import socket
         sock = socket.fromfd(rhandle.fileno(),
                              socket.AF_INET, socket.SOCK_STREAM)
         data1 = sock.recv(7)

From noreply at buildbot.pypy.org  Sun Apr  8 13:08:58 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 13:08:58 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix test,
 catching raised exception with 'raises' fails on win32
Message-ID: <20120408110858.E5CD482111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54246:36ff8218c07d
Date: 2012-04-08 13:30 +0300
http://bitbucket.org/pypy/pypy/changeset/36ff8218c07d/

Log:	fix test, catching raised exception with 'raises' fails on win32

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -617,7 +617,10 @@
         except timeout:
             pass
         # test sendall() timeout
-        raises(timeout, cli.sendall, 'foobar' * 70)
+        try:
+            cli.sendall('foobar'*70)
+        except:
+            pass
         # done
         cli.close()
         t.close()

From noreply at buildbot.pypy.org  Sun Apr  8 13:09:00 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 13:09:00 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: windir can be lower case on
	windows 7
Message-ID: <20120408110900.26A1E82111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54247:10aecff7bec5
Date: 2012-04-08 13:45 +0300
http://bitbucket.org/pypy/pypy/changeset/10aecff7bec5/

Log:	windir can be lower case on windows 7

diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -198,7 +198,10 @@
         import nt
         r = ExpandEnvironmentStrings(u"%windir%\\test")
         assert isinstance(r, unicode)
-        assert r == nt.environ["WINDIR"] + "\\test"
+        if 'WINDIR' in nt.environ.keys():
+            assert r == nt.environ["WINDIR"] + "\\test"
+        else:
+            assert r == nt.environ["windir"] + "\\test"
 
     def test_long_key(self):
         from _winreg import (

From noreply at buildbot.pypy.org  Sun Apr  8 13:09:01 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 13:09:01 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: sign of nan is not defined,
	skip test
Message-ID: <20120408110901.60BA882111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54248:d31590b69d3c
Date: 2012-04-08 13:49 +0300
http://bitbucket.org/pypy/pypy/changeset/d31590b69d3c/

Log:	sign of nan is not defined, skip test

diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -271,5 +271,6 @@
         assert math.trunc(foo()) == "truncated"
 
     def test_copysign_nan(self):
+        skip('sign of nan is undefined')
         import math
         assert math.copysign(1.0, float('-nan')) == -1.0

From noreply at buildbot.pypy.org  Sun Apr  8 13:09:02 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 13:09:02 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: do not use copysign in testing
	nan
Message-ID: <20120408110902.9FA4A82111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54249:281bca084dae
Date: 2012-04-08 13:58 +0300
http://bitbucket.org/pypy/pypy/changeset/281bca084dae/

Log:	do not use copysign in testing nan

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -149,7 +149,11 @@
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+        try:
+            import struct
+        except:
+            skip('no struct available, and copysign is unreliable. How to compare?')
+        assert struct.pack('d', fmax(nnan, nan)) == struct.pack('d', nnan)
 
     def test_fmin(self):
         from _numpypy import fmin
@@ -165,7 +169,11 @@
         assert math.isnan(fmin(0, nan))
         assert math.isnan(fmin(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+        try:
+            import struct
+        except:
+            skip('no struct available, and copysign is unreliable. How to compare?')
+        assert struct.pack('d', fmin(nan, nnan)) == struct.pack('d', nan)
 
     def test_fmod(self):
         from _numpypy import fmod

From noreply at buildbot.pypy.org  Sun Apr  8 14:11:49 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 14:11:49 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: force sign to be dfined for
 testing with float('nan')
Message-ID: <20120408121149.4384182111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54250:c5f3e2ef91fd
Date: 2012-04-08 14:29 +0300
http://bitbucket.org/pypy/pypy/changeset/c5f3e2ef91fd/

Log:	force sign to be dfined for testing with float('nan')

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -270,8 +270,8 @@
         assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
         assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
         assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
-        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
-        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(abs(float("nan")))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(-abs(float("nan")))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign

From noreply at buildbot.pypy.org  Sun Apr  8 14:11:50 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 14:11:50 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: low lying fruit
Message-ID: <20120408121150.7E87882287@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54251:bacd1ee8cfaa
Date: 2012-04-08 14:33 +0300
http://bitbucket.org/pypy/pypy/changeset/bacd1ee8cfaa/

Log:	low lying fruit

diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -596,7 +596,7 @@
         import sys
         size = 0x14FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
@@ -618,7 +618,7 @@
         import sys
         size = 0x17FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)

From noreply at buildbot.pypy.org  Sun Apr  8 14:11:51 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 14:11:51 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: add check for offset validiy
Message-ID: <20120408121151.B4DE782111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54252:be83d722b0fb
Date: 2012-04-08 15:04 +0300
http://bitbucket.org/pypy/pypy/changeset/be83d722b0fb/

Log:	add check for offset validiy

diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -747,6 +747,25 @@
             # SEEK_SET = 0
             # libc._lseek(fileno, 0, SEEK_SET)
 
+        # check file size
+        try:
+            st = os.fstat(fileno)
+        except OSError:
+            pass     # ignore errors and trust map_size
+        else:
+            mode = st[stat.ST_MODE]
+            size = st[stat.ST_SIZE]
+            if stat.S_ISREG(mode):
+                if map_size == 0:
+                    if offset > size:
+                        raise RValueError(
+                            "mmap offset is greater than file size")
+                    map_size = int(size - offset)
+                    if map_size != size - offset:
+                        raise RValueError("mmap length is too large")
+                elif offset + map_size > size:
+                    raise RValueError("mmap length is greater than file size")
+
         m = MMap(access, offset)
         m.file_handle = INVALID_HANDLE
         m.map_handle = INVALID_HANDLE

From noreply at buildbot.pypy.org  Sun Apr  8 14:46:59 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 14:46:59 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: make sure float('nan') does not
	set the sign bit
Message-ID: <20120408124659.571B682111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54253:7a8139560192
Date: 2012-04-08 15:42 +0300
http://bitbucket.org/pypy/pypy/changeset/7a8139560192/

Log:	make sure float('nan') does not set the sign bit

diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -108,7 +108,10 @@
         assert long(rctime.mktime(rctime.gmtime(t))) - rctime.timezone == long(t)
         ltime = rctime.localtime()
         assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime)
-
+        try:
+            rctime.localtime(-1)
+        except:
+            skip('localtime cannot be negative on this platform')
         assert rctime.mktime(rctime.localtime(-1)) == -1
 
     def test_asctime(self):
diff --git a/pypy/translator/c/src/ll_math.c b/pypy/translator/c/src/ll_math.c
--- a/pypy/translator/c/src/ll_math.c
+++ b/pypy/translator/c/src/ll_math.c
@@ -20,7 +20,7 @@
 #define PyPy_IS_INFINITY(X) ((X) &&                                   \
                              (Py_FORCE_DOUBLE(X)*0.5 == Py_FORCE_DOUBLE(X)))
 #endif
-#define PyPy_NAN (HUGE_VAL * 0.)
+#define PyPy_NAN fabs(HUGE_VAL * 0.)
 
 /* The following copyright notice applies to the original
    implementations of acosh, asinh and atanh. */

From noreply at buildbot.pypy.org  Sun Apr  8 14:47:01 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 14:47:01 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: remove invalid_parameter_handler
Message-ID: <20120408124701.61C0082111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54254:e3349e81f0e9
Date: 2012-04-08 15:42 +0300
http://bitbucket.org/pypy/pypy/changeset/e3349e81f0e9/

Log:	remove invalid_parameter_handler

diff --git a/pypy/translator/c/src/winstuff.c b/pypy/translator/c/src/winstuff.c
--- a/pypy/translator/c/src/winstuff.c
+++ b/pypy/translator/c/src/winstuff.c
@@ -27,7 +27,7 @@
 
 void pypy_Windows_startup(void)
 {
-#if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
+#if 0 &&defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__)
     /* Set CRT argument error handler */
     _set_invalid_parameter_handler(InvalidParameterHandler);
     /* turn off assertions within CRT in debug mode;

From noreply at buildbot.pypy.org  Sun Apr  8 15:56:00 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 15:56:00 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: more precise exception handling
	(antocuni)
Message-ID: <20120408135600.2EFB382111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54255:c0ded41a61e0
Date: 2012-04-08 16:55 +0300
http://bitbucket.org/pypy/pypy/changeset/c0ded41a61e0/

Log:	more precise exception handling (antocuni)

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -619,7 +619,7 @@
         # test sendall() timeout
         try:
             cli.sendall('foobar'*70)
-        except:
+        except timeout:
             pass
         # done
         cli.close()

From noreply at buildbot.pypy.org  Sun Apr  8 18:02:58 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Apr 2012 18:02:58 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: typo
Message-ID: <20120408160258.4BA0D82111@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54256:d6ba957b6cb5
Date: 2012-04-08 17:04 +0200
http://bitbucket.org/pypy/pypy/changeset/d6ba957b6cb5/

Log:	typo

diff --git a/pypy/jit/backend/x86/test/test_basic.py b/pypy/jit/backend/x86/test/test_basic.py
--- a/pypy/jit/backend/x86/test/test_basic.py
+++ b/pypy/jit/backend/x86/test/test_basic.py
@@ -14,7 +14,7 @@
 
 class TestBasic(Jit386Mixin, test_ajit.BaseLLtypeTests):
     # for the individual tests see
-    # ====> ../../../metainterp/test/test_basic.py
+    # ====> ../../../metainterp/test/test_ajit.py
     def test_bug(self):
         jitdriver = JitDriver(greens = [], reds = ['n'])
         class X(object):

From noreply at buildbot.pypy.org  Sun Apr  8 18:02:59 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  8 Apr 2012 18:02:59 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: fix the test on x86
Message-ID: <20120408160259.8AB7682111@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54257:7a3a9c99b85d
Date: 2012-04-08 18:02 +0200
http://bitbucket.org/pypy/pypy/changeset/7a3a9c99b85d/

Log:	fix the test on x86

diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -22,6 +22,7 @@
                  gcdescr=None):
         assert type(opts) is not bool
         self.opts = opts
+        self.arraydescr_cache = {}
 
         from pypy.jit.backend.llsupport.gc import get_ll_description
         AbstractCPU.__init__(self)
@@ -292,7 +293,14 @@
                                                  extrainfo, ffi_flags)
 
     def copy_and_change_descr_typeinfo_to_ptr(self, descr):
-        return ArrayDescr(descr.basesize, descr.itemsize, descr.lendescr, FLAG_POINTER)
+        key = (descr.basesize, descr.itemsize, descr.lendescr)
+        try:
+            return self.arraydescr_cache[key]
+        except KeyError:
+            new_descr = ArrayDescr(descr.basesize, descr.itemsize,
+                                   descr.lendescr, FLAG_POINTER)
+            self.arraydescr_cache[key] = new_descr
+            return new_descr
 
     def get_overflow_error(self):
         ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -1,4 +1,3 @@
-import copy
 
 import py
 

From noreply at buildbot.pypy.org  Sun Apr  8 19:07:26 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 19:07:26 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: sidestep copysign bug in
 underlying python for test
Message-ID: <20120408170726.A89DE82111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54258:9ced0e490fa8
Date: 2012-04-08 19:51 +0300
http://bitbucket.org/pypy/pypy/changeset/9ced0e490fa8/

Log:	sidestep copysign bug in underlying python for test

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -148,12 +148,11 @@
         assert math.isnan(fmax(nan, 0))
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
-        # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        try:
-            import struct
-        except:
-            skip('no struct available, and copysign is unreliable. How to compare?')
-        assert struct.pack('d', fmax(nnan, nan)) == struct.pack('d', nnan)
+        # The numpy docs specify that 
+        # the FIRST NaN should be used if both are NaN
+        # use copysign on both sides to sidestep bug in nan representaion 
+        # on Microsoft win32
+        assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
 
     def test_fmin(self):
         from _numpypy import fmin
@@ -167,13 +166,11 @@
         assert (fmin(a, [1]*5) == [ninf, -5, 0, 1, 1]).all()
         assert math.isnan(fmin(nan, 0))
         assert math.isnan(fmin(0, nan))
-        assert math.isnan(fmin(nan, nan))
-        # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        try:
-            import struct
-        except:
-            skip('no struct available, and copysign is unreliable. How to compare?')
-        assert struct.pack('d', fmin(nan, nnan)) == struct.pack('d', nan)
+        # The numpy docs specify that 
+        # the FIRST NaN should be used if both are NaN
+        # use copysign on both sides to sidestep bug in nan representaion 
+        # on Microsoft win32
+        assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
     def test_fmod(self):
         from _numpypy import fmod

From noreply at buildbot.pypy.org  Sun Apr  8 19:07:27 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun,  8 Apr 2012 19:07:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test for 32 bit platforms
Message-ID: <20120408170727.E2ADC82111@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54259:aaa93c295404
Date: 2012-04-08 20:06 +0300
http://bitbucket.org/pypy/pypy/changeset/aaa93c295404/

Log:	fix test for 32 bit platforms

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -592,9 +592,8 @@
 
     def test_shift(self):
         from _numpypy import left_shift, right_shift
-        import sys
 
-        assert (left_shift([5, 1], [2, 31]) == [20, 2**31]).all()
+        assert (left_shift([5, 1], [2, 13]) == [20, 2**13]).all()
         assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
 
     def test_comparisons(self):

From noreply at buildbot.pypy.org  Mon Apr  9 04:15:21 2012
From: noreply at buildbot.pypy.org (taavi_burns)
Date: Mon,  9 Apr 2012 04:15:21 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-ufuncs3: Update logaddexp(2) tests to run
 correctly under cpython/numpy. The current micronumpy implementation is
 wrong; fix in next commit.
Message-ID: <20120409021521.D9C8982111@wyvern.cs.uni-duesseldorf.de>

Author: Taavi Burns <taavi.burns at gmail.com>
Branch: numpy-ufuncs3
Changeset: r54260:2a1a61463bd1
Date: 2012-04-08 22:14 -0400
http://bitbucket.org/pypy/pypy/changeset/2a1a61463bd1/

Log:	Update logaddexp(2) tests to run correctly under cpython/numpy. The
	current micronumpy implementation is wrong; fix in next commit.

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -763,6 +763,7 @@
 
     def test_logaddexp(self):
         import math
+        import sys
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -773,7 +774,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(sys.float_info.max, sys.float_info.max) == sys.float_info.max
+        assert logaddexp(sys.float_info.min, sys.float_info.min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -786,6 +788,7 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -797,7 +800,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(sys.float_info.max, sys.float_info.max) == sys.float_info.max
+        assert logaddexp2(sys.float_info.min, sys.float_info.min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))

From noreply at buildbot.pypy.org  Mon Apr  9 10:19:17 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Apr 2012 10:19:17 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add enter/leave_transactional_mode
	operations.
Message-ID: <20120409081917.CC6E246E0F0@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54262:f7e7bb5d8f56
Date: 2012-04-05 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/f7e7bb5d8f56/

Log:	Add enter/leave_transactional_mode operations.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -269,10 +269,14 @@
 @rgc.no_collect
 def _run():
     # --- start the threads --- don't use the GC here any more! ---
+    rstm.enter_transactional_mode()
+    #
     for i in range(state.num_threads):
         threadintf.start_new_thread(_run_thread)
     #
     state.lock_unfinished()  # wait for all threads to finish
+    #
+    rstm.leave_transactional_mode()
     # --- done, we can use the GC again ---
 
 
diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -50,6 +50,12 @@
     if not we_are_translated():
         _global_lock.release()
 
+def enter_transactional_mode():
+    llop.stm_enter_transactional_mode(lltype.Void)
+
+def leave_transactional_mode():
+    llop.stm_leave_transactional_mode(lltype.Void)
+
 def descriptor_init():
     if not we_are_translated(): _global_lock.acquire()
     llop.stm_descriptor_init(lltype.Void)
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -403,6 +403,8 @@
     'stm_become_inevitable':  LLOp(),
     'stm_descriptor_init':    LLOp(canrun=True),
     'stm_descriptor_done':    LLOp(canrun=True),
+    'stm_enter_transactional_mode': LLOp(),
+    'stm_leave_transactional_mode': LLOp(),
     'stm_writebarrier':       LLOp(sideeffects=False),
     'stm_normalize_global':   LLOp(),
     'stm_start_transaction':  LLOp(canrun=True),
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -32,6 +32,12 @@
         self.stm_normalize_global_ptr = getfn(
             self.gcdata.gc.stm_normalize_global,
             [annmodel.SomeAddress()], annmodel.SomeAddress())
+        self.stm_enter_transactional_mode_ptr = getfn(
+            self.gcdata.gc.enter_transactional_mode.im_func
+            [s_sc], annmodel.s_None)
+        self.stm_leave_transactional_mode_ptr = getfn(
+            self.gcdata.gc.leave_transactional_mode.im_func
+            [s_sc], annmodel.s_None)
         self.stm_start_ptr = getfn(
             self.gcdata.gc.start_transaction.im_func,
             [s_gc], annmodel.s_None)

From noreply at buildbot.pypy.org  Mon Apr  9 10:19:19 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Apr 2012 10:19:19 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: A comment
Message-ID: <20120409081919.3145346E0F0@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54263:2d60ce23a2ec
Date: 2012-04-09 10:18 +0200
http://bitbucket.org/pypy/pypy/changeset/2d60ce23a2ec/

Log:	A comment

diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -155,7 +155,8 @@
              ((char *)localobj) + sizeof(orec_t),
              size - sizeof(orec_t));
       /* but we must only unlock the orec if it's the last time it
-         appears in the redolog list.  If it's not, then p == -1. */
+         appears in the redolog list.  If it's not, then p == -1.
+         XXX I think that duplicate orecs are not possible any more. */
       if (p != -1)
         {
           volatile orec_t* o = get_orec(globalobj);

From noreply at buildbot.pypy.org  Mon Apr  9 10:19:16 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  9 Apr 2012 10:19:16 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Crash with a clearer error message when
	not translated.
Message-ID: <20120409081916.8C1FC46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54261:f170650418ee
Date: 2012-04-05 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/f170650418ee/

Log:	Crash with a clearer error message when not translated.

diff --git a/pypy/rlib/debug.py b/pypy/rlib/debug.py
--- a/pypy/rlib/debug.py
+++ b/pypy/rlib/debug.py
@@ -1,5 +1,6 @@
 import sys, time
 from pypy.rpython.extregistry import ExtRegistryEntry
+from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.rarithmetic import is_valid_int
 
 
@@ -21,8 +22,13 @@
         hop.exception_cannot_occur()
         hop.genop('debug_assert', vlist)
 
+class FatalError(Exception):
+    pass
+
 def fatalerror(msg):
     # print the RPython traceback and abort with a fatal error
+    if not we_are_translated():
+        raise FatalError(msg)
     from pypy.rpython.lltypesystem import lltype
     from pypy.rpython.lltypesystem.lloperation import llop
     llop.debug_print_traceback(lltype.Void)
@@ -33,6 +39,8 @@
 
 def fatalerror_notb(msg):
     # a variant of fatalerror() that doesn't print the RPython traceback
+    if not we_are_translated():
+        raise FatalError(msg)
     from pypy.rpython.lltypesystem import lltype
     from pypy.rpython.lltypesystem.lloperation import llop
     llop.debug_fatalerror(lltype.Void, msg)

From noreply at buildbot.pypy.org  Mon Apr  9 11:40:14 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon,  9 Apr 2012 11:40:14 +0200 (CEST)
Subject: [pypy-commit] pypy pytest: upgrade py and _pytest to current tip
Message-ID: <20120409094014.3A8C746E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: pytest
Changeset: r54264:f1fdc8c6ed6a
Date: 2012-04-09 11:38 +0200
http://bitbucket.org/pypy/pypy/changeset/f1fdc8c6ed6a/

Log:	upgrade py and _pytest to current tip

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.2.3'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,7 +1,6 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
 import errno
 import itertools
 import imp
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -34,15 +34,21 @@
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
 def bin_xml_escape(arg):
     def repl(matchobj):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -132,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -91,22 +91,28 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -113,9 +113,12 @@
 
     # some backward compatibility helpers
     _basestring = str
-    def _totext(obj, encoding=None):
+    def _totext(obj, encoding=None, errors=None):
         if isinstance(obj, bytes):
-            obj = obj.decode(encoding)
+            if errors is None:
+                obj = obj.decode(encoding)
+            else:
+                obj = obj.decode(encoding, errors)
         elif not isinstance(obj, str):
             obj = str(obj)
         return obj
diff --git a/py/_error.py b/py/_error.py
--- a/py/_error.py
+++ b/py/_error.py
@@ -23,6 +23,7 @@
     2: errno.ENOENT,
     3: errno.ENOENT,
     17: errno.EEXIST,
+    13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -12,7 +12,7 @@
     class TextIO(StringIO):
         def write(self, data):
             if not isinstance(data, unicode):
-                data = unicode(data, getattr(self, '_encoding', 'UTF-8'))
+                data = unicode(data, getattr(self, '_encoding', 'UTF-8'), 'replace')
             StringIO.write(self, data)
 else:
     TextIO = StringIO
@@ -260,7 +260,7 @@
                 res = f.read()
                 enc = getattr(f, 'encoding', None)
                 if enc:
-                    res = py.builtin._totext(res, enc)
+                    res = py.builtin._totext(res, enc, 'replace')
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -64,7 +64,10 @@
                 else:
                     if bool(value) ^ bool(meth()) ^ invert:
                         return False
-            except (py.error.ENOENT, py.error.ENOTDIR):
+            except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
+                # EBUSY feels not entirely correct,
+                # but its kind of necessary since ENOMEDIUM
+                # is not accessible in python     
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -52,7 +52,7 @@
     def unicode(self, indent=2):
         l = []
         SimpleUnicodeVisitor(l.append, indent).visit(self)
-        return "".join(l)
+        return u("").join(l)
 
     def __repr__(self):
         name = self.__class__.__name__
@@ -122,11 +122,13 @@
                 if visitmethod is not None:
                     break
             else:
-                visitmethod = self.object
+                visitmethod = self.__object
             self.cache[cls] = visitmethod
         visitmethod(node)
 
-    def object(self, obj):
+    # the default fallback handler is marked private
+    # to avoid clashes with the tag name object
+    def __object(self, obj):
         #self.write(obj)
         self.write(escape(unicode(obj)))
 
@@ -182,7 +184,11 @@
             value = getattr(attrs, name)
             if name.endswith('_'):
                 name = name[:-1]
-            return ' %s="%s"' % (name, escape(unicode(value)))
+            if isinstance(value, raw):
+                insert = value.uniobj
+            else:
+                insert = escape(unicode(value))
+            return ' %s="%s"' % (name, insert)
 
     def getstyle(self, tag):
         """ return attribute list suitable for styling. """

From noreply at buildbot.pypy.org  Mon Apr  9 11:40:31 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon,  9 Apr 2012 11:40:31 +0200 (CEST)
Subject: [pypy-commit] pypy pytest: merge from default
Message-ID: <20120409094031.A548C46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: pytest
Changeset: r54265:026db0b5babb
Date: 2012-04-09 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/026db0b5babb/

Log:	merge from default

diff too long, truncating to 10000 out of 28906 lines

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py
--- a/lib-python/2.7/SimpleXMLRPCServer.py
+++ b/lib-python/2.7/SimpleXMLRPCServer.py
@@ -486,7 +486,10 @@
             L = []
             while size_remaining:
                 chunk_size = min(size_remaining, max_chunk_size)
-                L.append(self.rfile.read(chunk_size))
+                chunk = self.rfile.read(chunk_size)
+                if not chunk:
+                    break
+                L.append(chunk)
                 size_remaining -= len(L[-1])
             data = ''.join(L)
 
diff --git a/lib-python/2.7/test/test_xmlrpc.py b/lib-python/2.7/test/test_xmlrpc.py
--- a/lib-python/2.7/test/test_xmlrpc.py
+++ b/lib-python/2.7/test/test_xmlrpc.py
@@ -308,7 +308,7 @@
         global ADDR, PORT, URL
         ADDR, PORT = serv.socket.getsockname()
         #connect to IP address directly.  This avoids socket.create_connection()
-        #trying to connect to to "localhost" using all address families, which
+        #trying to connect to "localhost" using all address families, which
         #causes slowdown e.g. on vista which supports AF_INET6.  The server listens
         #on AF_INET only.
         URL = "http://%s:%d"%(ADDR, PORT)
@@ -367,7 +367,7 @@
         global ADDR, PORT, URL
         ADDR, PORT = serv.socket.getsockname()
         #connect to IP address directly.  This avoids socket.create_connection()
-        #trying to connect to to "localhost" using all address families, which
+        #trying to connect to "localhost" using all address families, which
         #causes slowdown e.g. on vista which supports AF_INET6.  The server listens
         #on AF_INET only.
         URL = "http://%s:%d"%(ADDR, PORT)
@@ -472,6 +472,9 @@
                 # protocol error; provide additional information in test output
                 self.fail("%s\n%s" % (e, getattr(e, "headers", "")))
 
+    def test_unicode_host(self):
+        server = xmlrpclib.ServerProxy(u"http://%s:%d/RPC2"%(ADDR, PORT))
+        self.assertEqual(server.add("a", u"\xe9"), u"a\xe9")
 
     # [ch] The test 404 is causing lots of false alarms.
     def XXXtest_404(self):
@@ -586,6 +589,12 @@
         # This avoids waiting for the socket timeout.
         self.test_simple1()
 
+    def test_partial_post(self):
+        # Check that a partial POST doesn't make the server loop: issue #14001.
+        conn = httplib.HTTPConnection(ADDR, PORT)
+        conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye')
+        conn.close()
+
 class MultiPathServerTestCase(BaseServerTestCase):
     threadFunc = staticmethod(http_multi_server)
     request_count = 2
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -311,7 +311,7 @@
     RegrTest('test_mimetypes.py'),
     RegrTest('test_MimeWriter.py', core=False),
     RegrTest('test_minidom.py'),
-    RegrTest('test_mmap.py'),
+    RegrTest('test_mmap.py', usemodules="mmap"),
     RegrTest('test_module.py', core=True),
     RegrTest('test_modulefinder.py'),
     RegrTest('test_msilib.py', skip=only_win32),
diff --git a/lib-python/modified-2.7/distutils/command/bdist_wininst.py b/lib-python/modified-2.7/distutils/command/bdist_wininst.py
--- a/lib-python/modified-2.7/distutils/command/bdist_wininst.py
+++ b/lib-python/modified-2.7/distutils/command/bdist_wininst.py
@@ -298,7 +298,8 @@
                              bitmaplen,        # number of bytes in bitmap
                              )
         file.write(header)
-        file.write(open(arcname, "rb").read())
+        with open(arcname, "rb") as arcfile:
+            file.write(arcfile.read())
 
     # create_exe()
 
diff --git a/lib-python/modified-2.7/distutils/sysconfig_pypy.py b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
--- a/lib-python/modified-2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
@@ -60,6 +60,7 @@
     g['EXE'] = ""
     g['SO'] = _get_so_extension() or ".so"
     g['SOABI'] = g['SO'].rsplit('.')[0]
+    g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
 
     global _config_vars
     _config_vars = g
diff --git a/lib-python/modified-2.7/opcode.py b/lib-python/modified-2.7/opcode.py
--- a/lib-python/modified-2.7/opcode.py
+++ b/lib-python/modified-2.7/opcode.py
@@ -192,5 +192,6 @@
 def_op('LOOKUP_METHOD', 201)          # Index in name list
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
+def_op('BUILD_LIST_FROM_ARG', 203)
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/modified-2.7/site.py b/lib-python/modified-2.7/site.py
--- a/lib-python/modified-2.7/site.py
+++ b/lib-python/modified-2.7/site.py
@@ -550,9 +550,18 @@
                 "'import usercustomize' failed; use -v for traceback"
 
 
+def import_builtin_stuff():
+    """PyPy specific: pre-import a few built-in modules, because
+    some programs actually rely on them to be in sys.modules :-("""
+    import exceptions
+    if 'zipimport' in sys.builtin_module_names:
+        import zipimport
+
+
 def main():
     global ENABLE_USER_SITE
 
+    import_builtin_stuff()
     abs__file__()
     known_paths = removeduppaths()
     if (os.name == "posix" and sys.path and
diff --git a/lib-python/modified-2.7/test/test_dis.py b/lib-python/modified-2.7/test/test_dis.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/test/test_dis.py
@@ -0,0 +1,150 @@
+# Minimal tests for dis module
+
+from test.test_support import run_unittest
+import unittest
+import sys
+import dis
+import StringIO
+
+
+def _f(a):
+    print a
+    return 1
+
+dis_f = """\
+ %-4d         0 LOAD_FAST                0 (a)
+              3 PRINT_ITEM
+              4 PRINT_NEWLINE
+
+ %-4d         5 LOAD_CONST               1 (1)
+              8 RETURN_VALUE
+"""%(_f.func_code.co_firstlineno + 1,
+     _f.func_code.co_firstlineno + 2)
+
+
+def bug708901():
+    for res in range(1,
+                     10):
+        pass
+
+dis_bug708901 = """\
+ %-4d         0 SETUP_LOOP              23 (to 26)
+              3 LOAD_GLOBAL              0 (range)
+              6 LOAD_CONST               1 (1)
+
+ %-4d         9 LOAD_CONST               2 (10)
+             12 CALL_FUNCTION            2
+             15 GET_ITER
+        >>   16 FOR_ITER                 6 (to 25)
+             19 STORE_FAST               0 (res)
+
+ %-4d        22 JUMP_ABSOLUTE           16
+        >>   25 POP_BLOCK
+        >>   26 LOAD_CONST               0 (None)
+             29 RETURN_VALUE
+"""%(bug708901.func_code.co_firstlineno + 1,
+     bug708901.func_code.co_firstlineno + 2,
+     bug708901.func_code.co_firstlineno + 3)
+
+
+def bug1333982(x=[]):
+    assert 0, ([s for s in x] +
+              1)
+    pass
+
+dis_bug1333982 = """\
+ %-4d         0 LOAD_CONST               1 (0)
+              3 POP_JUMP_IF_TRUE        38
+              6 LOAD_GLOBAL              0 (AssertionError)
+              9 LOAD_FAST                0 (x)
+             12 BUILD_LIST_FROM_ARG      0
+             15 GET_ITER
+        >>   16 FOR_ITER                12 (to 31)
+             19 STORE_FAST               1 (s)
+             22 LOAD_FAST                1 (s)
+             25 LIST_APPEND              2
+             28 JUMP_ABSOLUTE           16
+
+ %-4d   >>   31 LOAD_CONST               2 (1)
+             34 BINARY_ADD
+             35 RAISE_VARARGS            2
+
+ %-4d   >>   38 LOAD_CONST               0 (None)
+             41 RETURN_VALUE
+"""%(bug1333982.func_code.co_firstlineno + 1,
+     bug1333982.func_code.co_firstlineno + 2,
+     bug1333982.func_code.co_firstlineno + 3)
+
+_BIG_LINENO_FORMAT = """\
+%3d           0 LOAD_GLOBAL              0 (spam)
+              3 POP_TOP
+              4 LOAD_CONST               0 (None)
+              7 RETURN_VALUE
+"""
+
+class DisTests(unittest.TestCase):
+    def do_disassembly_test(self, func, expected):
+        s = StringIO.StringIO()
+        save_stdout = sys.stdout
+        sys.stdout = s
+        dis.dis(func)
+        sys.stdout = save_stdout
+        got = s.getvalue()
+        # Trim trailing blanks (if any).
+        lines = got.split('\n')
+        lines = [line.rstrip() for line in lines]
+        expected = expected.split("\n")
+        import difflib
+        if expected != lines:
+            self.fail(
+                "events did not match expectation:\n" +
+                "\n".join(difflib.ndiff(expected,
+                                        lines)))
+
+    def test_opmap(self):
+        self.assertEqual(dis.opmap["STOP_CODE"], 0)
+        self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst)
+        self.assertIn(dis.opmap["STORE_NAME"], dis.hasname)
+
+    def test_opname(self):
+        self.assertEqual(dis.opname[dis.opmap["LOAD_FAST"]], "LOAD_FAST")
+
+    def test_boundaries(self):
+        self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG)
+        self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT)
+
+    def test_dis(self):
+        self.do_disassembly_test(_f, dis_f)
+
+    def test_bug_708901(self):
+        self.do_disassembly_test(bug708901, dis_bug708901)
+
+    def test_bug_1333982(self):
+        # This one is checking bytecodes generated for an `assert` statement,
+        # so fails if the tests are run with -O.  Skip this test then.
+        if __debug__:
+            self.do_disassembly_test(bug1333982, dis_bug1333982)
+
+    def test_big_linenos(self):
+        def func(count):
+            namespace = {}
+            func = "def foo():\n " + "".join(["\n "] * count + ["spam\n"])
+            exec func in namespace
+            return namespace['foo']
+
+        # Test all small ranges
+        for i in xrange(1, 300):
+            expected = _BIG_LINENO_FORMAT % (i + 2)
+            self.do_disassembly_test(func(i), expected)
+
+        # Test some larger ranges too
+        for i in xrange(300, 5000, 10):
+            expected = _BIG_LINENO_FORMAT % (i + 2)
+            self.do_disassembly_test(func(i), expected)
+
+def test_main():
+    run_unittest(DisTests)
+
+
+if __name__ == "__main__":
+    test_main()
diff --git a/lib-python/modified-2.7/test/test_set.py b/lib-python/modified-2.7/test/test_set.py
--- a/lib-python/modified-2.7/test/test_set.py
+++ b/lib-python/modified-2.7/test/test_set.py
@@ -1568,7 +1568,7 @@
             for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint):
                 for g in (G, I, Ig, L, R):
                     expected = meth(data)
-                    actual = meth(G(data))
+                    actual = meth(g(data))
                     if isinstance(expected, bool):
                         self.assertEqual(actual, expected)
                     else:
diff --git a/lib_pypy/_csv.py b/lib_pypy/_csv.py
--- a/lib_pypy/_csv.py
+++ b/lib_pypy/_csv.py
@@ -414,7 +414,7 @@
 
     def _parse_add_char(self, c):
         if len(self.field) + len(c) > _field_limit:
-            raise Error("field larget than field limit (%d)" % (_field_limit))
+            raise Error("field larger than field limit (%d)" % (_field_limit))
         self.field += c
         
 
diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -31,24 +31,20 @@
     arg = cobj._get_buffer_value()
     return _rawffi.wcharp2rawunicode(arg, lgt)
 
-class ErrorObject(local):
-    def __init__(self):
-        self.errno = 0
-        self.winerror = 0
-_error_object = ErrorObject()
+_err = local()
 
 def get_errno():
-    return _error_object.errno
+    return getattr(_err, "errno", 0)
 
 def set_errno(errno):
-    old_errno = _error_object.errno
-    _error_object.errno = errno
+    old_errno = get_errno()
+    _err.errno = errno
     return old_errno
 
 def get_last_error():
-    return _error_object.winerror
+    return getattr(_err, "winerror", 0)
 
 def set_last_error(winerror):
-    old_winerror = _error_object.winerror
-    _error_object.winerror = winerror
+    old_winerror = get_last_error()
+    _err.winerror = winerror
     return old_winerror
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -3,7 +3,7 @@
 from _ctypes.primitive import SimpleType, _SimpleCData
 from _ctypes.basics import ArgumentError, keepalive_key
 from _ctypes.basics import is_struct_shape
-from _ctypes.builtin import set_errno, set_last_error
+from _ctypes.builtin import get_errno, set_errno, get_last_error, set_last_error
 import _rawffi
 import _ffi
 import sys
@@ -350,16 +350,24 @@
     def _call_funcptr(self, funcptr, *newargs):
 
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
-            set_errno(_rawffi.get_errno())
+            tmp = _rawffi.get_errno()
+            _rawffi.set_errno(get_errno())
+            set_errno(tmp)
         if self._flags_ & _rawffi.FUNCFLAG_USE_LASTERROR:
-            set_last_error(_rawffi.get_last_error())
+            tmp = _rawffi.get_last_error()
+            _rawffi.set_last_error(get_last_error())
+            set_last_error(tmp)
         try:
             result = funcptr(*newargs)
         finally:
             if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
-                set_errno(_rawffi.get_errno())
+                tmp = _rawffi.get_errno()
+                _rawffi.set_errno(get_errno())
+                set_errno(tmp)
             if self._flags_ & _rawffi.FUNCFLAG_USE_LASTERROR:
-                set_last_error(_rawffi.get_last_error())
+                tmp = _rawffi.get_last_error()
+                _rawffi.set_last_error(get_last_error())
+                set_last_error(tmp)
         #
         try:
             return self._build_result(self._restype_, result, newargs)
diff --git a/lib_pypy/_locale.py b/lib_pypy/_locale.py
deleted file mode 100644
--- a/lib_pypy/_locale.py
+++ /dev/null
@@ -1,337 +0,0 @@
-# ctypes implementation of _locale module by Victor Stinner, 2008-03-27
-
-# ------------------------------------------------------------
-#  Note that we also have our own interp-level implementation
-# ------------------------------------------------------------
-
-"""
-Support for POSIX locales.
-"""
-
-from ctypes import (Structure, POINTER, create_string_buffer,
-    c_ubyte, c_int, c_char_p, c_wchar_p, c_size_t)
-from ctypes_support import standard_c_lib as libc
-from ctypes_support import get_errno
-
-# load the platform-specific cache made by running locale.ctc.py
-from ctypes_config_cache._locale_cache import *
-
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
-
-
-# Ubuntu Gusty i386 structure
-class lconv(Structure):
-    _fields_ = (
-        # Numeric (non-monetary) information.
-        ("decimal_point", c_char_p),    # Decimal point character.
-        ("thousands_sep", c_char_p),    # Thousands separator.
-
-        # Each element is the number of digits in each group;
-        # elements with higher indices are farther left.
-        # An element with value CHAR_MAX means that no further grouping is done.
-        # An element with value 0 means that the previous element is used
-        # for all groups farther left.  */
-        ("grouping", c_char_p),
-
-        # Monetary information.
-
-        # First three chars are a currency symbol from ISO 4217.
-        # Fourth char is the separator.  Fifth char is '\0'.
-        ("int_curr_symbol", c_char_p),
-        ("currency_symbol", c_char_p),   # Local currency symbol.
-        ("mon_decimal_point", c_char_p), # Decimal point character.
-        ("mon_thousands_sep", c_char_p), # Thousands separator.
-        ("mon_grouping", c_char_p),      # Like `grouping' element (above).
-        ("positive_sign", c_char_p),     # Sign for positive values.
-        ("negative_sign", c_char_p),     # Sign for negative values.
-        ("int_frac_digits", c_ubyte),    # Int'l fractional digits.
-        ("frac_digits", c_ubyte),        # Local fractional digits.
-        # 1 if currency_symbol precedes a positive value, 0 if succeeds.
-        ("p_cs_precedes", c_ubyte),
-        # 1 iff a space separates currency_symbol from a positive value.
-        ("p_sep_by_space", c_ubyte),
-        # 1 if currency_symbol precedes a negative value, 0 if succeeds.
-        ("n_cs_precedes", c_ubyte),
-        # 1 iff a space separates currency_symbol from a negative value.
-        ("n_sep_by_space", c_ubyte),
-
-        # Positive and negative sign positions:
-        # 0 Parentheses surround the quantity and currency_symbol.
-        # 1 The sign string precedes the quantity and currency_symbol.
-        # 2 The sign string follows the quantity and currency_symbol.
-        # 3 The sign string immediately precedes the currency_symbol.
-        # 4 The sign string immediately follows the currency_symbol.
-        ("p_sign_posn", c_ubyte),
-        ("n_sign_posn", c_ubyte),
-        # 1 if int_curr_symbol precedes a positive value, 0 if succeeds.
-        ("int_p_cs_precedes", c_ubyte),
-        # 1 iff a space separates int_curr_symbol from a positive value.
-        ("int_p_sep_by_space", c_ubyte),
-        # 1 if int_curr_symbol precedes a negative value, 0 if succeeds.
-        ("int_n_cs_precedes", c_ubyte),
-        # 1 iff a space separates int_curr_symbol from a negative value.
-        ("int_n_sep_by_space", c_ubyte),
-         # Positive and negative sign positions:
-         # 0 Parentheses surround the quantity and int_curr_symbol.
-         # 1 The sign string precedes the quantity and int_curr_symbol.
-         # 2 The sign string follows the quantity and int_curr_symbol.
-         # 3 The sign string immediately precedes the int_curr_symbol.
-         # 4 The sign string immediately follows the int_curr_symbol.
-        ("int_p_sign_posn", c_ubyte),
-        ("int_n_sign_posn", c_ubyte),
-    )
-
-_setlocale = libc.setlocale
-_setlocale.argtypes = (c_int, c_char_p)
-_setlocale.restype = c_char_p
-
-_localeconv = libc.localeconv
-_localeconv.argtypes = None
-_localeconv.restype = POINTER(lconv)
-
-_strcoll = libc.strcoll
-_strcoll.argtypes = (c_char_p, c_char_p)
-_strcoll.restype = c_int
-
-_wcscoll = libc.wcscoll
-_wcscoll.argtypes = (c_wchar_p, c_wchar_p)
-_wcscoll.restype = c_int
-
-_strxfrm = libc.strxfrm
-_strxfrm.argtypes = (c_char_p, c_char_p, c_size_t)
-_strxfrm.restype = c_size_t
-
-HAS_LIBINTL = hasattr(libc, 'gettext')
-if HAS_LIBINTL:
-    _gettext = libc.gettext
-    _gettext.argtypes = (c_char_p,)
-    _gettext.restype = c_char_p
-
-    _dgettext = libc.dgettext
-    _dgettext.argtypes = (c_char_p, c_char_p)
-    _dgettext.restype = c_char_p
-
-    _dcgettext = libc.dcgettext
-    _dcgettext.argtypes = (c_char_p, c_char_p, c_int)
-    _dcgettext.restype = c_char_p
-
-    _textdomain = libc.textdomain
-    _textdomain.argtypes = (c_char_p,)
-    _textdomain.restype = c_char_p
-
-    _bindtextdomain = libc.bindtextdomain
-    _bindtextdomain.argtypes = (c_char_p, c_char_p)
-    _bindtextdomain.restype = c_char_p
-
-    HAS_BIND_TEXTDOMAIN_CODESET = hasattr(libc, 'bindtextdomain_codeset')
-    if HAS_BIND_TEXTDOMAIN_CODESET:
-        _bind_textdomain_codeset = libc.bindtextdomain_codeset
-        _bind_textdomain_codeset.argtypes = (c_char_p, c_char_p)
-        _bind_textdomain_codeset.restype = c_char_p
-
-class Error(Exception):
-    pass
-
-def fixup_ulcase():
-    import string
-    #import strop
-
-    # create uppercase map string
-    ul = []
-    for c in xrange(256):
-        c = chr(c)
-        if c.isupper():
-            ul.append(c)
-    ul = ''.join(ul)
-    string.uppercase = ul
-    #strop.uppercase = ul
-
-    # create lowercase string
-    ul = []
-    for c in xrange(256):
-        c = chr(c)
-        if c.islower():
-            ul.append(c)
-    ul = ''.join(ul)
-    string.lowercase = ul
-    #strop.lowercase = ul
-
-    # create letters string
-    ul = []
-    for c in xrange(256):
-        c = chr(c)
-        if c.isalpha():
-            ul.append(c)
-    ul = ''.join(ul)
-    string.letters = ul
-
- at builtinify
-def setlocale(category, locale=None):
-    "(integer,string=None) -> string. Activates/queries locale processing."
-    if locale:
-        # set locale
-        result = _setlocale(category, locale)
-        if not result:
-            raise Error("unsupported locale setting")
-
-        # record changes to LC_CTYPE
-        if category in (LC_CTYPE, LC_ALL):
-            fixup_ulcase()
-    else:
-        # get locale
-        result = _setlocale(category, None)
-        if not result:
-            raise Error("locale query failed")
-    return result
-
-def _copy_grouping(text):
-    groups = [ ord(group) for group in text ]
-    if groups:
-        groups.append(0)
-    return groups
-
- at builtinify
-def localeconv():
-    "() -> dict. Returns numeric and monetary locale-specific parameters."
-
-    # if LC_NUMERIC is different in the C library, use saved value
-    lp = _localeconv()
-    l = lp.contents
-
-    # hopefully, the localeconv result survives the C library calls
-    # involved herein
-
-    # Numeric information
-    result = {
-        "decimal_point": l.decimal_point,
-        "thousands_sep": l.thousands_sep,
-        "grouping": _copy_grouping(l.grouping),
-        "int_curr_symbol": l.int_curr_symbol,
-        "currency_symbol": l.currency_symbol,
-        "mon_decimal_point": l.mon_decimal_point,
-        "mon_thousands_sep": l.mon_thousands_sep,
-        "mon_grouping": _copy_grouping(l.mon_grouping),
-        "positive_sign": l.positive_sign,
-        "negative_sign": l.negative_sign,
-        "int_frac_digits": l.int_frac_digits,
-        "frac_digits": l.frac_digits,
-        "p_cs_precedes": l.p_cs_precedes,
-        "p_sep_by_space": l.p_sep_by_space,
-        "n_cs_precedes": l.n_cs_precedes,
-        "n_sep_by_space": l.n_sep_by_space,
-        "p_sign_posn": l.p_sign_posn,
-        "n_sign_posn": l.n_sign_posn,
-    }
-    return result
-
- at builtinify
-def strcoll(s1, s2):
-    "string,string -> int. Compares two strings according to the locale."
-
-    # If both arguments are byte strings, use strcoll.
-    if isinstance(s1, str) and isinstance(s2, str):
-        return _strcoll(s1, s2)
-
-    # If neither argument is unicode, it's an error.
-    if not isinstance(s1, unicode) and not isinstance(s2, unicode):
-        raise ValueError("strcoll arguments must be strings")
-
-    # Convert the non-unicode argument to unicode.
-    s1 = unicode(s1)
-    s2 = unicode(s2)
-
-    # Collate the strings.
-    return _wcscoll(s1, s2)
-
- at builtinify
-def strxfrm(s):
-    "string -> string. Returns a string that behaves for cmp locale-aware."
-
-    # assume no change in size, first
-    n1 = len(s) + 1
-    buf = create_string_buffer(n1)
-    n2 = _strxfrm(buf, s, n1) + 1
-    if n2 > n1:
-        # more space needed
-        buf = create_string_buffer(n2)
-        _strxfrm(buf, s, n2)
-    return buf.value
-
- at builtinify
-def getdefaultlocale():
-    # TODO: Port code from CPython for Windows and Mac OS
-    raise NotImplementedError()
-
-if HAS_LANGINFO:
-    _nl_langinfo = libc.nl_langinfo
-    _nl_langinfo.argtypes = (nl_item,)
-    _nl_langinfo.restype = c_char_p
-
-    def nl_langinfo(key):
-        """nl_langinfo(key) -> string
-        Return the value for the locale information associated with key."""
-        # Check whether this is a supported constant. GNU libc sometimes
-        # returns numeric values in the char* return value, which would
-        # crash PyString_FromString.
-        result = _nl_langinfo(key)
-        if result is not None:
-            return result
-        raise ValueError("unsupported langinfo constant")
-
-if HAS_LIBINTL:
-    @builtinify
-    def gettext(msg):
-        """gettext(msg) -> string
-        Return translation of msg."""
-        return _gettext(msg)
-
-    @builtinify
-    def dgettext(domain, msg):
-        """dgettext(domain, msg) -> string
-        Return translation of msg in domain."""
-        return _dgettext(domain, msg)
-
-    @builtinify
-    def dcgettext(domain, msg, category):
-        """dcgettext(domain, msg, category) -> string
-        Return translation of msg in domain and category."""
-        return _dcgettext(domain, msg, category)
-
-    @builtinify
-    def textdomain(domain):
-        """textdomain(domain) -> string
-        Set the C library's textdomain to domain, returning the new domain."""
-        return _textdomain(domain)
-
-    @builtinify
-    def bindtextdomain(domain, dir):
-        """bindtextdomain(domain, dir) -> string
-        Bind the C library's domain to dir."""
-        dirname = _bindtextdomain(domain, dir)
-        if not dirname:
-            errno = get_errno()
-            raise OSError(errno)
-        return dirname
-
-    if HAS_BIND_TEXTDOMAIN_CODESET:
-        @builtinify
-        def bind_textdomain_codeset(domain, codeset):
-            """bind_textdomain_codeset(domain, codeset) -> string
-            Bind the C library's domain to codeset."""
-            codeset = _bind_textdomain_codeset(domain, codeset)
-            if codeset:
-                return codeset
-            return None
-
-__all__ = (
-    'Error',
-    'setlocale', 'localeconv', 'strxfrm', 'strcoll',
-) + ALL_CONSTANTS
-if HAS_LIBINTL:
-    __all__ += ('gettext', 'dgettext', 'dcgettext', 'textdomain',
-                'bindtextdomain')
-    if HAS_BIND_TEXTDOMAIN_CODESET:
-        __all__ += ('bind_textdomain_codeset',)
-if HAS_LANGINFO:
-    __all__ += ('nl_langinfo',)
diff --git a/lib_pypy/array.py b/lib_pypy/array.py
deleted file mode 100644
--- a/lib_pypy/array.py
+++ /dev/null
@@ -1,531 +0,0 @@
-"""This module defines an object type which can efficiently represent
-an array of basic values: characters, integers, floating point
-numbers.  Arrays are sequence types and behave very much like lists,
-except that the type of objects stored in them is constrained.  The
-type is specified at object creation time by using a type code, which
-is a single character.  The following type codes are defined:
-
-    Type code   C Type             Minimum size in bytes 
-    'c'         character          1 
-    'b'         signed integer     1 
-    'B'         unsigned integer   1 
-    'u'         Unicode character  2 
-    'h'         signed integer     2 
-    'H'         unsigned integer   2 
-    'i'         signed integer     2 
-    'I'         unsigned integer   2 
-    'l'         signed integer     4 
-    'L'         unsigned integer   4 
-    'f'         floating point     4 
-    'd'         floating point     8 
-
-The constructor is:
-
-array(typecode [, initializer]) -- create a new array
-"""
-
-from struct import calcsize, pack, pack_into, unpack_from
-import operator
-
-# the buffer-like object to use internally: trying from
-# various places in order...
-try:
-    import _rawffi                    # a reasonable implementation based
-    _RAWARRAY = _rawffi.Array('c')    # on raw_malloc, and providing a
-    def bytebuffer(size):             # real address
-        return _RAWARRAY(size, autofree=True)
-    def getbufaddress(buf):
-        return buf.buffer
-except ImportError:
-    try:
-        from __pypy__ import bytebuffer     # a reasonable implementation
-        def getbufaddress(buf):             # compatible with oo backends,
-            return 0                        # but no address
-    except ImportError:
-        # not running on PyPy.  Fall back to ctypes...
-        import ctypes
-        bytebuffer = ctypes.create_string_buffer
-        def getbufaddress(buf):
-            voidp = ctypes.cast(ctypes.pointer(buf), ctypes.c_void_p)
-            return voidp.value
-
-# ____________________________________________________________
-
-TYPECODES = "cbBuhHiIlLfd"
-
-class array(object):
-    """array(typecode [, initializer]) -> array
-    
-    Return a new array whose items are restricted by typecode, and
-    initialized from the optional initializer value, which must be a list,
-    string. or iterable over elements of the appropriate type.
-    
-    Arrays represent basic values and behave very much like lists, except
-    the type of objects stored in them is constrained.
-    
-    Methods:
-    
-    append() -- append a new item to the end of the array
-    buffer_info() -- return information giving the current memory info
-    byteswap() -- byteswap all the items of the array
-    count() -- return number of occurences of an object
-    extend() -- extend array by appending multiple elements from an iterable
-    fromfile() -- read items from a file object
-    fromlist() -- append items from the list
-    fromstring() -- append items from the string
-    index() -- return index of first occurence of an object
-    insert() -- insert a new item into the array at a provided position
-    pop() -- remove and return item (default last)
-    read() -- DEPRECATED, use fromfile()
-    remove() -- remove first occurence of an object
-    reverse() -- reverse the order of the items in the array
-    tofile() -- write all items to a file object
-    tolist() -- return the array converted to an ordinary list
-    tostring() -- return the array converted to a string
-    write() -- DEPRECATED, use tofile()
-    
-    Attributes:
-    
-    typecode -- the typecode character used to create the array
-    itemsize -- the length in bytes of one array item
-    """
-    __slots__ = ["typecode", "itemsize", "_data", "_descriptor", "__weakref__"]
-
-    def __new__(cls, typecode, initializer=[], **extrakwds):
-        self = object.__new__(cls)
-        if cls is array and extrakwds:
-            raise TypeError("array() does not take keyword arguments")
-        if not isinstance(typecode, str) or len(typecode) != 1:
-            raise TypeError(
-                     "array() argument 1 must be char, not %s" % type(typecode))
-        if typecode not in TYPECODES:
-            raise ValueError(
-                  "bad typecode (must be one of %s)" % ', '.join(TYPECODES))
-        self._data = bytebuffer(0)
-        self.typecode = typecode
-        self.itemsize = calcsize(typecode)
-        if isinstance(initializer, list):
-            self.fromlist(initializer)
-        elif isinstance(initializer, str):
-            self.fromstring(initializer)
-        elif isinstance(initializer, unicode) and self.typecode == "u":
-            self.fromunicode(initializer)
-        else:
-            self.extend(initializer)
-        return self
-
-    def _clear(self):
-        self._data = bytebuffer(0)
-
-    ##### array-specific operations
-
-    def fromfile(self, f, n):
-        """Read n objects from the file object f and append them to the end of
-        the array. Also called as read."""
-        if not isinstance(f, file):
-            raise TypeError("arg1 must be open file")
-        size = self.itemsize * n
-        item = f.read(size)
-        if len(item) < size:
-            raise EOFError("not enough items in file")
-        self.fromstring(item)
-
-    def fromlist(self, l):
-        """Append items to array from list."""
-        if not isinstance(l, list):
-            raise TypeError("arg must be list")
-        self._fromiterable(l)
-        
-    def fromstring(self, s):
-        """Appends items from the string, interpreting it as an array of machine
-        values, as if it had been read from a file using the fromfile()
-        method."""
-        if isinstance(s, unicode):
-            s = str(s)
-        self._frombuffer(s)
-
-    def _frombuffer(self, s):
-        length = len(s)
-        if length % self.itemsize != 0:
-            raise ValueError("string length not a multiple of item size")
-        boundary = len(self._data)
-        newdata = bytebuffer(boundary + length)
-        newdata[:boundary] = self._data
-        newdata[boundary:] = s
-        self._data = newdata
-
-    def fromunicode(self, ustr):
-        """Extends this array with data from the unicode string ustr. The array
-        must be a type 'u' array; otherwise a ValueError is raised. Use
-        array.fromstring(ustr.encode(...)) to append Unicode data to an array of
-        some other type."""
-        if not self.typecode == "u":
-            raise ValueError(
-                          "fromunicode() may only be called on type 'u' arrays")
-        # XXX the following probable bug is not emulated:
-        # CPython accepts a non-unicode string or a buffer, and then
-        # behaves just like fromstring(), except that it strangely truncates
-        # string arguments at multiples of the unicode byte size.
-        # Let's only accept unicode arguments for now.
-        if not isinstance(ustr, unicode):
-            raise TypeError("fromunicode() argument should probably be "
-                            "a unicode string")
-        # _frombuffer() does the currect thing using
-        # the buffer behavior of unicode objects
-        self._frombuffer(buffer(ustr))
-
-    def tofile(self, f):
-        """Write all items (as machine values) to the file object f.  Also
-        called as write."""
-        if not isinstance(f, file):
-            raise TypeError("arg must be open file")
-        f.write(self.tostring())
-        
-    def tolist(self):
-        """Convert array to an ordinary list with the same items."""
-        count = len(self._data) // self.itemsize
-        return list(unpack_from('%d%s' % (count, self.typecode), self._data))
-
-    def tostring(self):
-        return self._data[:]
-
-    def __buffer__(self):
-        return buffer(self._data)
-
-    def tounicode(self):
-        """Convert the array to a unicode string. The array must be a type 'u'
-        array; otherwise a ValueError is raised. Use array.tostring().decode()
-        to obtain a unicode string from an array of some other type."""
-        if self.typecode != "u":
-            raise ValueError("tounicode() may only be called on type 'u' arrays")
-        # XXX performance is not too good
-        return u"".join(self.tolist())
-
-    def byteswap(self):
-        """Byteswap all items of the array.  If the items in the array are not
-        1, 2, 4, or 8 bytes in size, RuntimeError is raised."""
-        if self.itemsize not in [1, 2, 4, 8]:
-            raise RuntimeError("byteswap not supported for this array")
-        # XXX slowish
-        itemsize = self.itemsize
-        bytes = self._data
-        for start in range(0, len(bytes), itemsize):
-            stop = start + itemsize
-            bytes[start:stop] = bytes[start:stop][::-1]
-
-    def buffer_info(self):
-        """Return a tuple (address, length) giving the current memory address
-        and the length in items of the buffer used to hold array's contents. The
-        length should be multiplied by the itemsize attribute to calculate the
-        buffer length in bytes. On PyPy the address might be meaningless
-        (returned as 0), depending on the available modules."""
-        return (getbufaddress(self._data), len(self))
-    
-    read = fromfile
-
-    write = tofile
-
-    ##### general object protocol
-    
-    def __repr__(self):
-        if len(self._data) == 0:
-            return "array('%s')" % self.typecode
-        elif self.typecode == "c":
-            return "array('%s', %s)" % (self.typecode, repr(self.tostring()))
-        elif self.typecode == "u":
-            return "array('%s', %s)" % (self.typecode, repr(self.tounicode()))
-        else:
-            return "array('%s', %s)" % (self.typecode, repr(self.tolist()))
-
-    def __copy__(self):
-        a = array(self.typecode)
-        a._data = bytebuffer(len(self._data))
-        a._data[:] = self._data
-        return a
-
-    def __eq__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) == buffer(other._data)
-        else:
-            return self.tolist() == other.tolist()
-
-    def __ne__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) != buffer(other._data)
-        else:
-            return self.tolist() != other.tolist()
-
-    def __lt__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) < buffer(other._data)
-        else:
-            return self.tolist() < other.tolist()
-
-    def __gt__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) > buffer(other._data)
-        else:
-            return self.tolist() > other.tolist()
-
-    def __le__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) <= buffer(other._data)
-        else:
-            return self.tolist() <= other.tolist()
-
-    def __ge__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) >= buffer(other._data)
-        else:
-            return self.tolist() >= other.tolist()
-
-    def __reduce__(self):
-        dict = getattr(self, '__dict__', None)
-        data = self.tostring()
-        if data:
-            initargs = (self.typecode, data)
-        else:
-            initargs = (self.typecode,)
-        return (type(self), initargs, dict)
-
-    ##### list methods
-    
-    def append(self, x):
-        """Append new value x to the end of the array."""
-        self._frombuffer(pack(self.typecode, x))
-
-    def count(self, x):
-        """Return number of occurences of x in the array."""
-        return operator.countOf(self, x)
-
-    def extend(self, iterable):
-        """Append items to the end of the array."""
-        if isinstance(iterable, array) \
-                                    and not self.typecode == iterable.typecode:
-            raise TypeError("can only extend with array of same kind")
-        self._fromiterable(iterable)
-
-    def index(self, x):
-        """Return index of first occurence of x in the array."""
-        return operator.indexOf(self, x)
-    
-    def insert(self, i, x):
-        """Insert a new item x into the array before position i."""
-        seqlength = len(self)
-        if i < 0:
-            i += seqlength
-            if i < 0:
-                i = 0
-        elif i > seqlength:
-            i = seqlength
-        boundary = i * self.itemsize
-        data = pack(self.typecode, x)
-        newdata = bytebuffer(len(self._data) + len(data))
-        newdata[:boundary] = self._data[:boundary]
-        newdata[boundary:boundary+self.itemsize] = data
-        newdata[boundary+self.itemsize:] = self._data[boundary:]
-        self._data = newdata
-        
-    def pop(self, i=-1):
-        """Return the i-th element and delete it from the array. i defaults to
-        -1."""
-        seqlength = len(self)
-        if i < 0:
-            i += seqlength
-        if not (0 <= i < seqlength):
-            raise IndexError(i)
-        boundary = i * self.itemsize
-        result = unpack_from(self.typecode, self._data, boundary)[0]
-        newdata = bytebuffer(len(self._data) - self.itemsize)
-        newdata[:boundary] = self._data[:boundary]
-        newdata[boundary:] = self._data[boundary+self.itemsize:]
-        self._data = newdata
-        return result
-        
-    def remove(self, x):
-        """Remove the first occurence of x in the array."""
-        self.pop(self.index(x))
-        
-    def reverse(self):
-        """Reverse the order of the items in the array."""
-        lst = self.tolist()
-        lst.reverse()
-        self._clear()
-        self.fromlist(lst)
-
-    ##### list protocol
-    
-    def __len__(self):
-        return len(self._data) // self.itemsize
-    
-    def __add__(self, other):
-        if not isinstance(other, array):
-            raise TypeError("can only append array to array")
-        if self.typecode != other.typecode:
-            raise TypeError("bad argument type for built-in operation")
-        return array(self.typecode, buffer(self._data) + buffer(other._data))
-
-    def __mul__(self, repeat):
-        return array(self.typecode, buffer(self._data) * repeat)
-
-    __rmul__ = __mul__
-
-    def __getitem__(self, i):
-        seqlength = len(self)
-        if isinstance(i, slice):
-            start, stop, step = i.indices(seqlength)
-            if step != 1:
-                sublist = self.tolist()[i]    # fall-back
-                return array(self.typecode, sublist)
-            if start < 0:
-                start = 0
-            if stop < start:
-                stop = start
-            assert stop <= seqlength
-            return array(self.typecode, self._data[start * self.itemsize :
-                                                   stop * self.itemsize])
-        else:
-            if i < 0:
-                i += seqlength
-            if self.typecode == 'c':  # speed trick
-                return self._data[i]
-            if not (0 <= i < seqlength):
-                raise IndexError(i)
-            boundary = i * self.itemsize
-            return unpack_from(self.typecode, self._data, boundary)[0]
-
-    def __getslice__(self, i, j):
-        return self.__getitem__(slice(i, j))
-
-    def __setitem__(self, i, x):
-        if isinstance(i, slice):
-            if (not isinstance(x, array)
-                or self.typecode != x.typecode):
-                raise TypeError("can only assign array of same kind"
-                                " to array slice")
-            seqlength = len(self)
-            start, stop, step = i.indices(seqlength)
-            if step != 1:
-                sublist = self.tolist()    # fall-back
-                sublist[i] = x.tolist()
-                self._clear()
-                self.fromlist(sublist)
-                return
-            if start < 0:
-                start = 0
-            if stop < start:
-                stop = start
-            assert stop <= seqlength
-            boundary1 = start * self.itemsize
-            boundary2 = stop * self.itemsize
-            boundary2new = boundary1 + len(x._data)
-            if boundary2 == boundary2new:
-                self._data[boundary1:boundary2] = x._data
-            else:
-                newdata = bytebuffer(len(self._data) + boundary2new-boundary2)
-                newdata[:boundary1] = self._data[:boundary1]
-                newdata[boundary1:boundary2new] = x._data
-                newdata[boundary2new:] = self._data[boundary2:]
-                self._data = newdata
-        else:
-            seqlength = len(self)
-            if i < 0:
-                i += seqlength
-            if self.typecode == 'c':  # speed trick
-                self._data[i] = x
-                return
-            if not (0 <= i < seqlength):
-                raise IndexError(i)
-            boundary = i * self.itemsize
-            pack_into(self.typecode, self._data, boundary, x)
-
-    def __setslice__(self, i, j, x):
-        self.__setitem__(slice(i, j), x)
-
-    def __delitem__(self, i):
-        if isinstance(i, slice):
-            seqlength = len(self)
-            start, stop, step = i.indices(seqlength)
-            if start < 0:
-                start = 0
-            if stop < start:
-                stop = start
-            assert stop <= seqlength
-            if step != 1:
-                sublist = self.tolist()    # fall-back
-                del sublist[i]
-                self._clear()
-                self.fromlist(sublist)
-                return
-            dellength = stop - start
-            boundary1 = start * self.itemsize
-            boundary2 = stop * self.itemsize
-            newdata = bytebuffer(len(self._data) - (boundary2-boundary1))
-            newdata[:boundary1] = self._data[:boundary1]
-            newdata[boundary1:] = self._data[boundary2:]
-            self._data = newdata
-        else:            
-            seqlength = len(self)
-            if i < 0:
-                i += seqlength
-            if not (0 <= i < seqlength):
-                raise IndexError(i)
-            boundary = i * self.itemsize
-            newdata = bytebuffer(len(self._data) - self.itemsize)
-            newdata[:boundary] = self._data[:boundary]
-            newdata[boundary:] = self._data[boundary+self.itemsize:]
-            self._data = newdata
-
-    def __delslice__(self, i, j):
-        self.__delitem__(slice(i, j))
-
-    def __contains__(self, item):
-        for x in self:
-            if x == item:
-                return True
-        return False
-
-    def __iadd__(self, other):
-        if not isinstance(other, array):
-            raise TypeError("can only extend array with array")
-        self.extend(other)
-        return self
-
-    def __imul__(self, repeat):
-        newdata = buffer(self._data) * repeat
-        self._data = bytebuffer(len(newdata))
-        self._data[:] = newdata
-        return self
-
-    def __iter__(self):
-        p = 0
-        typecode = self.typecode
-        itemsize = self.itemsize
-        while p < len(self._data):
-            yield unpack_from(typecode, self._data, p)[0]
-            p += itemsize
-
-    ##### internal methods
-
-    def _fromiterable(self, iterable):
-        iterable = tuple(iterable)
-        n = len(iterable)
-        boundary = len(self._data)
-        newdata = bytebuffer(boundary + n * self.itemsize)
-        newdata[:boundary] = self._data
-        pack_into('%d%s' % (n, self.typecode), newdata, boundary, *iterable)
-        self._data = newdata
-
-ArrayType = array
diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py
deleted file mode 100644
--- a/lib_pypy/binascii.py
+++ /dev/null
@@ -1,720 +0,0 @@
-"""A pure Python implementation of binascii.
-
-Rather slow and buggy in corner cases.
-PyPy provides an RPython version too.
-"""
-
-class Error(Exception):
-    pass
-
-class Done(Exception):
-    pass
-
-class Incomplete(Exception):
-    pass
-
-def a2b_uu(s):
-    if not s:
-        return ''
-    
-    length = (ord(s[0]) - 0x20) % 64
-
-    def quadruplets_gen(s):
-        while s:
-            try:
-                yield ord(s[0]), ord(s[1]), ord(s[2]), ord(s[3])
-            except IndexError:
-                s += '   '
-                yield ord(s[0]), ord(s[1]), ord(s[2]), ord(s[3])
-                return
-            s = s[4:]
-
-    try:
-        result = [''.join(
-            [chr((A - 0x20) << 2 | (((B - 0x20) >> 4) & 0x3)),
-            chr(((B - 0x20) & 0xf) << 4 | (((C - 0x20) >> 2) & 0xf)),
-            chr(((C - 0x20) & 0x3) << 6 | ((D - 0x20) & 0x3f))
-            ]) for A, B, C, D in quadruplets_gen(s[1:].rstrip())]
-    except ValueError:
-        raise Error('Illegal char')
-    result = ''.join(result)
-    trailingdata = result[length:]
-    if trailingdata.strip('\x00'):
-        raise Error('Trailing garbage')
-    result = result[:length]
-    if len(result) < length:
-        result += ((length - len(result)) * '\x00')
-    return result
-
-                               
-def b2a_uu(s):
-    length = len(s)
-    if length > 45:
-        raise Error('At most 45 bytes at once')
-
-    def triples_gen(s):
-        while s:
-            try:
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-            except IndexError:
-                s += '\0\0'
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-                return
-            s = s[3:]
-
-    result = [''.join(
-        [chr(0x20 + (( A >> 2                    ) & 0x3F)),
-         chr(0x20 + (((A << 4) | ((B >> 4) & 0xF)) & 0x3F)),
-         chr(0x20 + (((B << 2) | ((C >> 6) & 0x3)) & 0x3F)),
-         chr(0x20 + (( C                         ) & 0x3F))])
-              for A, B, C in triples_gen(s)]
-    return chr(ord(' ') + (length & 077)) + ''.join(result) + '\n'
-
-
-table_a2b_base64 = {
-    'A': 0,
-    'B': 1,
-    'C': 2,
-    'D': 3,
-    'E': 4,
-    'F': 5,
-    'G': 6,
-    'H': 7,
-    'I': 8,
-    'J': 9,
-    'K': 10,
-    'L': 11,
-    'M': 12,
-    'N': 13,
-    'O': 14,
-    'P': 15,
-    'Q': 16,
-    'R': 17,
-    'S': 18,
-    'T': 19,
-    'U': 20,
-    'V': 21,
-    'W': 22,
-    'X': 23,
-    'Y': 24,
-    'Z': 25,
-    'a': 26,
-    'b': 27,
-    'c': 28,
-    'd': 29,
-    'e': 30,
-    'f': 31,
-    'g': 32,
-    'h': 33,
-    'i': 34,
-    'j': 35,
-    'k': 36,
-    'l': 37,
-    'm': 38,
-    'n': 39,
-    'o': 40,
-    'p': 41,
-    'q': 42,
-    'r': 43,
-    's': 44,
-    't': 45,
-    'u': 46,
-    'v': 47,
-    'w': 48,
-    'x': 49,
-    'y': 50,
-    'z': 51,
-    '0': 52,
-    '1': 53,
-    '2': 54,
-    '3': 55,
-    '4': 56,
-    '5': 57,
-    '6': 58,
-    '7': 59,
-    '8': 60,
-    '9': 61,
-    '+': 62,
-    '/': 63,
-    '=': 0,
-}
-
-
-def a2b_base64(s):
-    if not isinstance(s, (str, unicode)):
-        raise TypeError("expected string or unicode, got %r" % (s,))
-    s = s.rstrip()
-    # clean out all invalid characters, this also strips the final '=' padding
-    # check for correct padding
-
-    def next_valid_char(s, pos):
-        for i in range(pos + 1, len(s)):
-            c = s[i]
-            if c < '\x7f':
-                try:
-                    table_a2b_base64[c]
-                    return c
-                except KeyError:
-                    pass
-        return None
-    
-    quad_pos = 0
-    leftbits = 0
-    leftchar = 0
-    res = []
-    for i, c in enumerate(s):
-        if c > '\x7f' or c == '\n' or c == '\r' or c == ' ':
-            continue
-        if c == '=':
-            if quad_pos < 2 or (quad_pos == 2 and next_valid_char(s, i) != '='):
-                continue
-            else:
-                leftbits = 0
-                break
-        try:
-            next_c = table_a2b_base64[c]
-        except KeyError:
-            continue
-        quad_pos = (quad_pos + 1) & 0x03
-        leftchar = (leftchar << 6) | next_c
-        leftbits += 6
-        if leftbits >= 8:
-            leftbits -= 8
-            res.append((leftchar >> leftbits & 0xff))
-            leftchar &= ((1 << leftbits) - 1)
-    if leftbits != 0:
-        raise Error('Incorrect padding')
-    
-    return ''.join([chr(i) for i in res])
-    
-table_b2a_base64 = \
-"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
-
-def b2a_base64(s):
-    length = len(s)
-    final_length = length % 3
-
-    def triples_gen(s):
-        while s:
-            try:
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-            except IndexError:
-                s += '\0\0'
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-                return
-            s = s[3:]
-
-    
-    a = triples_gen(s[ :length - final_length])
-
-    result = [''.join(
-        [table_b2a_base64[( A >> 2                    ) & 0x3F],
-         table_b2a_base64[((A << 4) | ((B >> 4) & 0xF)) & 0x3F],
-         table_b2a_base64[((B << 2) | ((C >> 6) & 0x3)) & 0x3F],
-         table_b2a_base64[( C                         ) & 0x3F]])
-              for A, B, C in a]
-
-    final = s[length - final_length:]
-    if final_length == 0:
-        snippet = ''
-    elif final_length == 1:
-        a = ord(final[0])
-        snippet = table_b2a_base64[(a >> 2 ) & 0x3F] + \
-                  table_b2a_base64[(a << 4 ) & 0x3F] + '=='
-    else:
-        a = ord(final[0])
-        b = ord(final[1])
-        snippet = table_b2a_base64[(a >> 2) & 0x3F] + \
-                  table_b2a_base64[((a << 4) | (b >> 4) & 0xF) & 0x3F] + \
-                  table_b2a_base64[(b << 2) & 0x3F] + '='
-    return ''.join(result) + snippet + '\n'
-
-def a2b_qp(s, header=False):
-    inp = 0
-    odata = []
-    while inp < len(s):
-        if s[inp] == '=':
-            inp += 1
-            if inp >= len(s):
-                break
-            # Soft line breaks
-            if (s[inp] == '\n') or (s[inp] == '\r'):
-                if s[inp] != '\n':
-                    while inp < len(s) and s[inp] != '\n':
-                        inp += 1
-                if inp < len(s):
-                    inp += 1
-            elif s[inp] == '=':
-                # broken case from broken python qp
-                odata.append('=')
-                inp += 1
-            elif s[inp] in hex_numbers and s[inp + 1] in hex_numbers:
-                ch = chr(int(s[inp:inp+2], 16))
-                inp += 2
-                odata.append(ch)
-            else:
-                odata.append('=')
-        elif header and s[inp] == '_':
-            odata.append(' ')
-            inp += 1
-        else:
-            odata.append(s[inp])
-            inp += 1
-    return ''.join(odata)
-
-def b2a_qp(data, quotetabs=False, istext=True, header=False):
-    """quotetabs=True means that tab and space characters are always
-       quoted.
-       istext=False means that \r and \n are treated as regular characters
-       header=True encodes space characters with '_' and requires
-       real '_' characters to be quoted.
-    """
-    MAXLINESIZE = 76
-
-    # See if this string is using CRLF line ends
-    lf = data.find('\n')
-    crlf = lf > 0 and data[lf-1] == '\r'
-
-    inp = 0
-    linelen = 0
-    odata = []
-    while inp < len(data):
-        c = data[inp]
-        if (c > '~' or
-            c == '=' or
-            (header and c == '_') or
-            (c == '.' and linelen == 0 and (inp+1 == len(data) or
-                                            data[inp+1] == '\n' or
-                                            data[inp+1] == '\r')) or
-            (not istext and (c == '\r' or c == '\n')) or
-            ((c == '\t' or c == ' ') and (inp + 1 == len(data))) or
-            (c <= ' ' and c != '\r' and c != '\n' and
-             (quotetabs or (not quotetabs and (c != '\t' and c != ' '))))):
-            linelen += 3
-            if linelen >= MAXLINESIZE:
-                odata.append('=')
-                if crlf: odata.append('\r')
-                odata.append('\n')
-                linelen = 3
-            odata.append('=' + two_hex_digits(ord(c)))
-            inp += 1
-        else:
-            if (istext and
-                (c == '\n' or (inp+1 < len(data) and c == '\r' and
-                               data[inp+1] == '\n'))):
-                linelen = 0
-                # Protect against whitespace on end of line
-                if (len(odata) > 0 and
-                    (odata[-1] == ' ' or odata[-1] == '\t')):
-                    ch = ord(odata[-1])
-                    odata[-1] = '='
-                    odata.append(two_hex_digits(ch))
-
-                if crlf: odata.append('\r')
-                odata.append('\n')
-                if c == '\r':
-                    inp += 2
-                else:
-                    inp += 1
-            else:
-                if (inp + 1 < len(data) and
-                    data[inp+1] != '\n' and
-                    (linelen + 1) >= MAXLINESIZE):
-                    odata.append('=')
-                    if crlf: odata.append('\r')
-                    odata.append('\n')
-                    linelen = 0
-
-                linelen += 1
-                if header and c == ' ':
-                    c = '_'
-                odata.append(c)
-                inp += 1
-    return ''.join(odata)
-
-hex_numbers = '0123456789ABCDEF'
-def hex(n):
-    if n == 0:
-        return '0'
-    
-    if n < 0:
-        n = -n
-        sign = '-'
-    else:
-        sign = ''
-    arr = []
-
-    def hex_gen(n):
-        """ Yield a nibble at a time. """
-        while n:
-            yield n % 0x10
-            n = n / 0x10
-
-    for nibble in hex_gen(n):
-        arr = [hex_numbers[nibble]] + arr
-    return sign + ''.join(arr)
-
-def two_hex_digits(n):
-    return hex_numbers[n / 0x10] + hex_numbers[n % 0x10]
-    
-
-def strhex_to_int(s):
-    i = 0
-    for c in s:
-        i = i * 0x10 + hex_numbers.index(c)
-    return i
-
-hqx_encoding = '!"#$%&\'()*+,-012345689 at ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr'
-
-DONE = 0x7f
-SKIP = 0x7e
-FAIL = 0x7d
-    
-table_a2b_hqx = [
-    #^@    ^A    ^B    ^C    ^D    ^E    ^F    ^G   
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #\b    \t    \n    ^K    ^L    \r    ^N    ^O   
-    FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL,
-    #^P    ^Q    ^R    ^S    ^T    ^U    ^V    ^W   
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #^X    ^Y    ^Z    ^[    ^\    ^]    ^^    ^_   
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #      !     "     #     $     %     &     '   
-    FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
-    #(     )     *     +     ,     -     .     /   
-    0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL,
-    #0     1     2     3     4     5     6     7   
-    0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL,
-    #8     9     :     ;     <     =     >     ?   
-    0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #@     A     B     C     D     E     F     G   
-    0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D,
-    #H     I     J     K     L     M     N     O   
-    0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL,
-    #P     Q     R     S     T     U     V     W   
-    0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL,
-    #X     Y     Z     [     \     ]     ^     _   
-    0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL,
-    #`     a     b     c     d     e     f     g   
-    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL,
-    #h     i     j     k     l     m     n     o   
-    0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL,
-    #p     q     r     s     t     u     v     w   
-    0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #x     y     z     {     |     }     ~    ^?   
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-]
-
-def a2b_hqx(s):
-    result = []
-
-    def quadruples_gen(s):
-        t = []
-        for c in s:
-            res = table_a2b_hqx[ord(c)]
-            if res == SKIP:
-                continue
-            elif res == FAIL:
-                raise Error('Illegal character')
-            elif res == DONE:
-                yield t
-                raise Done
-            else:
-                t.append(res)
-            if len(t) == 4:
-                yield t
-                t = []
-        yield t
-        
-    done = 0
-    try:
-        for snippet in quadruples_gen(s):
-            length = len(snippet)
-            if length == 4:
-                result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) 
-                result.append(chr(((snippet[1] & 0x0f) << 4) | (snippet[2] >> 2))) 
-                result.append(chr(((snippet[2] & 0x03) << 6) | (snippet[3]))) 
-            elif length == 3:
-                result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) 
-                result.append(chr(((snippet[1] & 0x0f) << 4) | (snippet[2] >> 2))) 
-            elif length == 2:
-                result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) 
-    except Done:
-        done = 1
-    except Error:
-        raise
-    return (''.join(result), done)
-
-def b2a_hqx(s):
-    result =[]
-
-    def triples_gen(s):
-        while s:
-            try:
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-            except IndexError:
-                yield tuple([ord(c) for c in s])
-            s = s[3:]
-
-    for snippet in triples_gen(s):
-        length = len(snippet)
-        if length == 3:
-            result.append(
-                hqx_encoding[(snippet[0] & 0xfc) >> 2])
-            result.append(hqx_encoding[
-                ((snippet[0] & 0x03) << 4) | ((snippet[1] & 0xf0) >> 4)])
-            result.append(hqx_encoding[
-                (snippet[1] & 0x0f) << 2 | ((snippet[2] & 0xc0) >> 6)])
-            result.append(hqx_encoding[snippet[2] & 0x3f])
-        elif length == 2:
-            result.append(
-                hqx_encoding[(snippet[0] & 0xfc) >> 2])
-            result.append(hqx_encoding[
-                ((snippet[0] & 0x03) << 4) | ((snippet[1] & 0xf0) >> 4)])
-            result.append(hqx_encoding[
-                (snippet[1] & 0x0f) << 2])
-        elif length == 1:
-            result.append(
-                hqx_encoding[(snippet[0] & 0xfc) >> 2])
-            result.append(hqx_encoding[
-                ((snippet[0] & 0x03) << 4)])
-    return ''.join(result)
-
-crctab_hqx = [
-        0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
-        0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
-        0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
-        0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
-        0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
-        0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
-        0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
-        0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
-        0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
-        0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
-        0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
-        0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
-        0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
-        0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
-        0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
-        0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
-        0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
-        0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
-        0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
-        0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
-        0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
-        0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
-        0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
-        0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
-        0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
-        0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
-        0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
-        0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
-        0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
-        0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
-        0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
-        0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
-]
-
-def crc_hqx(s, crc):
-    for c in s:
-        crc = ((crc << 8) & 0xff00) ^ crctab_hqx[((crc >> 8) & 0xff) ^ ord(c)]
-
-    return crc
-
-def rlecode_hqx(s):
-    """
-    Run length encoding for binhex4.
-    The CPython implementation does not do run length encoding
-    of \x90 characters. This implementation does.
-    """
-    if not s:
-        return ''
-    result = []
-    prev = s[0]
-    count = 1
-    # Add a dummy character to get the loop to go one extra round.
-    # The dummy must be different from the last character of s.
-    # In the same step we remove the first character, which has
-    # already been stored in prev.
-    if s[-1] == '!':
-        s = s[1:] + '?'
-    else:
-        s = s[1:] + '!'
-        
-    for c in s:
-        if c == prev and count < 255:
-            count += 1
-        else:
-            if count == 1:
-                if prev != '\x90':
-                    result.append(prev)
-                else:
-                    result.extend(['\x90', '\x00'])
-            elif count < 4:
-                if prev != '\x90':
-                    result.extend([prev] * count)
-                else:
-                    result.extend(['\x90', '\x00'] * count)
-            else:
-                if prev != '\x90':
-                    result.extend([prev, '\x90', chr(count)])
-                else:
-                    result.extend(['\x90', '\x00', '\x90', chr(count)]) 
-            count = 1
-            prev = c
-        
-    return ''.join(result)
-
-def rledecode_hqx(s):
-    s = s.split('\x90')
-    result = [s[0]]
-    prev = s[0]
-    for snippet in s[1:]:
-        count = ord(snippet[0])
-        if count > 0:
-            result.append(prev[-1] * (count-1))
-            prev = snippet
-        else:
-            result. append('\x90')
-            prev = '\x90'
-        result.append(snippet[1:])
-
-    return ''.join(result)
-
-crc_32_tab = [
-    0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L,
-    0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L,
-    0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L,
-    0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL,
-    0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L,
-    0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L,
-    0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L,
-    0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL,
-    0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L,
-    0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL,
-    0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L,
-    0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L,
-    0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L,
-    0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL,
-    0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL,
-    0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L,
-    0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL,
-    0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L,
-    0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L,
-    0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L,
-    0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL,
-    0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L,
-    0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L,
-    0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL,
-    0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L,
-    0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L,
-    0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L,
-    0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L,
-    0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L,
-    0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL,
-    0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL,
-    0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L,
-    0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L,
-    0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL,
-    0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL,
-    0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L,
-    0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL,
-    0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L,
-    0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL,
-    0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L,
-    0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL,
-    0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L,
-    0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L,
-    0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL,
-    0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L,
-    0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L,
-    0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L,
-    0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L,
-    0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L,
-    0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L,
-    0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL,
-    0x2d02ef8dL
-]
-
-def crc32(s, crc=0):
-    result = 0
-    crc = ~long(crc) & 0xffffffffL
-    for c in s:
-        crc = crc_32_tab[(crc ^ long(ord(c))) & 0xffL] ^ (crc >> 8)
-        #/* Note:  (crc >> 8) MUST zero fill on left
-
-    result = crc ^ 0xffffffffL
-    
-    if result > 2**31:
-        result = ((result + 2**31) % 2**32) - 2**31
-
-    return result
-
-def b2a_hex(s):
-    result = []
-    for char in s:
-        c = (ord(char) >> 4) & 0xf
-        if c > 9:
-            c = c + ord('a') - 10
-        else:
-            c = c + ord('0')
-        result.append(chr(c))
-        c = ord(char) & 0xf
-        if c > 9:
-            c = c + ord('a') - 10
-        else:
-            c = c + ord('0')
-        result.append(chr(c))
-    return ''.join(result)
-
-hexlify = b2a_hex
-
-table_hex = [
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    0, 1, 2, 3,  4, 5, 6, 7,  8, 9,-1,-1, -1,-1,-1,-1,
-    -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1
-]
-
-
-def a2b_hex(t):
-    result = []
-
-    def pairs_gen(s):
-        while s:
-            try:
-                yield table_hex[ord(s[0])], table_hex[ord(s[1])]
-            except IndexError:
-                if len(s):
-                    raise TypeError('Odd-length string')
-                return
-            s = s[2:]
-
-    for a, b in pairs_gen(t):
-        if a < 0 or b < 0:
-            raise TypeError('Non-hexadecimal digit found')
-        result.append(chr((a << 4) + b))
-    return ''.join(result)
-    
-
-unhexlify = a2b_hex
diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py
--- a/lib_pypy/cPickle.py
+++ b/lib_pypy/cPickle.py
@@ -2,16 +2,95 @@
 # One-liner implementation of cPickle
 #
 
-from pickle import *
+from pickle import Pickler, dump, dumps, PickleError, PicklingError, UnpicklingError, _EmptyClass
 from pickle import __doc__, __version__, format_version, compatible_formats
+from types import *
+from copy_reg import dispatch_table
+from copy_reg import _extension_registry, _inverted_registry, _extension_cache
+import marshal, struct, sys
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
 
+# These are purely informational; no code uses these.
+format_version = "2.0"                  # File format version we write
+compatible_formats = ["1.0",            # Original protocol 0
+                      "1.1",            # Protocol 0 with INST added
+                      "1.2",            # Original protocol 1
+                      "1.3",            # Protocol 1 with BINFLOAT added
+                      "2.0",            # Protocol 2
+                      ]                 # Old format versions we can read
+
+# Keep in synch with cPickle.  This is the highest protocol number we
+# know how to read.
+HIGHEST_PROTOCOL = 2
 
 BadPickleGet = KeyError
 UnpickleableError = PicklingError
 
+MARK            = ord('(')   # push special markobject on stack
+STOP            = ord('.')   # every pickle ends with STOP
+POP             = ord('0')   # discard topmost stack item
+POP_MARK        = ord('1')   # discard stack top through topmost markobject
+DUP             = ord('2')   # duplicate top stack item
+FLOAT           = ord('F')   # push float object; decimal string argument
+INT             = ord('I')   # push integer or bool; decimal string argument
+BININT          = ord('J')   # push four-byte signed int
+BININT1         = ord('K')   # push 1-byte unsigned int
+LONG            = ord('L')   # push long; decimal string argument
+BININT2         = ord('M')   # push 2-byte unsigned int
+NONE            = ord('N')   # push None
+PERSID          = ord('P')   # push persistent object; id is taken from string arg
+BINPERSID       = ord('Q')   #  "       "         "  ;  "  "   "     "  stack
+REDUCE          = ord('R')   # apply callable to argtuple, both on stack
+STRING          = ord('S')   # push string; NL-terminated string argument
+BINSTRING       = ord('T')   # push string; counted binary string argument
+SHORT_BINSTRING = ord('U')   #  "     "   ;    "      "       "      " < 256 bytes
+UNICODE         = ord('V')   # push Unicode string; raw-unicode-escaped'd argument
+BINUNICODE      = ord('X')   #   "     "       "  ; counted UTF-8 string argument
+APPEND          = ord('a')   # append stack top to list below it
+BUILD           = ord('b')   # call __setstate__ or __dict__.update()
+GLOBAL          = ord('c')   # push self.find_class(modname, name); 2 string args
+DICT            = ord('d')   # build a dict from stack items
+EMPTY_DICT      = ord('}')   # push empty dict
+APPENDS         = ord('e')   # extend list on stack by topmost stack slice
+GET             = ord('g')   # push item from memo on stack; index is string arg
+BINGET          = ord('h')   #   "    "    "    "   "   "  ;   "    " 1-byte arg
+INST            = ord('i')   # build & push class instance
+LONG_BINGET     = ord('j')   # push item from memo on stack; index is 4-byte arg
+LIST            = ord('l')   # build list from topmost stack items
+EMPTY_LIST      = ord(']')   # push empty list
+OBJ             = ord('o')   # build & push class instance
+PUT             = ord('p')   # store stack top in memo; index is string arg
+BINPUT          = ord('q')   #   "     "    "   "   " ;   "    " 1-byte arg
+LONG_BINPUT     = ord('r')   #   "     "    "   "   " ;   "    " 4-byte arg
+SETITEM         = ord('s')   # add key+value pair to dict
+TUPLE           = ord('t')   # build tuple from topmost stack items
+EMPTY_TUPLE     = ord(')')   # push empty tuple
+SETITEMS        = ord('u')   # modify dict by adding topmost key+value pairs
+BINFLOAT        = ord('G')   # push float; arg is 8-byte float encoding
+
+TRUE            = 'I01\n'  # not an opcode; see INT docs in pickletools.py
+FALSE           = 'I00\n'  # not an opcode; see INT docs in pickletools.py
+
+# Protocol 2
+
+PROTO           = ord('\x80')  # identify pickle protocol
+NEWOBJ          = ord('\x81')  # build object by applying cls.__new__ to argtuple
+EXT1            = ord('\x82')  # push object from extension registry; 1-byte index
+EXT2            = ord('\x83')  # ditto, but 2-byte index
+EXT4            = ord('\x84')  # ditto, but 4-byte index
+TUPLE1          = ord('\x85')  # build 1-tuple from stack top
+TUPLE2          = ord('\x86')  # build 2-tuple from two topmost stack items
+TUPLE3          = ord('\x87')  # build 3-tuple from three topmost stack items
+NEWTRUE         = ord('\x88')  # push True
+NEWFALSE        = ord('\x89')  # push False
+LONG1           = ord('\x8a')  # push long from < 256 bytes
+LONG4           = ord('\x8b')  # push really big long
+
+_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
+
+
 # ____________________________________________________________
 # XXX some temporary dark magic to produce pickled dumps that are
 #     closer to the ones produced by cPickle in CPython
@@ -44,3 +123,474 @@
     file = StringIO()
     Pickler(file, protocol).dump(obj)
     return file.getvalue()
+
+# Why use struct.pack() for pickling but marshal.loads() for
+# unpickling?  struct.pack() is 40% faster than marshal.dumps(), but
+# marshal.loads() is twice as fast as struct.unpack()!
+mloads = marshal.loads
+
+# Unpickling machinery
+
+class Unpickler(object):
+
+    def __init__(self, file):
+        """This takes a file-like object for reading a pickle data stream.
+
+        The protocol version of the pickle is detected automatically, so no
+        proto argument is needed.
+
+        The file-like object must have two methods, a read() method that
+        takes an integer argument, and a readline() method that requires no
+        arguments.  Both methods should return a string.  Thus file-like
+        object can be a file object opened for reading, a StringIO object,
+        or any other custom object that meets this interface.
+        """
+        self.readline = file.readline
+        self.read = file.read
+        self.memo = {}
+
+    def load(self):
+        """Read a pickled object representation from the open file.
+
+        Return the reconstituted object hierarchy specified in the file.
+        """
+        self.mark = object() # any new unique object
+        self.stack = []
+        self.append = self.stack.append
+        try:
+            key = ord(self.read(1))
+            while key != STOP:
+                self.dispatch[key](self)
+                key = ord(self.read(1))
+        except TypeError:
+            if self.read(1) == '':
+                raise EOFError
+            raise
+        return self.stack.pop()
+
+    # Return largest index k such that self.stack[k] is self.mark.
+    # If the stack doesn't contain a mark, eventually raises IndexError.
+    # This could be sped by maintaining another stack, of indices at which
+    # the mark appears.  For that matter, the latter stack would suffice,
+    # and we wouldn't need to push mark objects on self.stack at all.
+    # Doing so is probably a good thing, though, since if the pickle is
+    # corrupt (or hostile) we may get a clue from finding self.mark embedded
+    # in unpickled objects.
+    def marker(self):
+        k = len(self.stack)-1
+        while self.stack[k] is not self.mark: k -= 1
+        return k
+
+    dispatch = {}
+
+    def load_proto(self):
+        proto = ord(self.read(1))
+        if not 0 <= proto <= 2:
+            raise ValueError, "unsupported pickle protocol: %d" % proto
+    dispatch[PROTO] = load_proto
+
+    def load_persid(self):
+        pid = self.readline()[:-1]
+        self.append(self.persistent_load(pid))
+    dispatch[PERSID] = load_persid
+
+    def load_binpersid(self):
+        pid = self.stack.pop()
+        self.append(self.persistent_load(pid))
+    dispatch[BINPERSID] = load_binpersid
+
+    def load_none(self):
+        self.append(None)
+    dispatch[NONE] = load_none
+
+    def load_false(self):
+        self.append(False)
+    dispatch[NEWFALSE] = load_false
+
+    def load_true(self):
+        self.append(True)
+    dispatch[NEWTRUE] = load_true
+
+    def load_int(self):
+        data = self.readline()
+        if data == FALSE[1:]:
+            val = False
+        elif data == TRUE[1:]:
+            val = True
+        else:
+            try:
+                val = int(data)
+            except ValueError:
+                val = long(data)
+        self.append(val)
+    dispatch[INT] = load_int
+
+    def load_binint(self):
+        self.append(mloads('i' + self.read(4)))
+    dispatch[BININT] = load_binint
+
+    def load_binint1(self):
+        self.append(ord(self.read(1)))
+    dispatch[BININT1] = load_binint1
+
+    def load_binint2(self):
+        self.append(mloads('i' + self.read(2) + '\000\000'))
+    dispatch[BININT2] = load_binint2
+
+    def load_long(self):
+        self.append(long(self.readline()[:-1], 0))
+    dispatch[LONG] = load_long
+
+    def load_long1(self):
+        n = ord(self.read(1))
+        bytes = self.read(n)
+        self.append(decode_long(bytes))
+    dispatch[LONG1] = load_long1
+
+    def load_long4(self):
+        n = mloads('i' + self.read(4))
+        bytes = self.read(n)
+        self.append(decode_long(bytes))
+    dispatch[LONG4] = load_long4
+
+    def load_float(self):
+        self.append(float(self.readline()[:-1]))
+    dispatch[FLOAT] = load_float
+
+    def load_binfloat(self, unpack=struct.unpack):
+        self.append(unpack('>d', self.read(8))[0])
+    dispatch[BINFLOAT] = load_binfloat
+
+    def load_string(self):
+        rep = self.readline()
+        if len(rep) < 3:
+            raise ValueError, "insecure string pickle"
+        if rep[0] == "'" == rep[-2]:
+            rep = rep[1:-2]
+        elif rep[0] == '"' == rep[-2]:
+            rep = rep[1:-2]
+        else:
+            raise ValueError, "insecure string pickle"
+        self.append(rep.decode("string-escape"))
+    dispatch[STRING] = load_string
+
+    def load_binstring(self):
+        L = mloads('i' + self.read(4))
+        self.append(self.read(L))
+    dispatch[BINSTRING] = load_binstring
+
+    def load_unicode(self):
+        self.append(unicode(self.readline()[:-1],'raw-unicode-escape'))
+    dispatch[UNICODE] = load_unicode
+
+    def load_binunicode(self):
+        L = mloads('i' + self.read(4))
+        self.append(unicode(self.read(L),'utf-8'))
+    dispatch[BINUNICODE] = load_binunicode
+
+    def load_short_binstring(self):
+        L = ord(self.read(1))
+        self.append(self.read(L))
+    dispatch[SHORT_BINSTRING] = load_short_binstring
+
+    def load_tuple(self):
+        k = self.marker()
+        self.stack[k:] = [tuple(self.stack[k+1:])]
+    dispatch[TUPLE] = load_tuple
+
+    def load_empty_tuple(self):
+        self.stack.append(())
+    dispatch[EMPTY_TUPLE] = load_empty_tuple
+
+    def load_tuple1(self):
+        self.stack[-1] = (self.stack[-1],)
+    dispatch[TUPLE1] = load_tuple1
+
+    def load_tuple2(self):
+        self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
+    dispatch[TUPLE2] = load_tuple2
+
+    def load_tuple3(self):
+        self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
+    dispatch[TUPLE3] = load_tuple3
+
+    def load_empty_list(self):
+        self.stack.append([])
+    dispatch[EMPTY_LIST] = load_empty_list
+
+    def load_empty_dictionary(self):
+        self.stack.append({})
+    dispatch[EMPTY_DICT] = load_empty_dictionary
+
+    def load_list(self):
+        k = self.marker()
+        self.stack[k:] = [self.stack[k+1:]]
+    dispatch[LIST] = load_list
+
+    def load_dict(self):
+        k = self.marker()
+        d = {}
+        items = self.stack[k+1:]
+        for i in range(0, len(items), 2):
+            key = items[i]
+            value = items[i+1]
+            d[key] = value
+        self.stack[k:] = [d]
+    dispatch[DICT] = load_dict
+
+    # INST and OBJ differ only in how they get a class object.  It's not
+    # only sensible to do the rest in a common routine, the two routines
+    # previously diverged and grew different bugs.
+    # klass is the class to instantiate, and k points to the topmost mark
+    # object, following which are the arguments for klass.__init__.
+    def _instantiate(self, klass, k):
+        args = tuple(self.stack[k+1:])
+        del self.stack[k:]
+        instantiated = 0
+        if (not args and
+                type(klass) is ClassType and
+                not hasattr(klass, "__getinitargs__")):
+            try:
+                value = _EmptyClass()
+                value.__class__ = klass
+                instantiated = 1
+            except RuntimeError:
+                # In restricted execution, assignment to inst.__class__ is
+                # prohibited
+                pass
+        if not instantiated:
+            try:
+                value = klass(*args)
+            except TypeError, err:
+                raise TypeError, "in constructor for %s: %s" % (
+                    klass.__name__, str(err)), sys.exc_info()[2]
+        self.append(value)
+
+    def load_inst(self):
+        module = self.readline()[:-1]
+        name = self.readline()[:-1]
+        klass = self.find_class(module, name)
+        self._instantiate(klass, self.marker())
+    dispatch[INST] = load_inst
+
+    def load_obj(self):
+        # Stack is ... markobject classobject arg1 arg2 ...
+        k = self.marker()
+        klass = self.stack.pop(k+1)
+        self._instantiate(klass, k)
+    dispatch[OBJ] = load_obj
+
+    def load_newobj(self):
+        args = self.stack.pop()
+        cls = self.stack[-1]
+        obj = cls.__new__(cls, *args)
+        self.stack[-1] = obj
+    dispatch[NEWOBJ] = load_newobj
+
+    def load_global(self):
+        module = self.readline()[:-1]
+        name = self.readline()[:-1]
+        klass = self.find_class(module, name)
+        self.append(klass)
+    dispatch[GLOBAL] = load_global
+
+    def load_ext1(self):
+        code = ord(self.read(1))
+        self.get_extension(code)
+    dispatch[EXT1] = load_ext1
+
+    def load_ext2(self):
+        code = mloads('i' + self.read(2) + '\000\000')
+        self.get_extension(code)
+    dispatch[EXT2] = load_ext2
+
+    def load_ext4(self):
+        code = mloads('i' + self.read(4))
+        self.get_extension(code)
+    dispatch[EXT4] = load_ext4
+
+    def get_extension(self, code):
+        nil = []
+        obj = _extension_cache.get(code, nil)
+        if obj is not nil:
+            self.append(obj)
+            return
+        key = _inverted_registry.get(code)
+        if not key:
+            raise ValueError("unregistered extension code %d" % code)
+        obj = self.find_class(*key)
+        _extension_cache[code] = obj
+        self.append(obj)
+
+    def find_class(self, module, name):
+        # Subclasses may override this
+        __import__(module)
+        mod = sys.modules[module]
+        klass = getattr(mod, name)
+        return klass
+
+    def load_reduce(self):
+        args = self.stack.pop()
+        func = self.stack[-1]
+        value = self.stack[-1](*args)
+        self.stack[-1] = value
+    dispatch[REDUCE] = load_reduce
+
+    def load_pop(self):
+        del self.stack[-1]
+    dispatch[POP] = load_pop
+
+    def load_pop_mark(self):
+        k = self.marker()
+        del self.stack[k:]
+    dispatch[POP_MARK] = load_pop_mark
+
+    def load_dup(self):
+        self.append(self.stack[-1])
+    dispatch[DUP] = load_dup
+
+    def load_get(self):
+        self.append(self.memo[self.readline()[:-1]])
+    dispatch[GET] = load_get
+
+    def load_binget(self):
+        i = ord(self.read(1))
+        self.append(self.memo[repr(i)])
+    dispatch[BINGET] = load_binget
+
+    def load_long_binget(self):
+        i = mloads('i' + self.read(4))
+        self.append(self.memo[repr(i)])
+    dispatch[LONG_BINGET] = load_long_binget
+
+    def load_put(self):
+        self.memo[self.readline()[:-1]] = self.stack[-1]
+    dispatch[PUT] = load_put
+
+    def load_binput(self):
+        i = ord(self.read(1))
+        self.memo[repr(i)] = self.stack[-1]
+    dispatch[BINPUT] = load_binput
+
+    def load_long_binput(self):
+        i = mloads('i' + self.read(4))
+        self.memo[repr(i)] = self.stack[-1]
+    dispatch[LONG_BINPUT] = load_long_binput
+
+    def load_append(self):
+        value = self.stack.pop()
+        self.stack[-1].append(value)
+    dispatch[APPEND] = load_append
+
+    def load_appends(self):
+        stack = self.stack
+        mark = self.marker()
+        lst = stack[mark - 1]
+        lst.extend(stack[mark + 1:])
+        del stack[mark:]
+    dispatch[APPENDS] = load_appends
+
+    def load_setitem(self):
+        stack = self.stack
+        value = stack.pop()
+        key = stack.pop()
+        dict = stack[-1]
+        dict[key] = value
+    dispatch[SETITEM] = load_setitem
+
+    def load_setitems(self):
+        stack = self.stack
+        mark = self.marker()
+        dict = stack[mark - 1]
+        for i in range(mark + 1, len(stack), 2):
+            dict[stack[i]] = stack[i + 1]
+
+        del stack[mark:]
+    dispatch[SETITEMS] = load_setitems
+
+    def load_build(self):
+        stack = self.stack
+        state = stack.pop()
+        inst = stack[-1]
+        setstate = getattr(inst, "__setstate__", None)
+        if setstate:
+            setstate(state)
+            return
+        slotstate = None
+        if isinstance(state, tuple) and len(state) == 2:
+            state, slotstate = state
+        if state:
+            try:
+                d = inst.__dict__
+                try:
+                    for k, v in state.iteritems():
+                        d[intern(k)] = v
+                # keys in state don't have to be strings
+                # don't blow up, but don't go out of our way
+                except TypeError:
+                    d.update(state)
+
+            except RuntimeError:
+                # XXX In restricted execution, the instance's __dict__
+                # is not accessible.  Use the old way of unpickling
+                # the instance variables.  This is a semantic
+                # difference when unpickling in restricted
+                # vs. unrestricted modes.
+                # Note, however, that cPickle has never tried to do the
+                # .update() business, and always uses
+                #     PyObject_SetItem(inst.__dict__, key, value) in a
+                # loop over state.items().
+                for k, v in state.items():
+                    setattr(inst, k, v)
+        if slotstate:
+            for k, v in slotstate.items():
+                setattr(inst, k, v)
+    dispatch[BUILD] = load_build
+
+    def load_mark(self):
+        self.append(self.mark)
+    dispatch[MARK] = load_mark
+
+#from pickle import decode_long
+
+def decode_long(data):
+    r"""Decode a long from a two's complement little-endian binary string.
+
+    >>> decode_long('')
+    0L
+    >>> decode_long("\xff\x00")
+    255L
+    >>> decode_long("\xff\x7f")
+    32767L
+    >>> decode_long("\x00\xff")
+    -256L
+    >>> decode_long("\x00\x80")
+    -32768L
+    >>> decode_long("\x80")
+    -128L
+    >>> decode_long("\x7f")
+    127L
+    """
+
+    nbytes = len(data)
+    if nbytes == 0:
+        return 0L
+    ind = nbytes - 1
+    while ind and ord(data[ind]) == 0:
+        ind -= 1
+    n = ord(data[ind])
+    while ind:
+        n <<= 8
+        ind -= 1
+        if ord(data[ind]):
+            n += ord(data[ind])
+    if ord(data[nbytes - 1]) >= 128:
+        n -= 1L << (nbytes << 3)
+    return n
+
+def load(f):
+    return Unpickler(f).load()
+
+def loads(str):
+    f = StringIO(str)
+    return Unpickler(f).load()
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -968,8 +968,7 @@
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
+        return NotImplemented    # note that this doesn't work on CPython 2.2
 
     __radd__ = __add__
 
@@ -1032,8 +1031,8 @@
     def __setstate(self, string):
         if len(string) != 4 or not (1 <= ord(string[2]) <= 12):
             raise TypeError("not enough arguments")
-        yhi, ylo, self._month, self._day = map(ord, string)
-        self._year = yhi * 256 + ylo
+        self._month, self._day = ord(string[2]), ord(string[3])
+        self._year = ord(string[0]) * 256 + ord(string[1])
 
     def __reduce__(self):
         return (self.__class__, self._getstate())
@@ -1421,9 +1420,10 @@
     def __setstate(self, string, tzinfo):
         if len(string) != 6 or ord(string[0]) >= 24:
             raise TypeError("an integer is required")
-        self._hour, self._minute, self._second, us1, us2, us3 = \
-                                                            map(ord, string)
-        self._microsecond = (((us1 << 8) | us2) << 8) | us3
+        self._hour, self._minute, self._second = ord(string[0]), \
+                                                 ord(string[1]), ord(string[2])
+        self._microsecond = (((ord(string[3]) << 8) | \
+                            ord(string[4])) << 8) | ord(string[5])
         self._tzinfo = tzinfo
 
     def __reduce__(self):
@@ -1903,10 +1903,11 @@
             return (basestate, self._tzinfo)
 
     def __setstate(self, string, tzinfo):
-        (yhi, ylo, self._month, self._day, self._hour,
-         self._minute, self._second, us1, us2, us3) = map(ord, string)
-        self._year = yhi * 256 + ylo
-        self._microsecond = (((us1 << 8) | us2) << 8) | us3
+        (self._month, self._day, self._hour, self._minute,
+            self._second) = (ord(string[2]), ord(string[3]), ord(string[4]),
+                             ord(string[5]), ord(string[6]))
+        self._year = ord(string[0]) * 256 + ord(string[1])
+        self._microsecond = (((ord(string[7]) << 8) | ord(string[8])) << 8) | ord(string[9])
         self._tzinfo = tzinfo
 
     def __reduce__(self):
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -6,7 +6,7 @@
 import _numpypy as multiarray # ARGH
 from numpypy.core.arrayprint import array2string
 
-
+newaxis = None
 
 def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
     """
@@ -306,6 +306,125 @@
     else:
         return multiarray.set_string_function(f, repr)
 
+def array_equal(a1, a2):
+    """
+    True if two arrays have the same shape and elements, False otherwise.
+
+    Parameters
+    ----------
+    a1, a2 : array_like
+        Input arrays.
+
+    Returns
+    -------
+    b : bool
+        Returns True if the arrays are equal.
+
+    See Also
+    --------
+    allclose: Returns True if two arrays are element-wise equal within a
+              tolerance.
+    array_equiv: Returns True if input arrays are shape consistent and all
+                 elements equal.
+
+    Examples
+    --------
+    >>> np.array_equal([1, 2], [1, 2])
+    True
+    >>> np.array_equal(np.array([1, 2]), np.array([1, 2]))
+    True
+    >>> np.array_equal([1, 2], [1, 2, 3])
+    False
+    >>> np.array_equal([1, 2], [1, 4])
+    False
+
+    """
+    try:
+        a1, a2 = asarray(a1), asarray(a2)
+    except:
+        return False
+    if a1.shape != a2.shape:
+        return False
+    return bool((a1 == a2).all())
+
+def asarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
+    """
+    Convert the input to an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data, in any form that can be converted to an array.  This
+        includes lists, lists of tuples, tuples, tuples of tuples, tuples
+        of lists and ndarrays.
+    dtype : data-type, optional
+        By default, the data-type is inferred from the input data.
+    order : {'C', 'F'}, optional
+        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
+        memory representation.  Defaults to 'C'.
+   maskna : bool or None, optional
+        If this is set to True, it forces the array to have an NA mask.
+        If this is set to False, it forces the array to not have an NA
+        mask.
+    ownmaskna : bool, optional
+        If this is set to True, forces the array to have a mask which
+        it owns.
+
+    Returns
+    -------
+    out : ndarray
+        Array interpretation of `a`.  No copy is performed if the input
+        is already an ndarray.  If `a` is a subclass of ndarray, a base
+        class ndarray is returned.
+
+    See Also
+    --------
+    asanyarray : Similar function which passes through subclasses.
+    ascontiguousarray : Convert input to a contiguous array.
+    asfarray : Convert input to a floating point ndarray.
+    asfortranarray : Convert input to an ndarray with column-major
+                     memory order.
+    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
+    fromiter : Create an array from an iterator.
+    fromfunction : Construct an array by executing a function on grid
+                   positions.
+
+    Examples
+    --------
+    Convert a list into an array:
+
+    >>> a = [1, 2]
+    >>> np.asarray(a)
+    array([1, 2])
+
+    Existing arrays are not copied:
+
+    >>> a = np.array([1, 2])
+    >>> np.asarray(a) is a
+    True
+
+    If `dtype` is set, array is copied only if dtype does not match:
+
+    >>> a = np.array([1, 2], dtype=np.float32)
+    >>> np.asarray(a, dtype=np.float32) is a
+    True
+    >>> np.asarray(a, dtype=np.float64) is a
+    False
+
+    Contrary to `asanyarray`, ndarray subclasses are not passed through:
+
+    >>> issubclass(np.matrix, np.ndarray)
+    True
+    >>> a = np.matrix([[1, 2]])
+    >>> np.asarray(a) is a
+    False
+    >>> np.asanyarray(a) is a
+    True
+
+    """
+    return array(a, dtype, copy=False, order=order,
+                            maskna=maskna, ownmaskna=ownmaskna)
+
 set_string_function(array_str, 0)
 set_string_function(array_repr, 1)
 
@@ -319,4 +438,4 @@
 False_ = bool_(False)
 True_ = bool_(True)
 e = math.e
-pi = math.pi
\ No newline at end of file
+pi = math.pi
diff --git a/lib_pypy/pypy_test/test_binascii.py b/lib_pypy/pypy_test/test_binascii.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_binascii.py
+++ /dev/null
@@ -1,168 +0,0 @@
-from __future__ import absolute_import
-import py
-from lib_pypy import binascii
-
-# Create binary test data
-data = "The quick brown fox jumps over the lazy dog.\r\n"
-# Be slow so we don't depend on other modules
-data += "".join(map(chr, xrange(256)))
-data += "\r\nHello world.\n"
-
-def test_exceptions():
-    # Check module exceptions
-    assert issubclass(binascii.Error, Exception)
-    assert issubclass(binascii.Incomplete, Exception)
-
-def test_functions():
-    # Check presence of all functions
-    funcs = []
-    for suffix in "base64", "hqx", "uu", "hex":
-        prefixes = ["a2b_", "b2a_"]
-        if suffix == "hqx":
-            prefixes.extend(["crc_", "rlecode_", "rledecode_"])
-        for prefix in prefixes:
-            name = prefix + suffix
-            assert callable(getattr(binascii, name))
-            py.test.raises(TypeError, getattr(binascii, name))
-    for name in ("hexlify", "unhexlify"):
-        assert callable(getattr(binascii, name))
-        py.test.raises(TypeError, getattr(binascii, name))
-
-def test_base64valid():
-    # Test base64 with valid data
-    MAX_BASE64 = 57
-    lines = []
-    for i in range(0, len(data), MAX_BASE64):
-        b = data[i:i+MAX_BASE64]
-        a = binascii.b2a_base64(b)
-        lines.append(a)
-    res = ""
-    for line in lines:
-        b = binascii.a2b_base64(line)
-        res = res + b
-    assert res == data
-
-def test_base64invalid():
-    # Test base64 with random invalid characters sprinkled throughout
-    # (This requires a new version of binascii.)
-    MAX_BASE64 = 57
-    lines = []
-    for i in range(0, len(data), MAX_BASE64):
-        b = data[i:i+MAX_BASE64]
-        a = binascii.b2a_base64(b)
-        lines.append(a)
-
-    fillers = ""
-    valid = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/"
-    for i in xrange(256):
-        c = chr(i)
-        if c not in valid:
-            fillers += c
-    def addnoise(line):
-        noise = fillers
-        ratio = len(line) // len(noise)
-        res = ""
-        while line and noise:
-            if len(line) // len(noise) > ratio:
-                c, line = line[0], line[1:]
-            else:
-                c, noise = noise[0], noise[1:]
-            res += c
-        return res + noise + line
-    res = ""
-    for line in map(addnoise, lines):
-        b = binascii.a2b_base64(line)
-        res += b
-    assert res == data
-
-    # Test base64 with just invalid characters, which should return
-    # empty strings. TBD: shouldn't it raise an exception instead ?
-    assert binascii.a2b_base64(fillers) == ''
-
-def test_uu():
-    MAX_UU = 45
-    lines = []
-    for i in range(0, len(data), MAX_UU):
-        b = data[i:i+MAX_UU]
-        a = binascii.b2a_uu(b)
-        lines.append(a)
-    res = ""
-    for line in lines:
-        b = binascii.a2b_uu(line)
-        res += b
-    assert res == data
-
-    assert binascii.a2b_uu("\x7f") == "\x00"*31
-    assert binascii.a2b_uu("\x80") == "\x00"*32
-    assert binascii.a2b_uu("\xff") == "\x00"*31
-    py.test.raises(binascii.Error, binascii.a2b_uu, "\xff\x00")
-    py.test.raises(binascii.Error, binascii.a2b_uu, "!!!!")
-
-    py.test.raises(binascii.Error, binascii.b2a_uu, 46*"!")
-
-def test_crc32():
-    crc = binascii.crc32("Test the CRC-32 of")
-    crc = binascii.crc32(" this string.", crc)
-    assert crc == 1571220330
-    
-    crc = binascii.crc32('frotz\n', 0)
-    assert crc == -372923920
-
-    py.test.raises(TypeError, binascii.crc32)
-
-def test_hex():
-    # test hexlification
-    s = '{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'
-    t = binascii.b2a_hex(s)
-    u = binascii.a2b_hex(t)
-    assert s == u
-    py.test.raises(TypeError, binascii.a2b_hex, t[:-1])
-    py.test.raises(TypeError, binascii.a2b_hex, t[:-1] + 'q')
-
-    # Verify the treatment of Unicode strings
-    assert binascii.hexlify(unicode('a', 'ascii')) == '61'
-
-def test_qp():
-    # A test for SF bug 534347 (segfaults without the proper fix)
-    try:
-        binascii.a2b_qp("", **{1:1})
-    except TypeError:
-        pass
-    else:
-        fail("binascii.a2b_qp(**{1:1}) didn't raise TypeError")
-    assert binascii.a2b_qp("= ") == "= "
-    assert binascii.a2b_qp("==") == "="
-    assert binascii.a2b_qp("=AX") == "=AX"
-    py.test.raises(TypeError, binascii.b2a_qp, foo="bar")
-    assert binascii.a2b_qp("=00\r\n=00") == "\x00\r\n\x00"
-    assert binascii.b2a_qp("\xff\r\n\xff\n\xff") == "=FF\r\n=FF\r\n=FF"
-    target = "0"*75+"=\r\n=FF\r\n=FF\r\n=FF"
-    assert binascii.b2a_qp("0"*75+"\xff\r\n\xff\r\n\xff") == target
-
-def test_empty_string():
-    # A test for SF bug #1022953.  Make sure SystemError is not raised.
-    for n in ['b2a_qp', 'a2b_hex', 'b2a_base64', 'a2b_uu', 'a2b_qp',
-              'b2a_hex', 'unhexlify', 'hexlify', 'crc32', 'b2a_hqx',
-              'a2b_hqx', 'a2b_base64', 'rlecode_hqx', 'b2a_uu',
-              'rledecode_hqx']:
-        f = getattr(binascii, n)
-        f('')
-    binascii.crc_hqx('', 0)
-
-def test_qp_bug_case():
-    assert binascii.b2a_qp('y'*77, False, False) == 'y'*75 + '=\nyy'
-    assert binascii.b2a_qp(' '*77, False, False) == ' '*75 + '=\n =20'
-    assert binascii.b2a_qp('y'*76, False, False) == 'y'*76
-    assert binascii.b2a_qp(' '*76, False, False) == ' '*75 + '=\n=20'
-
-def test_wrong_padding():
-    s = 'CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3IQ'
-    py.test.raises(binascii.Error, binascii.a2b_base64, s)
-
-def test_crap_after_padding():
-    s = 'xxx=axxxx'
-    assert binascii.a2b_base64(s) == '\xc7\x1c'
-
-def test_wrong_args():
-    # this should grow as a way longer list
-    py.test.raises(TypeError, binascii.a2b_base64, 42)
diff --git a/lib_pypy/pypy_test/test_locale.py b/lib_pypy/pypy_test/test_locale.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_locale.py
+++ /dev/null
@@ -1,79 +0,0 @@
-from __future__ import absolute_import
-import py
-import sys
-
-from lib_pypy.ctypes_config_cache import rebuild
-rebuild.rebuild_one('locale.ctc.py')
-
-from lib_pypy import _locale
-
-
-def setup_module(mod):
-    if sys.platform == 'darwin':
-        py.test.skip("Locale support on MacOSX is minimal and cannot be tested")
-
-class TestLocale:
-    def setup_class(cls):
-        cls.oldlocale = _locale.setlocale(_locale.LC_NUMERIC)
-        if sys.platform.startswith("win"):
-            cls.tloc = "en"
-        elif sys.platform.startswith("freebsd"):
-            cls.tloc = "en_US.US-ASCII"
-        else:
-            cls.tloc = "en_US.UTF8"
-        try:
-            _locale.setlocale(_locale.LC_NUMERIC, cls.tloc)
-        except _locale.Error:
-            py.test.skip("test locale %s not supported" % cls.tloc)
-            
-    def teardown_class(cls):
-        _locale.setlocale(_locale.LC_NUMERIC, cls.oldlocale)
-
-    def test_format(self):
-        py.test.skip("XXX fix or kill me")
-
-        def testformat(formatstr, value, grouping = 0, output=None):
-            if output:
-                print "%s %% %s =? %s ..." %\
-                      (repr(formatstr), repr(value), repr(output)),
-            else:
-                print "%s %% %s works? ..." % (repr(formatstr), repr(value)),
-            result = locale.format(formatstr, value, grouping = grouping)
-            assert result == output
-
-        testformat("%f", 1024, grouping=1, output='1,024.000000')
-        testformat("%f", 102, grouping=1, output='102.000000')
-        testformat("%f", -42, grouping=1, output='-42.000000')
-        testformat("%+f", -42, grouping=1, output='-42.000000')
-        testformat("%20.f", -42, grouping=1, output='                 -42')
-        testformat("%+10.f", -4200, grouping=1, output='    -4,200')
-        testformat("%-10.f", 4200, grouping=1, output='4,200     ')
-
-    def test_getpreferredencoding(self):
-        py.test.skip("XXX fix or kill me")
-        # Invoke getpreferredencoding to make sure it does not cause exceptions
-        _locale.getpreferredencoding()
-
-    # Test BSD Rune locale's bug for isctype functions.
-    def test_bsd_bug(self):
-        def teststrop(s, method, output):
-            print "%s.%s() =? %s ..." % (repr(s), method, repr(output)),
-            result = getattr(s, method)()
-            assert result == output
-
-        oldlocale = _locale.setlocale(_locale.LC_CTYPE)
-        _locale.setlocale(_locale.LC_CTYPE, self.tloc)
-        try:
-            teststrop('\x20', 'isspace', True)
-            teststrop('\xa0', 'isspace', False)
-            teststrop('\xa1', 'isspace', False)
-            teststrop('\xc0', 'isalpha', False)
-            teststrop('\xc0', 'isalnum', False)
-            teststrop('\xc0', 'isupper', False)
-            teststrop('\xc0', 'islower', False)
-            teststrop('\xec\xa0\xbc', 'split', ['\xec\xa0\xbc'])
-            teststrop('\xed\x95\xa0', 'strip', '\xed\x95\xa0')
-            teststrop('\xcc\x85', 'lower', '\xcc\x85')
-            teststrop('\xed\x95\xa0', 'upper', '\xed\x95\xa0')
-        finally:
-            _locale.setlocale(_locale.LC_CTYPE, oldlocale)
diff --git a/lib_pypy/pypy_test/test_site_extra.py b/lib_pypy/pypy_test/test_site_extra.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/pypy_test/test_site_extra.py
@@ -0,0 +1,13 @@
+import sys, os
+
+
+def test_preimported_modules():
+    lst = ['__builtin__', '_codecs', '_warnings', 'codecs', 'encodings',
+           'exceptions', 'signal', 'sys', 'zipimport']
+    g = os.popen("'%s' -c 'import sys; print sorted(sys.modules)'" %
+                 (sys.executable,))
+    real_data = g.read()
+    g.close()
+    for name in lst:
+        quoted_name = repr(name)
+        assert quoted_name in real_data
diff --git a/lib_pypy/pypy_test/test_struct_extra.py b/lib_pypy/pypy_test/test_struct_extra.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_struct_extra.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from __future__ import absolute_import
-from lib_pypy import struct 
-
-def test_simple():
-    morezeros = '\x00' * (struct.calcsize('l')-4)
-    assert struct.pack('<l', 16) == '\x10\x00\x00\x00' + morezeros
-    assert struct.pack('4s', 'WAVE') == 'WAVE'
-    assert struct.pack('<4sl', 'WAVE', 16) == 'WAVE\x10\x00\x00\x00' + morezeros
-    s = 'ABCD01234567\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00'
-    assert struct.unpack('<4s4H2lH', s) == ('ABCD', 0x3130, 0x3332, 0x3534,
-                                            0x3736, 1, 2, 3)
-
-def test_infinity():
-    INFINITY = 1e200 * 1e200
-    assert str(struct.unpack("!d", struct.pack("!d", INFINITY))[0]) \
-           == str(INFINITY)
-    assert str(struct.unpack("!d", struct.pack("!d", -INFINITY))[0]) \
-           == str(-INFINITY)
-
-def test_nan():
-    INFINITY = 1e200 * 1e200
-    NAN = INFINITY / INFINITY
-    assert str(struct.unpack("!d", '\xff\xf8\x00\x00\x00\x00\x00\x00')[0]) \
-           == str(NAN)
-    assert str(struct.unpack("!d", struct.pack("!d", NAN))[0]) == str(NAN)
diff --git a/lib_pypy/struct.py b/lib_pypy/struct.py
deleted file mode 100644
--- a/lib_pypy/struct.py
+++ /dev/null
@@ -1,417 +0,0 @@
-#
-# This module is a pure Python version of pypy.module.struct.
-# It is only imported if the vastly faster pypy.module.struct is not
-# compiled in.  For now we keep this version for reference and
-# because pypy.module.struct is not ootype-backend-friendly yet.
-#
-
-"""Functions to convert between Python values and C structs.
-Python strings are used to hold the data representing the C struct
-and also as format strings to describe the layout of data in the C struct.
-
-The optional first format char indicates byte order, size and alignment:
- @: native order, size & alignment (default)
- =: native order, std. size & alignment
- <: little-endian, std. size & alignment
- >: big-endian, std. size & alignment
- !: same as >
-
-The remaining chars indicate types of args and must match exactly;
-these can be preceded by a decimal repeat count:
-   x: pad byte (no data);
-   c:char;
-   b:signed byte;
-   B:unsigned byte;
-   h:short;
-   H:unsigned short;
-   i:int;
-   I:unsigned int;
-   l:long;
-   L:unsigned long;
-   f:float;
-   d:double.
-Special cases (preceding decimal count indicates length):
-   s:string (array of char); p: pascal string (with count byte).
-Special case (only available in native format):
-   P:an integer type that is wide enough to hold a pointer.
-Special case (not in native mode unless 'long long' in platform C):
-   q:long long;
-   Q:unsigned long long
-Whitespace between formats is ignored.
-
-The variable struct.error is an exception raised on errors."""
-
-import math, sys
-
-# TODO: XXX Find a way to get information on native sizes and alignments
-class StructError(Exception):
-    pass
-error = StructError
-def unpack_int(data,index,size,le):
-    bytes = [ord(b) for b in data[index:index+size]]
-    if le == 'little':
-        bytes.reverse()
-    number = 0L
-    for b in bytes:
-        number = number << 8 | b
-    return int(number)
-
-def unpack_signed_int(data,index,size,le):
-    number = unpack_int(data,index,size,le)
-    max = 2**(size*8)
-    if number > 2**(size*8 - 1) - 1:
-        number = int(-1*(max - number))
-    return number
-
-INFINITY = 1e200 * 1e200
-NAN = INFINITY / INFINITY
-
-def unpack_char(data,index,size,le):
-    return data[index:index+size]
-
-def pack_int(number,size,le):
-    x=number
-    res=[]
-    for i in range(size):
-        res.append(chr(x&0xff))
-        x >>= 8
-    if le == 'big':
-        res.reverse()
-    return ''.join(res)
-
-def pack_signed_int(number,size,le):
-    if not isinstance(number, (int,long)):
-        raise StructError,"argument for i,I,l,L,q,Q,h,H must be integer"
-    if  number > 2**(8*size-1)-1 or number < -1*2**(8*size-1):
-        raise OverflowError,"Number:%i too large to convert" % number
-    return pack_int(number,size,le)
-
-def pack_unsigned_int(number,size,le):
-    if not isinstance(number, (int,long)):
-        raise StructError,"argument for i,I,l,L,q,Q,h,H must be integer"
-    if number < 0:
-        raise TypeError,"can't convert negative long to unsigned"
-    if number > 2**(8*size)-1:
-        raise OverflowError,"Number:%i too large to convert" % number
-    return pack_int(number,size,le)
-
-def pack_char(char,size,le):
-    return str(char)
-
-def isinf(x):
-    return x != 0.0 and x / 2 == x
-def isnan(v):
-    return v != v*1.0 or (v == 1.0 and v == 2.0)
-
-def pack_float(x, size, le):
-    unsigned = float_pack(x, size)
-    result = []
-    for i in range(8):
-        result.append(chr((unsigned >> (i * 8)) & 0xFF))
-    if le == "big":
-        result.reverse()
-    return ''.join(result)
-
-def unpack_float(data, index, size, le):
-    binary = [data[i] for i in range(index, index + 8)]
-    if le == "big":
-        binary.reverse()
-    unsigned = 0
-    for i in range(8):
-        unsigned |= ord(binary[i]) << (i * 8)
-    return float_unpack(unsigned, size, le)
-
-def round_to_nearest(x):
-    """Python 3 style round:  round a float x to the nearest int, but
-    unlike the builtin Python 2.x round function:
-
-      - return an int, not a float
-      - do round-half-to-even, not round-half-away-from-zero.
-
-    We assume that x is finite and nonnegative; except wrong results
-    if you use this for negative x.
-
-    """
-    int_part = int(x)
-    frac_part = x - int_part
-    if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1:
-        int_part += 1
-    return int_part
-
-def float_unpack(Q, size, le):
-    """Convert a 32-bit or 64-bit integer created
-    by float_pack into a Python float."""
-
-    if size == 8:
-        MIN_EXP = -1021  # = sys.float_info.min_exp
-        MAX_EXP = 1024   # = sys.float_info.max_exp
-        MANT_DIG = 53    # = sys.float_info.mant_dig
-        BITS = 64
-    elif size == 4:
-        MIN_EXP = -125   # C's FLT_MIN_EXP
-        MAX_EXP = 128    # FLT_MAX_EXP
-        MANT_DIG = 24    # FLT_MANT_DIG
-        BITS = 32
-    else:
-        raise ValueError("invalid size value")
-
-    if Q >> BITS:
-         raise ValueError("input out of range")
-
-    # extract pieces
-    sign = Q >> BITS - 1
-    exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1
-    mant = Q & ((1 << MANT_DIG - 1) - 1)
-
-    if exp == MAX_EXP - MIN_EXP + 2:
-        # nan or infinity
-        result = float('nan') if mant else float('inf')
-    elif exp == 0:
-        # subnormal or zero
-        result = math.ldexp(float(mant), MIN_EXP - MANT_DIG)
-    else:
-        # normal
-        mant += 1 << MANT_DIG - 1
-        result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1)
-    return -result if sign else result
-
-
-def float_pack(x, size):
-    """Convert a Python float x into a 64-bit unsigned integer
-    with the same byte representation."""
-
-    if size == 8:
-        MIN_EXP = -1021  # = sys.float_info.min_exp
-        MAX_EXP = 1024   # = sys.float_info.max_exp
-        MANT_DIG = 53    # = sys.float_info.mant_dig
-        BITS = 64
-    elif size == 4:
-        MIN_EXP = -125   # C's FLT_MIN_EXP
-        MAX_EXP = 128    # FLT_MAX_EXP
-        MANT_DIG = 24    # FLT_MANT_DIG
-        BITS = 32
-    else:
-        raise ValueError("invalid size value")
-
-    sign = math.copysign(1.0, x) < 0.0
-    if math.isinf(x):
-        mant = 0
-        exp = MAX_EXP - MIN_EXP + 2
-    elif math.isnan(x):
-        mant = 1 << (MANT_DIG-2) # other values possible
-        exp = MAX_EXP - MIN_EXP + 2
-    elif x == 0.0:
-        mant = 0
-        exp = 0
-    else:
-        m, e = math.frexp(abs(x))  # abs(x) == m * 2**e
-        exp = e - (MIN_EXP - 1)
-        if exp > 0:
-            # Normal case.
-            mant = round_to_nearest(m * (1 << MANT_DIG))
-            mant -= 1 << MANT_DIG - 1
-        else:
-            # Subnormal case.
-            if exp + MANT_DIG - 1 >= 0:
-                mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1))
-            else:
-                mant = 0
-            exp = 0
-
-        # Special case: rounding produced a MANT_DIG-bit mantissa.
-        assert 0 <= mant <= 1 << MANT_DIG - 1
-        if mant == 1 << MANT_DIG - 1:
-            mant = 0
-            exp += 1
-
-        # Raise on overflow (in some circumstances, may want to return
-        # infinity instead).
-        if exp >= MAX_EXP - MIN_EXP + 2:
-             raise OverflowError("float too large to pack in this format")
-
-    # check constraints
-    assert 0 <= mant < 1 << MANT_DIG - 1
-    assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
-    assert 0 <= sign <= 1
-    return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant
-
-
-big_endian_format = {
-    'x':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'b':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'B':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'c':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_char, 'unpack' : unpack_char},
-    's':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'p':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'h':{ 'size' : 2, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'H':{ 'size' : 2, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'i':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'I':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'l':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'L':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'q':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'Q':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'f':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
-    'd':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
-    }
-default = big_endian_format
-formatmode={ '<' : (default, 'little'),
-             '>' : (default, 'big'),
-             '!' : (default, 'big'),
-             '=' : (default, sys.byteorder),
-             '@' : (default, sys.byteorder)
-            }
-
-def getmode(fmt):
-    try:
-        formatdef,endianness = formatmode[fmt[0]]
-        index = 1
-    except KeyError:
-        formatdef,endianness = formatmode['@']
-        index = 0
-    return formatdef,endianness,index
-def getNum(fmt,i):
-    num=None
-    cur = fmt[i]
-    while ('0'<= cur ) and ( cur <= '9'):
-        if num == None:
-            num = int(cur)
-        else:
-            num = 10*num + int(cur)
-        i += 1
-        cur = fmt[i]
-    return num,i
-
-def calcsize(fmt):
-    """calcsize(fmt) -> int
-    Return size of C struct described by format string fmt.
-    See struct.__doc__ for more on format strings."""
-
-    formatdef,endianness,i = getmode(fmt)
-    num = 0
-    result = 0
-    while i<len(fmt):
-        num,i = getNum(fmt,i)
-        cur = fmt[i]
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError,"%s is not a valid format"%cur
-        if num != None :
-            result += num*format['size']
-        else:
-            result += format['size']
-        num = 0
-        i += 1
-    return result
-
-def pack(fmt,*args):
-    """pack(fmt, v1, v2, ...) -> string
-       Return string containing values v1, v2, ... packed according to fmt.
-       See struct.__doc__ for more on format strings."""
-    formatdef,endianness,i = getmode(fmt)
-    args = list(args)
-    n_args = len(args)
-    result = []
-    while i<len(fmt):
-        num,i = getNum(fmt,i)
-        cur = fmt[i]
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError,"%s is not a valid format"%cur
-        if num == None :
-            num_s = 0
-            num = 1
-        else:
-            num_s = num
-
-        if cur == 'x':
-            result += ['\0'*num]
-        elif cur == 's':
-            if isinstance(args[0], str):
-                padding = num - len(args[0])
-                result += [args[0][:num] + '\0'*padding]
-                args.pop(0)
-            else:
-                raise StructError,"arg for string format not a string"
-        elif cur == 'p':
-            if isinstance(args[0], str):
-                padding = num - len(args[0]) - 1
-
-                if padding > 0:
-                    result += [chr(len(args[0])) + args[0][:num-1] + '\0'*padding]
-                else:
-                    if num<255:
-                        result += [chr(num-1) + args[0][:num-1]]
-                    else:
-                        result += [chr(255) + args[0][:num-1]]
-                args.pop(0)
-            else:
-                raise StructError,"arg for string format not a string"
-
-        else:
-            if len(args) < num:
-                raise StructError,"insufficient arguments to pack"
-            for var in args[:num]:
-                result += [format['pack'](var,format['size'],endianness)]
-            args=args[num:]
-        num = None
-        i += 1
-    if len(args) != 0:
-        raise StructError,"too many arguments for pack format"
-    return ''.join(result)
-
-def unpack(fmt,data):
-    """unpack(fmt, string) -> (v1, v2, ...)
-       Unpack the string, containing packed C structure data, according
-       to fmt.  Requires len(string)==calcsize(fmt).
-       See struct.__doc__ for more on format strings."""
-    formatdef,endianness,i = getmode(fmt)
-    j = 0
-    num = 0
-    result = []
-    length= calcsize(fmt)
-    if length != len (data):
-        raise StructError,"unpack str size does not match format"
-    while i<len(fmt):
-        num,i=getNum(fmt,i)
-        cur = fmt[i]
-        i += 1
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError,"%s is not a valid format"%cur
-
-        if not num :
-            num = 1
-
-        if cur == 'x':
-            j += num
-        elif cur == 's':
-            result.append(data[j:j+num])
-            j += num
-        elif cur == 'p':
-            n=ord(data[j])
-            if n >= num:
-                n = num-1
-            result.append(data[j+1:j+n+1])
-            j += num
-        else:
-            for n in range(num):
-                result += [format['unpack'](data,j,format['size'],endianness)]
-                j += format['size']
-
-    return tuple(result)
-
-def pack_into(fmt, buf, offset, *args):
-    data = pack(fmt, *args)
-    buffer(buf)[offset:offset+len(data)] = data
-
-def unpack_from(fmt, buf, offset=0):
-    size = calcsize(fmt)
-    data = buffer(buf)[offset:offset+size]
-    if len(data) != size:
-        raise error("unpack_from requires a buffer of at least %d bytes"
-                    % (size,))
-    return unpack(fmt, data)
diff --git a/pypy/__init__.py b/pypy/__init__.py
--- a/pypy/__init__.py
+++ b/pypy/__init__.py
@@ -1,1 +1,16 @@
 # Empty
+
+# XXX Should be empty again, soon.
+# XXX hack for win64:
+# This patch must stay here until the END OF STAGE 1
+# When all tests work, this branch will be merged
+# and the branch stage 2 is started, where we remove this patch.
+import sys
+if hasattr(sys, "maxsize"):
+    if sys.maxint != sys.maxsize:
+        sys.maxint = sys.maxsize
+        import warnings
+        warnings.warn("""\n
+---> This win64 port is now in stage 1: sys.maxint was modified.
+---> When pypy/__init__.py becomes empty again, we have reached stage 2.
+""")
diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py
--- a/pypy/annotation/builtin.py
+++ b/pypy/annotation/builtin.py
@@ -37,7 +37,11 @@
     try:
         realresult = func(*args)
     except (ValueError, OverflowError):
-        return s_ImpossibleValue   # no possible answer for this precise input
+        # no possible answer for this precise input.  Be conservative
+        # and keep the computation non-constant.  Example:
+        # unichr(constant-that-doesn't-fit-16-bits) on platforms where
+        # the underlying Python has sys.maxunicode == 0xffff.
+        return s_result
     s_realresult = immutablevalue(realresult)
     if not s_result.contains(s_realresult):
         raise Exception("%s%r returned %r, which is not contained in %s" % (
@@ -163,7 +167,7 @@
                         r.const = False
                 return r
                 
-            assert not issubclass(typ, (int,long)) or typ in (bool, int), (
+            assert not issubclass(typ, (int, long)) or typ in (bool, int, long), (
                 "for integers only isinstance(.,int|r_uint) are supported")
  
             if s_obj.is_constant():
@@ -297,7 +301,7 @@
 def robjmodel_instantiate(s_clspbc):
     assert isinstance(s_clspbc, SomePBC)
     clsdef = None
-    more_than_one = len(s_clspbc.descriptions)
+    more_than_one = len(s_clspbc.descriptions) > 1
     for desc in s_clspbc.descriptions:
         cdef = desc.getuniqueclassdef()
         if more_than_one:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -134,13 +134,19 @@
             if self.name not in homedef.classdesc.all_enforced_attrs:
                 self.attr_allowed = False
                 if not self.readonly:
-                    raise NoSuchAttrError(homedef, self.name)
+                    raise NoSuchAttrError(
+                        "setting forbidden attribute %r on %r" % (
+                        self.name, homedef))
 
     def modified(self, classdef='?'):
         self.readonly = False
         if not self.attr_allowed:
-            raise NoSuchAttrError(classdef, self.name)
-
+            raise NoSuchAttrError(
+                "Attribute %r on %r should be read-only.\n" % (self.name,
+                                                               classdef) +
+                "This error can be caused by another 'getattr' that promoted\n"
+                "the attribute here; the list of read locations is:\n" +
+                '\n'.join([str(loc[0]) for loc in self.read_locations]))
 
 class ClassDef(object):
     "Wraps a user class."
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -398,7 +398,6 @@
             cls = pyobj
             base = object
             baselist = list(cls.__bases__)
-            baselist.reverse()
 
             # special case: skip BaseException in Python 2.5, and pretend
             # that all exceptions ultimately inherit from Exception instead
@@ -408,17 +407,27 @@
             elif baselist == [py.builtin.BaseException]:
                 baselist = [Exception]
 
+            mixins_before = []
+            mixins_after = []
             for b1 in baselist:
                 if b1 is object:
                     continue
                 if b1.__dict__.get('_mixin_', False):
-                    self.add_mixin(b1)
+                    if base is object:
+                        mixins_before.append(b1)
+                    else:
+                        mixins_after.append(b1)
                 else:
                     assert base is object, ("multiple inheritance only supported "
                                             "with _mixin_: %r" % (cls,))
                     base = b1
+            if mixins_before and mixins_after:
+                raise Exception("unsupported: class %r has mixin bases both"
+                                " before and after the regular base" % (self,))
+            self.add_mixins(mixins_after, check_not_in=base)
+            self.add_mixins(mixins_before)
+            self.add_sources_for_class(cls)
 
-            self.add_sources_for_class(cls)
             if base is not object:
                 self.basedesc = bookkeeper.getdesc(base)
 
@@ -480,14 +489,30 @@
                 return
         self.classdict[name] = Constant(value)
 
-    def add_mixin(self, base):
-        for subbase in base.__bases__:
-            if subbase is object:
-                continue
-            assert subbase.__dict__.get("_mixin_", False), ("Mixin class %r has non"
-                "mixin base class %r" % (base, subbase))
-            self.add_mixin(subbase)
-        self.add_sources_for_class(base, mixin=True)
+    def add_mixins(self, mixins, check_not_in=object):
+        if not mixins:
+            return
+        A = type('tmp', tuple(mixins) + (object,), {})
+        mro = A.__mro__
+        assert mro[0] is A and mro[-1] is object
+        mro = mro[1:-1]
+        #
+        skip = set()
+        def add(cls):
+            if cls is not object:
+                for base in cls.__bases__:
+                    add(base)
+                for name in cls.__dict__:
+                    skip.add(name)
+        add(check_not_in)
+        #
+        for base in reversed(mro):
+            assert base.__dict__.get("_mixin_", False), ("Mixin class %r has non"
+                "mixin base class %r" % (mixins, base))
+            for name, value in base.__dict__.items():
+                if name in skip:
+                    continue
+                self.add_source_attribute(name, value, mixin=True)
 
     def add_sources_for_class(self, cls, mixin=False):
         for name, value in cls.__dict__.items():
diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py
--- a/pypy/annotation/model.py
+++ b/pypy/annotation/model.py
@@ -786,12 +786,15 @@
 #
 # safety check that no-one is trying to make annotation and translation
 # faster by providing the -O option to Python.
-try:
-    assert False
-except AssertionError:
-    pass   # fine
-else:
-    raise RuntimeError("The annotator relies on 'assert' statements from the\n"
+import os
+if "WINGDB_PYTHON" not in os.environ:
+    # ...but avoiding this boring check in the IDE
+    try:
+        assert False
+    except AssertionError:
+        pass   # fine
+    else:
+        raise RuntimeError("The annotator relies on 'assert' statements from the\n"
                      "\tannotated program: you cannot run it with 'python -O'.")
 
 # this has the side-effect of registering the unary and binary operations
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1,15 +1,12 @@
 from __future__ import with_statement
-import autopath
 import py.test
 import sys
 from pypy import conftest
-from pypy.tool.udir import udir
 
 from pypy.annotation import model as annmodel
 from pypy.annotation.annrpython import RPythonAnnotator as _RPythonAnnotator
 from pypy.translator.translator import graphof as tgraphof
 from pypy.annotation import policy
-from pypy.annotation import specialize
 from pypy.annotation.listdef import ListDef, ListChangeUnallowed
 from pypy.annotation.dictdef import DictDef
 from pypy.objspace.flow.model import *
@@ -2431,6 +2428,93 @@
         assert isinstance(s.items[1], annmodel.SomeChar)
         assert isinstance(s.items[2], annmodel.SomeChar)
 
+    def test_mixin_first(self):
+        class Mixin(object):
+            _mixin_ = True
+            def foo(self): return 4
+        class Base(object):
+            def foo(self): return 5
+        class Concrete(Mixin, Base):
+            pass
+        def f():
+            return Concrete().foo()
+
+        assert f() == 4
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [])
+        assert s.const == 4
+
+    def test_mixin_last(self):
+        class Mixin(object):
+            _mixin_ = True
+            def foo(self): return 4
+        class Base(object):
+            def foo(self): return 5
+        class Concrete(Base, Mixin):
+            pass
+        def f():
+            return Concrete().foo()
+
+        assert f() == 5
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [])
+        assert s.const == 5
+
+    def test_mixin_concrete(self):
+        class Mixin(object):
+            _mixin_ = True
+            def foo(self): return 4
+        class Concrete(Mixin):
+            def foo(self): return 5
+        def f():
+            return Concrete().foo()
+
+        assert f() == 5
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [])
+        assert s.const == 5
+
+    def test_multiple_mixins_mro(self):
+        # an obscure situation, but it occurred in module/micronumpy/types.py
+        class A(object):
+            _mixin_ = True
+            def foo(self): return 1
+        class B(A):
+            _mixin_ = True
+            def foo(self): return 2
+        class C(A):
+            _mixin_ = True
+        class D(B, C):
+            _mixin_ = True
+        class Concrete(D):
+            pass
+        def f():
+            return Concrete().foo()
+
+        assert f() == 2
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [])
+        assert s.const == 2
+
+    def test_multiple_mixins_mro_2(self):
+        class A(object):
+            _mixin_ = True
+            def foo(self): return 1
+        class B(A):
+            _mixin_ = True
+            def foo(self): return 2
+        class C(A):
+            _mixin_ = True
+        class Concrete(C, B):
+            pass
+        def f():
+            return Concrete().foo()
+
+        assert f() == 2
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [])
+        assert s.const == 2
+
     def test___class___attribute(self):
         class Base(object): pass
         class A(Base): pass
@@ -2469,6 +2553,26 @@
         s = a.build_types(f, [int])
         assert s.knowntype == int
 
+    def test_slots_reads(self):
+        class A(object):
+            __slots__ = ()
+        class B(A):
+            def __init__(self, x):
+                self.x = x
+        def f(x):
+            if x:
+                a = A()
+            else:
+                a = B(x)
+            return a.x   # should explode here
+
+        a = self.RPythonAnnotator()
+        e = py.test.raises(Exception, a.build_types, f, [int])
+        # this should explode on reading the attribute 'a.x', but it can
+        # sometimes explode on 'self.x = x', which does not make much sense.
+        # But it looks hard to fix in general: we don't know yet during 'a.x'
+        # if the attribute x will be read-only or read-write.
+
     def test_unboxed_value(self):
         class A(object):
             __slots__ = ()
diff --git a/pypy/bin/rpython b/pypy/bin/rpython
new file mode 100755
--- /dev/null
+++ b/pypy/bin/rpython
@@ -0,0 +1,18 @@
+#!/usr/bin/env pypy
+
+"""RPython translation usage:
+
+rpython <translation options> target <targetoptions>
+
+run with --help for more information
+"""
+
+import sys
+from pypy.translator.goal.translate import main
+
+# no implicit targets
+if len(sys.argv) == 1:
+    print __doc__
+    sys.exit(1)
+
+main()
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -176,9 +176,6 @@
                cmdline="--translationmodules",
                suggests=[("objspace.allworkingmodules", False)]),
 
-    BoolOption("geninterp", "specify whether geninterp should be used",
-               default=False),
-
     BoolOption("logbytecodes",
                "keep track of bytecode usage",
                default=False),
@@ -392,10 +389,6 @@
             config.objspace.std.suggest(withsmalllong=True)
         # xxx other options? ropes maybe?
 
-    # completely disable geninterp in a level 0 translation
-    if level == '0':
-        config.objspace.suggest(geninterp=False)
-
     # some optimizations have different effects depending on the typesystem
     if type_system == 'ootype':
         config.objspace.std.suggest(multimethods="doubledispatch")
diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py
--- a/pypy/config/translationoption.py
+++ b/pypy/config/translationoption.py
@@ -182,11 +182,6 @@
 
     # Flags of the TranslationContext:
     BoolOption("simplifying", "Simplify flow graphs", default=True),
-    BoolOption("builtins_can_raise_exceptions",
-               "When true, assume any call to a 'simple' builtin such as "
-               "'hex' can raise an arbitrary exception",
-               default=False,
-               cmdline=None),
     BoolOption("list_comprehension_operations",
                "When true, look for and special-case the sequence of "
                "operations that results from a list comprehension and "
diff --git a/pypy/doc/discussion/win64_todo.txt b/pypy/doc/discussion/win64_todo.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/discussion/win64_todo.txt
@@ -0,0 +1,9 @@
+2011-11-04
+ll_os.py has a problem with the file rwin32.py.
+Temporarily disabled for the win64_gborg branch. This needs to be
+investigated and re-enabled.
+Resolved, enabled.
+
+2011-11-05
+test_typed.py needs explicit tests to ensure that we
+handle word sizes right.
\ No newline at end of file
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -103,21 +103,13 @@
 
 * A concurrent garbage collector (a lot of work)
 
-Remove the GIL
---------------
+STM, a.k.a. "remove the GIL"
+----------------------------
 
-This is a major task that requires lots of thinking. However, few subprojects
-can be potentially specified, unless a better plan can be thought out:
+Removing the GIL --- or more precisely, a GIL-less thread-less solution ---
+is `now work in progress.`__  Contributions welcome.
 
-* A thread-aware garbage collector
-
-* Better RPython primitives for dealing with concurrency
-
-* JIT passes to remove locks on objects
-
-* (maybe) implement locking in Python interpreter
-
-* alternatively, look at Software Transactional Memory
+.. __: http://pypy.org/tmdonate.html
 
 Introduce new benchmarks
 ------------------------
@@ -157,6 +149,22 @@
 exported.  This would give us a one-size-fits-all generic .so file to be
 imported by any application that wants to load .so files :-)
 
+Optimising cpyext (CPython C-API compatibility layer)
+-----------------------------------------------------
+
+A lot of work has gone into PyPy's implementation of CPython's C-API over
+the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However,
+there are still many edges and corner cases where it misbehaves, and it has
+not received any substantial optimisation so far.
+
+The objective of this project is to fix bugs in cpyext and to optimise
+several performance critical parts of it, such as the reference counting
+support and other heavily used C-API functions. The net result would be to
+have CPython extensions run much faster on PyPy than they currently do, or
+to make them work at all if they currently don't. A part of this work would
+be to get cpyext into a shape where it supports running Cython generated
+extensions.
 
 .. _`issue tracker`: http://bugs.pypy.org
 .. _`mailing list`: http://mail.python.org/mailman/listinfo/pypy-dev
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -82,7 +82,10 @@
 
 In pypy/translator/goal::
 
-   ./translate.py --sandbox targetpypystandalone.py
+   ./translate.py -O2 --sandbox targetpypystandalone.py
+
+If you don't have a regular PyPy installed, you should, because it's
+faster to translate, but you can also run ``python translate.py`` instead.
 
 
 To run it, use the tools in the pypy/translator/sandbox directory::
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -199,17 +199,11 @@
 The following features (present in some past Stackless version of PyPy)
 are for the time being not supported any more:
 
-* Tasklets and channels (currently ``stackless.py`` seems to import,
-  but you have tasklets on top of coroutines on top of greenlets on
-  top of continulets on top of stacklets, and it's probably not too
-  hard to cut two of these levels by adapting ``stackless.py`` to
-  use directly continulets)
-
 * Coroutines (could be rewritten at app-level)
 
-* Pickling and unpickling continulets (*)
-
-* Continuing execution of a continulet in a different thread (*)
+* Continuing execution of a continulet in a different thread
+  (but if it is "simple enough", you can pickle it and unpickle it
+  in the other thread).
 
 * Automatic unlimited stack (must be emulated__ so far)
 
@@ -217,15 +211,6 @@
 
 .. __: `recursion depth limit`_
 
-(*) Pickling, as well as changing threads, could be implemented by using
-a "soft" stack switching mode again.  We would get either "hard" or
-"soft" switches, similarly to Stackless Python 3rd version: you get a
-"hard" switch (like now) when the C stack contains non-trivial C frames
-to save, and a "soft" switch (like previously) when it contains only
-simple calls from Python to Python.  Soft-switched continulets would
-also consume a bit less RAM, and the switch might be a bit faster too
-(unsure about that; what is the Stackless Python experience?).
-
 
 Recursion depth limit
 +++++++++++++++++++++
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
new file mode 100644
--- /dev/null
+++ b/pypy/doc/tool/makecontributor.py
@@ -0,0 +1,133 @@
+import py
+import sys
+from collections import defaultdict
+import operator
+import re
+import mercurial.localrepo
+import mercurial.ui
+
+ROOT = py.path.local(__file__).join('..', '..', '..', '..')
+author_re = re.compile('(.*) <.*>')
+pair_programming_re = re.compile(r'^\((.*?)\)')
+excluded = set(["pypy", "convert-repo"])
+
+alias = {
+    'Anders Chrigstrom': ['arre'],
+    'Antonio Cuni': ['antocuni', 'anto'],
+    'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
+    'Maciej Fijalkowski': ['fijal'],
+    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Michael Hudson': ['mwh'],
+    'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
+    "Amaury Forgeot d'Arc": ['afa'],
+    'Alex Gaynor': ['alex', 'agaynor'],
+    'David Schneider': ['bivab', 'david'],
+    'Christian Tismer': ['chris', 'christian', 'tismer',
+                         'tismer at christia-wjtqxl.localdomain'],
+    'Benjamin Peterson': ['benjamin'],
+    'Hakan Ardo': ['hakan', 'hakanardo'],
+    'Niklaus Haldimann': ['nik'],
+    'Alexander Schremmer': ['xoraxax'],
+    'Anders Hammarquist': ['iko'],
+    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'Niko Matsakis': ['niko'],
+    'Jakub Gustak': ['jlg'],
+    'Guido Wesdorp': ['guido'],
+    'Michael Foord': ['mfoord'],
+    'Mark Pearse': ['mwp'],
+    'Toon Verwaest': ['tverwaes'],
+    'Eric van Riet Paap': ['ericvrp'],
+    'Jacob Hallen': ['jacob', 'jakob'],
+    'Anders Lehmann': ['ale', 'anders'],
+    'Bert Freudenberg': ['bert'],
+    'Boris Feigin': ['boris', 'boria'],
+    'Valentino Volonghi': ['valentino', 'dialtone'],
+    'Aurelien Campeas': ['aurelien', 'aureliene'],
+    'Adrien Di Mascio': ['adim'],
+    'Jacek Generowicz': ['Jacek', 'jacek'],
+    'Jim Hunziker': ['landtuna at gmail.com'],
+    'Kristjan Valur Jonsson': ['kristjan at kristjan-lp.ccp.ad.local'],
+    'Laura Creighton': ['lac'],
+    'Aaron Iles': ['aliles'],
+    'Ludovic Aubry': ['ludal', 'ludovic'],
+    'Lukas Diekmann': ['l.diekmann', 'ldiekmann'],
+    'Matti Picus': ['Matti Picus matti.picus at gmail.com',
+                    'matthp', 'mattip', 'mattip>'],
+    'Michael Cheng': ['mikefc'],
+    'Richard Emslie': ['rxe'],
+    'Roberto De Ioris': ['roberto at goyle'],
+    'Roberto De Ioris': ['roberto at mrspurr'],
+    'Sven Hager': ['hager'],
+    'Tomo Cocoa': ['cocoatomo'],
+    }
+
+alias_map = {}
+for name, nicks in alias.iteritems():
+    for nick in nicks:
+        alias_map[nick] = name
+
+def get_canonical_author(name):
+    match = author_re.match(name)
+    if match:
+        name = match.group(1)
+    return alias_map.get(name, name)
+
+ignored_nicknames = defaultdict(int)
+
+def get_more_authors(log):
+    match = pair_programming_re.match(log)
+    if not match:
+        return set()
+    ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
+                    'in-progress', 'bits of', 'even a little', 'floating',]
+    sep_words = ['and', ';', '+', '/', 'with special  by']
+    nicknames = match.group(1)
+    for word in ignore_words:
+        nicknames = nicknames.replace(word, '')
+    for word in sep_words:
+        nicknames = nicknames.replace(word, ',')
+    nicknames = [nick.strip().lower() for nick in nicknames.split(',')]
+    authors = set()
+    for nickname in nicknames:
+        author = alias_map.get(nickname)
+        if not author:
+            ignored_nicknames[nickname] += 1
+        else:
+            authors.add(author)
+    return authors
+
+def main(show_numbers):
+    ui = mercurial.ui.ui()
+    repo = mercurial.localrepo.localrepository(ui, str(ROOT))
+    authors_count = defaultdict(int)
+    for i in repo:
+        ctx = repo[i]
+        authors = set()
+        authors.add(get_canonical_author(ctx.user()))
+        authors.update(get_more_authors(ctx.description()))
+        for author in authors:
+            if author not in excluded:
+                authors_count[author] += 1
+
+    # uncomment the next lines to get the list of nicknamed which could not be
+    # parsed from commit logs
+    ## items = ignored_nicknames.items()
+    ## items.sort(key=operator.itemgetter(1), reverse=True)
+    ## for name, n in items:
+    ##     if show_numbers:
+    ##         print '%5d %s' % (n, name)
+    ##     else:
+    ##         print name
+                
+    items = authors_count.items()
+    items.sort(key=operator.itemgetter(1), reverse=True)
+    for name, n in items:
+        if show_numbers:
+            print '%5d %s' % (n, name)
+        else:
+            print name
+
+if __name__ == '__main__':
+    show_numbers = '-n' in sys.argv
+    main(show_numbers)
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -18,7 +18,8 @@
 Edition.  Other configurations may work as well.
 
 The translation scripts will set up the appropriate environment variables
-for the compiler.  They will attempt to locate the same compiler version that
+for the compiler, so you do not need to run vcvars before translation.  
+They will attempt to locate the same compiler version that
 was used to build the Python interpreter doing the
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
@@ -26,7 +27,7 @@
 using a 32 bit Python and vice versa.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
-will be aborted in this case.
+will fail in this case.
 
 The compiler is all you need to build pypy-c, but it will miss some
 modules that relies on third-party libraries.  See below how to get
@@ -57,7 +58,8 @@
 install third-party libraries.  We chose to install them in the parent
 directory of the pypy checkout.  For example, if you installed pypy in
 ``d:\pypy\trunk\`` (This directory contains a README file), the base
-directory is ``d:\pypy``.
+directory is ``d:\pypy``. You may choose different values by setting the
+INCLUDE, LIB and PATH (for DLLs)
 
 The Boehm garbage collector
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -126,18 +128,54 @@
 ------------------------
 
 You can compile pypy with the mingw compiler, using the --cc=mingw32 option;
-mingw.exe must be on the PATH.
+gcc.exe must be on the PATH. If the -cc flag does not begin with "ming", it should be
+the name of a valid gcc-derivative compiler, i.e. x86_64-w64-mingw32-gcc for the 64 bit
+compiler creating a 64 bit target.
 
-libffi for the mingw32 compiler
+libffi for the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To enable the _rawffi (and ctypes) module, you need to compile a mingw32
-version of libffi.  I downloaded the `libffi source files`_, and extracted
-them in the base directory.  Then run::
+To enable the _rawffi (and ctypes) module, you need to compile a mingw
+version of libffi.  Here is one way to do this, wich should allow you to try
+to build for win64 or win32:
+
+#. Download and unzip a `mingw32 build`_ or `mingw64 build`_, say into c:\mingw
+#. If you do not use cygwin, you will need msys to provide make, 
+   autoconf tools and other goodies.
+
+    #. Download and unzip a `msys for mingw`_, say into c:\msys
+    #. Edit the c:\msys\etc\fstab file to mount c:\mingw
+
+#. Download and unzip the `libffi source files`_, and extract
+   them in the base directory.  
+#. Run c:\msys\msys.bat or a cygwin shell which should make you
+   feel better since it is a shell prompt with shell tools.
+#. From inside the shell, cd to the libffi directory and do::
 
     sh ./configure
     make
     cp .libs/libffi-5.dll <somewhere on the PATH>
 
+If you can't find the dll, and the libtool issued a warning about 
+"undefined symbols not allowed", you will need to edit the libffi
+Makefile in the toplevel directory. Add the flag -no-undefined to
+the definition of libffi_la_LDFLAGS
+
+If you wish to experiment with win64, you must run configure with flags::
+
+    sh ./configure --build=x86_64-w64-mingw32 --host=x86_64-w64-mingw32
+
+or such, depending on your mingw64 download.
+
+hacking on Pypy with the mingw compiler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Since hacking on Pypy means running tests, you will need a way to specify
+the mingw compiler when hacking (as opposed to translating). As of
+March 2012, --cc is not a valid option for pytest.py. However if you set an
+environment variable CC it will allow you to choose a compiler.
+
+.. _'mingw32 build': http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Automated%20Builds
+.. _`mingw64 build`: http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Automated%20Builds
+.. _`msys for mingw`: http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/MSYS%20%2832-bit%29   
 .. _`libffi source files`: http://sourceware.org/libffi/
 .. _`RPython translation toolchain`: translation.html
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/you-want-to-help.rst
@@ -0,0 +1,86 @@
+
+You want to help with PyPy, now what?
+=====================================
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  the development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy
+  approach is great if you do TDD, not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below
+
+Architecture
+============
+
+PyPy has layers. The 100 miles view:
+
+* `RPython`_ is the language in which we write interpreters. Not the entire
+  PyPy project is written in RPython, only the parts that are compiled in
+  the translation process. The interesting point is that RPython has no parser,
+  it's compiled from the live python objects, which make it possible to do
+  all kinds of metaprogramming during import time. In short, Python is a meta
+  programming language for RPython.
+
+  The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+.. _`RPython`: coding-guide.html#RPython
+
+* The translation toolchain - this is the part that takes care about translating
+  RPython to flow graphs and then to C. There is more in the `architecture`_
+  document written about it.
+
+  It mostly lives in ``rpython``, ``annotator`` and ``objspace/flow``.
+
+.. _`architecture`: architecture.html 
+
+* Python Interpreter
+
+  xxx
+
+* Python modules
+
+  xxx
+
+* Just-in-Time Compiler (JIT): `we have a tracing JIT`_ that traces the
+  interpreter written in RPython, rather than the user program that it
+  interprets.  As a result it applies to any interpreter, i.e. any
+  language.  But getting it to work correctly is not trivial: it
+  requires a small number of precise "hints" and possibly some small
+  refactorings of the interpreter.  The JIT itself also has several
+  almost-independent parts: the tracer itself in ``jit/metainterp``, the
+  optimizer in ``jit/metainterp/optimizer`` that optimizes a list of
+  residual operations, and the backend in ``jit/backend/<machine-name>``
+  that turns it into machine code.  Writing a new backend is a
+  traditional way to get into the project.
+
+.. _`we have a tracing JIT`: jit/index.html
+
+* Garbage Collectors (GC): as you can notice if you are used to CPython's
+  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+  `Garbage collection in PyPy`_ is inserted
+  during translation.  Moreover, this is not reference counting; it is a real
+  GC written as more RPython code.  The best one we have so far is in
+  ``rpython/memory/gc/minimark.py``.
+
+.. _`Garbage collection in PyPy`: garbage_collection.html
+
+
+Toolset
+=======
+
+xxx
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -610,6 +610,8 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+
+    ops.BUILD_LIST_FROM_ARG: 1,
 }
 
 
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -965,7 +965,7 @@
         self.emit_op_arg(ops.CALL_METHOD, (kwarg_count << 8) | arg_count)
         return True
 
-    def _listcomp_generator(self, gens, gen_index, elt):
+    def _listcomp_generator(self, gens, gen_index, elt, single=False):
         start = self.new_block()
         skip = self.new_block()
         if_cleanup = self.new_block()
@@ -973,6 +973,8 @@
         gen = gens[gen_index]
         assert isinstance(gen, ast.comprehension)
         gen.iter.walkabout(self)
+        if single:
+            self.emit_op_arg(ops.BUILD_LIST_FROM_ARG, 0)
         self.emit_op(ops.GET_ITER)
         self.use_next_block(start)
         self.emit_jump(ops.FOR_ITER, anchor)
@@ -998,8 +1000,12 @@
 
     def visit_ListComp(self, lc):
         self.update_position(lc.lineno)
-        self.emit_op_arg(ops.BUILD_LIST, 0)
-        self._listcomp_generator(lc.generators, 0, lc.elt)
+        if len(lc.generators) != 1 or lc.generators[0].ifs:
+            single = False
+            self.emit_op_arg(ops.BUILD_LIST, 0)
+        else:
+            single = True
+        self._listcomp_generator(lc.generators, 0, lc.elt, single=single)
 
     def _comp_generator(self, node, generators, gen_index):
         start = self.new_block()
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -10,16 +10,6 @@
 from pypy.interpreter.astcompiler import ast, consts
 
 
-try:
-    all
-except NameError:
-    def all(iterable):
-        for x in iterable:
-            if not x:
-                return False
-        return True
-
-
 class TestAstBuilder:
 
     def setup_class(cls):
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -58,7 +58,8 @@
         w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
         res = space.str_w(space.repr(w_res))
         if not isinstance(expected, float):
-            assert res == repr(expected)
+            noL = lambda expr: expr.replace('L', '')
+            assert noL(res) == noL(repr(expected))
         else:
             # Float representation can vary a bit between interpreter
             # versions, compare the numbers instead.
@@ -908,3 +909,17 @@
             return d['f'](5)
         """)
         assert 'generator' in space.str_w(space.repr(w_generator))
+        
+    def test_list_comprehension(self):
+        source = "def f(): [i for i in l]"
+        source2 = "def f(): [i for i in l for j in l]"
+        source3 = "def f(): [i for i in l if i]"
+        counts = self.count_instructions(source)
+        assert ops.BUILD_LIST not in counts
+        assert counts[ops.BUILD_LIST_FROM_ARG] == 1
+        counts = self.count_instructions(source2)
+        assert counts[ops.BUILD_LIST] == 1
+        assert ops.BUILD_LIST_FROM_ARG not in counts
+        counts = self.count_instructions(source3)
+        assert counts[ops.BUILD_LIST] == 1
+        assert ops.BUILD_LIST_FROM_ARG not in counts
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -7,7 +7,8 @@
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
+from pypy.rlib.objectmodel import we_are_translated, newlist_hint,\
+     compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
@@ -295,6 +296,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self.frame_trace_action = FrameTraceAction(self)
+        self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
         self.our_magic = default_magic
@@ -466,9 +468,9 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: time2 or rctime take precedence over time
+        # a bit of custom logic: rctime take precedence over time
         # XXX this could probably be done as a "requires" in the config
-        if ('time2' in modules or 'rctime' in modules) and 'time' in modules:
+        if 'rctime' in modules and 'time' in modules:
             modules.remove('time')
 
         if not self.config.objspace.nofaking:
@@ -833,7 +835,7 @@
             items = []
         else:
             try:
-                items = newlist(lgt_estimate)
+                items = newlist_hint(lgt_estimate)
             except MemoryError:
                 items = [] # it might have lied
         #
@@ -1335,7 +1337,7 @@
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
             raise OperationError(self.w_TypeError,
                                  self.wrap('argument must be a string'))
-        return self.str_w(w_obj)
+        return self.str_w(w_obj)            
 
     def unicode_w(self, w_obj):
         return w_obj.unicode_w(self)
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -20,6 +20,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError
 from pypy.rlib.objectmodel import compute_hash
+from pypy.rlib.rstring import StringBuilder
 
 
 class Buffer(Wrappable):
@@ -152,12 +153,13 @@
     if space.isinstance_w(w_object, space.w_unicode):
         # unicode objects support the old buffer interface
         # but not the new buffer interface (change in python  2.7)
-        from pypy.rlib.rstruct.unichar import pack_unichar
-        charlist = []
-        for unich in space.unicode_w(w_object):
-            pack_unichar(unich, charlist)
+        from pypy.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
+        unistr = space.unicode_w(w_object)
+        builder = StringBuilder(len(unistr) * UNICODE_SIZE)
+        for unich in unistr:
+            pack_unichar(unich, builder)
         from pypy.interpreter.buffer import StringBuffer
-        w_buffer = space.wrap(StringBuffer(''.join(charlist)))
+        w_buffer = space.wrap(StringBuffer(builder.build()))
     else:
         w_buffer = space.buffer(w_object)
 
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -47,6 +47,11 @@
 
     def async(self, space):
         "Check if this is an exception that should better not be caught."
+        if not space.full_exceptions:
+            # flow objspace does not support such exceptions and more
+            # importantly, raises KeyboardInterrupt if you try to access
+            # space.w_KeyboardInterrupt
+            return False
         return (self.match(space, space.w_SystemExit) or
                 self.match(space, space.w_KeyboardInterrupt))
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -154,6 +154,7 @@
         #operationerr.print_detailed_traceback(self.space)
 
     def _convert_exc(self, operr):
+        # Only for the flow object space
         return operr
 
     def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
@@ -166,6 +167,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -113,6 +113,12 @@
         from pypy.interpreter.pycode import PyCode
 
         code = self.getcode() # hook for the jit
+        #
+        if (jit.we_are_jitted() and code is self.space._code_of_sys_exc_info
+                                and nargs == 0):
+            from pypy.module.sys.vm import exc_info_direct
+            return exc_info_direct(self.space, frame)
+        #
         fast_natural_arity = code.fast_natural_arity
         if nargs == fast_natural_arity:
             if nargs == 0:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -874,6 +874,12 @@
             fn.add_to_table()
         if gateway.as_classmethod:
             fn = ClassMethod(space.wrap(fn))
+        #
+        from pypy.module.sys.vm import exc_info
+        if code._bltin is exc_info:
+            assert space._code_of_sys_exc_info is None
+            space._code_of_sys_exc_info = code
+        #
         return fn
 
 
@@ -901,24 +907,20 @@
 
     def __init__(self, source, filename=None, modname='__builtin__'):
         # HAAACK (but a good one)
+        self.filename = filename
+        self.source = str(py.code.Source(source).deindent())
+        self.modname = modname
         if filename is None:
             f = sys._getframe(1)
             filename = '<%s:%d>' % (f.f_code.co_filename, f.f_lineno)
+        if not os.path.exists(filename):
+            # make source code available for tracebacks
+            lines = [x + "\n" for x in source.split("\n")]
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
         self.filename = filename
-        self.source = str(py.code.Source(source).deindent())
-        self.modname = modname
-        # look at the first three lines for a NOT_RPYTHON tag
-        first = "\n".join(source.split("\n", 3)[:3])
-        if "NOT_RPYTHON" in first:
-            self.can_use_geninterp = False
-        else:
-            self.can_use_geninterp = True
-        # make source code available for tracebacks
-        lines = [x + "\n" for x in source.split("\n")]
-        py.std.linecache.cache[filename] = (1, None, lines, filename)
 
     def __repr__(self):
-        return "<ApplevelClass filename=%r can_use_geninterp=%r>" % (self.filename, self.can_use_geninterp)
+        return "<ApplevelClass filename=%r>" % (self.filename,)
 
     def getwdict(self, space):
         return space.fromcache(ApplevelCache).getorbuild(self)
@@ -979,10 +981,7 @@
 
     def build(self, app):
         "NOT_RPYTHON.  Called indirectly by Applevel.getwdict()."
-        if self.space.config.objspace.geninterp and app.can_use_geninterp:
-            return PyPyCacheDir.build_applevelinterp_dict(app, self.space)
-        else:
-            return build_applevel_dict(app, self.space)
+        return build_applevel_dict(app, self.space)
 
 
 # __________ pure applevel version __________
@@ -996,157 +995,6 @@
                 filename=self.filename)
     return w_glob
 
-# __________ geninterplevel version __________
-
-class PyPyCacheDir:
-    "NOT_RPYTHON"
-    # similar to applevel, but using translation to interp-level.
-    # This version maintains a cache folder with single files.
-
-    def build_applevelinterp_dict(cls, self, space):
-        "NOT_RPYTHON"
-        # N.B. 'self' is the ApplevelInterp; this is a class method,
-        # just so that we have a convenient place to store the global state.
-        if not cls._setup_done:
-            cls._setup()
-
-        from pypy.translator.geninterplevel import translate_as_module
-        import marshal
-        scramble = md5(cls.seed)
-        scramble.update(marshal.dumps(self.source))
-        key = scramble.hexdigest()
-        initfunc = cls.known_code.get(key)
-        if not initfunc:
-            # try to get it from file
-            name = key
-            if self.filename:
-                prename = os.path.splitext(os.path.basename(self.filename))[0]
-            else:
-                prename = 'zznoname'
-            name = "%s_%s" % (prename, name)
-            try:
-                __import__("pypy._cache."+name)
-            except ImportError, x:
-                # print x
-                pass
-            else:
-                initfunc = cls.known_code[key]
-        if not initfunc:
-            # build it and put it into a file
-            initfunc, newsrc = translate_as_module(
-                self.source, self.filename, self.modname)
-            fname = cls.cache_path.join(name+".py").strpath
-            f = file(get_tmp_file_name(fname), "w")
-            print >> f, """\
-# self-destruct on double-click:
-if __name__ == "__main__":
-    from pypy import _cache
-    import os
-    namestart = os.path.join(os.path.split(_cache.__file__)[0], '%s')
-    for ending in ('.py', '.pyc', '.pyo'):
-        try:
-            os.unlink(namestart+ending)
-        except os.error:
-            pass""" % name
-            print >> f
-            print >> f, newsrc
-            print >> f, "from pypy._cache import known_code"
-            print >> f, "known_code[%r] = %s" % (key, initfunc.__name__)
-            f.close()
-            rename_tmp_to_eventual_file_name(fname)
-        w_glob = initfunc(space)
-        return w_glob
-    build_applevelinterp_dict = classmethod(build_applevelinterp_dict)
-
-    _setup_done = False
-
-    def _setup(cls):
-        """NOT_RPYTHON"""
-        lp = py.path.local
-        import pypy, os
-        p = lp(pypy.__file__).new(basename='_cache').ensure(dir=1)
-        cls.cache_path = p
-        ini = p.join('__init__.py')
-        try:
-            if not ini.check():
-                raise ImportError  # don't import if only a .pyc file left!!!
-            from pypy._cache import known_code, \
-                 GI_VERSION_RENDERED
-        except ImportError:
-            GI_VERSION_RENDERED = 0
-        from pypy.translator.geninterplevel import GI_VERSION
-        cls.seed = md5(str(GI_VERSION)).digest()
-        if GI_VERSION != GI_VERSION_RENDERED or GI_VERSION is None:
-            for pth in p.listdir():
-                if pth.check(file=1):
-                    try:
-                        pth.remove()
-                    except: pass
-            f = file(get_tmp_file_name(str(ini)), "w")
-            f.write("""\
-# This folder acts as a cache for code snippets which have been
-# compiled by compile_as_module().
-# It will get a new entry for every piece of code that has
-# not been seen, yet.
-#
-# Caution! Only the code snippet is checked. If something
-# is imported, changes are not detected. Also, changes
-# to geninterplevel or gateway are also not checked.
-# Exception: There is a checked version number in geninterplevel.py
-#
-# If in doubt, remove this file from time to time.
-
-GI_VERSION_RENDERED = %r
-
-known_code = {}
-
-# self-destruct on double-click:
-def harakiri():
-    import pypy._cache as _c
-    import py
-    lp = py.path.local
-    for pth in lp(_c.__file__).dirpath().listdir():
-        try:
-            pth.remove()
-        except: pass
-
-if __name__ == "__main__":
-    harakiri()
-
-del harakiri
-""" % GI_VERSION)
-            f.close()
-            rename_tmp_to_eventual_file_name(str(ini))
-        import pypy._cache
-        cls.known_code = pypy._cache.known_code
-        cls._setup_done = True
-    _setup = classmethod(_setup)
-
-
-def gethostname(_cache=[]):
-    if not _cache:
-        try:
-            import socket
-            hostname = socket.gethostname()
-        except:
-            hostname = ''
-        _cache.append(hostname)
-    return _cache[0]
-
-def get_tmp_file_name(fname):
-    return '%s~%s~%d' % (fname, gethostname(), os.getpid())
-
-def rename_tmp_to_eventual_file_name(fname):
-    # generated files are first written to the host- and process-specific
-    # file 'tmpname', and then atomically moved to their final 'fname'
-    # to avoid problems if py.py is started several times in parallel
-    tmpname = get_tmp_file_name(fname)
-    try:
-        os.rename(tmpname, fname)
-    except (OSError, IOError):
-        os.unlink(fname)    # necessary on Windows
-        os.rename(tmpname, fname)
-
 # ____________________________________________________________
 
 def appdef(source, applevel=ApplevelClass, filename=None):
@@ -1184,11 +1032,6 @@
         return build_applevel_dict(self, space)
 
 
-class applevelinterp_temp(ApplevelClass):
-    hidden_applevel = False
-    def getwdict(self, space):   # no cache
-        return PyPyCacheDir.build_applevelinterp_dict(self, space)
-
 # app2interp_temp is used for testing mainly
 def app2interp_temp(func, applevel_temp=applevel_temp, filename=None):
     """ NOT_RPYTHON """
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -15,9 +15,8 @@
 from pypy.rlib.rarithmetic import r_uint, intmask
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.debug import check_nonneg
-from pypy.tool.stdlib_opcode import (bytecode_spec, host_bytecode_spec,
-                                     unrolling_all_opcode_descs, opmap,
-                                     host_opmap)
+from pypy.tool.stdlib_opcode import (bytecode_spec,
+                                     unrolling_all_opcode_descs)
 
 def unaryoperation(operationname):
     """NOT_RPYTHON"""
@@ -713,6 +712,19 @@
         w_list = self.space.newlist(items)
         self.pushvalue(w_list)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # this is a little dance, because list has to be before the
+        # value
+        last_val = self.popvalue()
+        try:
+            lgt = self.space.len_w(last_val)
+        except OperationError, e:
+            if e.async(self.space):
+                raise
+            lgt = 0 # oh well
+        self.pushvalue(self.space.newlist([], sizehint=lgt))
+        self.pushvalue(last_val)
+
     def LOAD_ATTR(self, nameindex, next_instr):
         "obj.attributename"
         w_obj = self.popvalue()
@@ -1419,11 +1431,9 @@
             if lastchar.isspace() and lastchar != ' ':
                 return
         file_softspace(stream, True)
-    print_item_to._annspecialcase_ = "specialize:argtype(0)"
 
     def print_item(x):
         print_item_to(x, sys_stdout())
-    print_item._annspecialcase_ = "flowspace:print_item"
 
     def print_newline_to(stream):
         stream.write("\n")
@@ -1431,7 +1441,6 @@
 
     def print_newline():
         print_newline_to(sys_stdout())
-    print_newline._annspecialcase_ = "flowspace:print_newline"
 
     def file_softspace(file, newflag):
         try:
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -2,34 +2,39 @@
 from pypy.interpreter import unicodehelper
 from pypy.rlib.rstring import StringBuilder
 
-def parsestr(space, encoding, s, unicode_literals=False):
-    # compiler.transformer.Transformer.decode_literal depends on what 
-    # might seem like minor details of this function -- changes here 
-    # must be reflected there.
+def parsestr(space, encoding, s, unicode_literal=False):
+    """Parses a string or unicode literal, and return a wrapped value.
+
+    If encoding=iso8859-1, the source string is also in this encoding.
+    If encoding=None, the source string is ascii only.
+    In other cases, the source string is in utf-8 encoding.
+
+    When a bytes string is returned, it will be encoded with the
+    original encoding.
+
+    Yes, it's very inefficient.
+    Yes, CPython has very similar code.
+    """
 
     # we use ps as "pointer to s"
     # q is the virtual last char index of the string
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = unicode_literals
 
     # string decoration handling
-    o = ord(quote)
-    isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
-    if isalpha or quote == '_':
-        if quote == 'b' or quote == 'B':
-            ps += 1
-            quote = s[ps]
-            unicode = False
-        elif quote == 'u' or quote == 'U':
-            ps += 1
-            quote = s[ps]
-            unicode = True
-        if quote == 'r' or quote == 'R':
-            ps += 1
-            quote = s[ps]
-            rawmode = True
+    if quote == 'b' or quote == 'B':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = False
+    elif quote == 'u' or quote == 'U':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = True
+    if quote == 'r' or quote == 'R':
+        ps += 1
+        quote = s[ps]
+        rawmode = True
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
                              'Internal error: parser passed unquoted literal')
@@ -46,21 +51,28 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
-    if unicode: # XXX Py_UnicodeFlag is ignored for now
+    if unicode_literal: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None or encoding == "iso-8859-1":
+            # 'unicode_escape' expects latin-1 bytes, string is ready.
             buf = s
             bufp = ps
             bufq = q
             u = None
         else:
-            # "\XX" may become "\u005c\uHHLL" (12 bytes)
+            # String is utf8-encoded, but 'unicode_escape' expects
+            # latin-1; So multibyte sequences must be escaped.
             lis = [] # using a list to assemble the value
             end = q
+            # Worst case: "\XX" may become "\u005c\uHHLL" (12 bytes)
             while ps < end:
                 if s[ps] == '\\':
                     lis.append(s[ps])
                     ps += 1
                     if ord(s[ps]) & 0x80:
+                        # A multibyte sequence will follow, it will be
+                        # escaped like \u1234. To avoid confusion with
+                        # the backslash we just wrote, we emit "\u005c"
+                        # instead.
                         lis.append("u005c")
                 if ord(s[ps]) & 0x80: # XXX inefficient
                     w, ps = decode_utf8(space, s, ps, end, "utf-16-be")
@@ -86,13 +98,11 @@
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
-    # XXX add strchr like interface to rtyper
     assert 0 <= ps <= q
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
             w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
-            #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
diff --git a/pypy/interpreter/test/test_appinterp.py b/pypy/interpreter/test/test_appinterp.py
--- a/pypy/interpreter/test/test_appinterp.py
+++ b/pypy/interpreter/test/test_appinterp.py
@@ -1,6 +1,6 @@
 
 import py
-from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp, applevelinterp_temp
+from pypy.interpreter.gateway import appdef, ApplevelClass, applevel_temp
 from pypy.interpreter.error import OperationError
 
 def test_execwith_novars(space): 
@@ -82,9 +82,6 @@
     w_res = g(space, space.wrap(10), space.wrap(1))
     assert space.eq_w(w_res, space.wrap(-9))
 
-def test_applevelinterp_functions(space):
-    test_applevel_functions(space, applevel_temp = applevelinterp_temp)
-
 def test_applevel_class(space, applevel_temp = applevel_temp):
     app = applevel_temp('''
         class C(object):
@@ -99,9 +96,6 @@
     w_clsattr = space.getattr(c, space.wrap('attr'))
     assert space.eq_w(w_clsattr, space.wrap(17))
 
-def test_applevelinterp_class(space):
-    test_applevel_class(space, applevel_temp = applevelinterp_temp)
-
 def app_test_something_at_app_level(): 
     x = 2
     assert x/2 == 1
@@ -161,7 +155,7 @@
         w_str = space1.getattr(w_mymod1, space1.wrap("hi"))
         assert space1.str_w(w_str) == "hello"
 
-    def test_geninterp_can_unfreeze(self):
+    def test_random_stuff_can_unfreeze(self):
         # When a module contains an "import" statement in applevel code, the
         # imported module is initialized, possibly after it has been already
         # frozen.
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -101,14 +101,6 @@
         g3 = gateway.app2interp_temp(noapp_g3, gateway.applevel_temp)
         assert self.space.eq_w(g3(self.space, w('foo'), w('bar')), w('foobar'))
 
-    def test_app2interp2(self):
-        """same but using transformed code"""
-        w = self.space.wrap
-        def noapp_g3(a, b):
-            return a+b
-        g3 = gateway.app2interp_temp(noapp_g3, gateway.applevelinterp_temp)
-        assert self.space.eq_w(g3(self.space, w('foo'), w('bar')), w('foobar'))
-
     def test_app2interp_general_args(self):
         w = self.space.wrap
         def app_general(x, *args, **kwds):
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -312,8 +312,8 @@
             mods = space.get_builtinmodule_to_install()
             
             assert '__pypy__' in mods                # real builtin
-            assert 'array' not in mods               # in lib_pypy
-            assert 'faked+array' not in mods         # in lib_pypy
+            assert '_functools' not in mods               # in lib_pypy
+            assert 'faked+_functools' not in mods         # in lib_pypy
             assert 'this_doesnt_exist' not in mods   # not in lib_pypy
             assert 'faked+this_doesnt_exist' in mods # not in lib_pypy, but in
                                                      # ALL_BUILTIN_MODULES
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -75,6 +75,7 @@
 class AppTestInterpObjectPickling:
     pytestmark = py.test.mark.skipif("config.option.runappdirect")
     def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['struct'])
         _attach_helpers(cls.space)
 
     def teardown_class(cls):
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -171,7 +171,7 @@
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
     'cast_int_to_ptr' : (('int',), 'ref'),
-    'debug_merge_point': (('ref', 'int'), None),
+    'debug_merge_point': (('ref', 'int', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
     'guard_not_forced': ((), None),
@@ -1797,6 +1797,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1813,6 +1814,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -208,6 +208,7 @@
     This is the class supporting --gcrootfinder=asmgcc.
     """
     is_shadow_stack = False
+    is_64_bit = (WORD == 8)
 
     LOC_REG       = 0
     LOC_ESP_PLUS  = 1
@@ -336,17 +337,17 @@
             self._gcmap_deadentries += 1
             item += asmgcroot.arrayitemsize
 
-    def get_basic_shape(self, is_64_bit=False):
+    def get_basic_shape(self):
         # XXX: Should this code even really know about stack frame layout of
         # the JIT?
-        if is_64_bit:
-            return [chr(self.LOC_EBP_PLUS  | 8),
-                    chr(self.LOC_EBP_MINUS | 8),
-                    chr(self.LOC_EBP_MINUS | 16),
-                    chr(self.LOC_EBP_MINUS | 24),
-                    chr(self.LOC_EBP_MINUS | 32),
-                    chr(self.LOC_EBP_MINUS | 40),
-                    chr(self.LOC_EBP_PLUS  | 0),
+        if self.is_64_bit:
+            return [chr(self.LOC_EBP_PLUS  | 4),    # return addr: at   8(%rbp)
+                    chr(self.LOC_EBP_MINUS | 4),    # saved %rbx:  at  -8(%rbp)
+                    chr(self.LOC_EBP_MINUS | 8),    # saved %r12:  at -16(%rbp)
+                    chr(self.LOC_EBP_MINUS | 12),   # saved %r13:  at -24(%rbp)
+                    chr(self.LOC_EBP_MINUS | 16),   # saved %r14:  at -32(%rbp)
+                    chr(self.LOC_EBP_MINUS | 20),   # saved %r15:  at -40(%rbp)
+                    chr(self.LOC_EBP_PLUS  | 0),    # saved %rbp:  at    (%rbp)
                     chr(0)]
         else:
             return [chr(self.LOC_EBP_PLUS  | 4),    # return addr: at   4(%ebp)
@@ -366,7 +367,11 @@
         shape.append(chr(number | flag))
 
     def add_frame_offset(self, shape, offset):
-        assert (offset & 3) == 0
+        if self.is_64_bit:
+            assert (offset & 7) == 0
+            offset >>= 1
+        else:
+            assert (offset & 3) == 0
         if offset >= 0:
             num = self.LOC_EBP_PLUS | offset
         else:
@@ -518,7 +523,7 @@
     def initialize(self):
         pass
 
-    def get_basic_shape(self, is_64_bit=False):
+    def get_basic_shape(self):
         return []
 
     def add_frame_offset(self, shape, offset):
@@ -594,7 +599,7 @@
         # if convenient for the backend, we compute the info about
         # the flag as (byte-offset, single-byte-flag).
         import struct
-        value = struct.pack("l", flag_word)
+        value = struct.pack(lltype.SignedFmt, flag_word)
         assert value.count('\x00') == len(value) - 1    # only one byte is != 0
         i = 0
         while value[i] == '\x00': i += 1
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -321,7 +321,7 @@
         except KeyError:
             pass   # 'var' is already not in a register
 
-    def loc(self, box):
+    def loc(self, box, must_exist=False):
         """ Return the location of 'box'.
         """
         self._check_type(box)
@@ -332,6 +332,8 @@
         except KeyError:
             if box in self.bindings_to_frame_reg:
                 return self.frame_reg
+            if must_exist:
+                return self.frame_manager.bindings[box]
             return self.frame_manager.loc(box)
 
     def return_constant(self, v, forbidden_vars=[], selected_reg=None):
@@ -360,7 +362,7 @@
         self._check_type(v)
         if isinstance(v, Const):
             return self.return_constant(v, forbidden_vars, selected_reg)
-        prev_loc = self.loc(v)
+        prev_loc = self.loc(v, must_exist=True)
         if prev_loc is self.frame_reg and selected_reg is None:
             return prev_loc
         loc = self.force_allocate_reg(v, forbidden_vars, selected_reg,
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -148,7 +148,7 @@
     #
     def get_alignment(code):
         # Retrieve default alignment for the compiler/platform
-        return struct.calcsize('l' + code) - struct.calcsize(code)
+        return struct.calcsize(lltype.SignedFmt + code) - struct.calcsize(code)
     assert descr1.basesize == get_alignment('c')
     assert descr2.basesize == get_alignment('p')
     assert descr3.basesize == get_alignment('p')
diff --git a/pypy/jit/backend/llsupport/test/test_ffisupport.py b/pypy/jit/backend/llsupport/test/test_ffisupport.py
--- a/pypy/jit/backend/llsupport/test/test_ffisupport.py
+++ b/pypy/jit/backend/llsupport/test/test_ffisupport.py
@@ -2,6 +2,7 @@
 from pypy.jit.codewriter.longlong import is_64_bit
 from pypy.jit.backend.llsupport.descr import *
 from pypy.jit.backend.llsupport.ffisupport import *
+from pypy.rlib.rarithmetic import is_emulated_long
 
 
 class FakeCPU:
@@ -43,7 +44,7 @@
     assert descr.result_flag == FLAG_UNSIGNED
     assert descr.is_result_signed() == False
 
-    if not is_64_bit:
+    if not is_64_bit or is_emulated_long:
         descr = get_call_descr_dynamic(FakeCPU(), [], types.slonglong,
                                        None, 42)
         assert descr is None   # missing longlongs
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -11,6 +11,7 @@
 from pypy.jit.tool.oparser import parse
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
 from pypy.jit.metainterp.optimizeopt.util import equaloplists
+from pypy.rlib.rarithmetic import is_valid_int
 
 def test_boehm():
     gc_ll_descr = GcLLDescr_boehm(None, None, None)
@@ -57,6 +58,7 @@
         def frame_pos(n):
             return -4*(4+n)
         gcrootmap = GcRootMap_asmgcc()
+        gcrootmap.is_64_bit = False
         num1 = frame_pos(-5)
         num1a = num1|2
         num2 = frame_pos(55)
@@ -102,7 +104,7 @@
         gcrootmap.put(retaddr, shapeaddr)
         assert gcrootmap._gcmap[0] == retaddr
         assert gcrootmap._gcmap[1] == shapeaddr
-        p = rffi.cast(rffi.LONGP, gcrootmap.gcmapstart())
+        p = rffi.cast(rffi.SIGNEDP, gcrootmap.gcmapstart())
         assert p[0] == retaddr
         assert (gcrootmap.gcmapend() ==
                 gcrootmap.gcmapstart() + rffi.sizeof(lltype.Signed) * 2)
@@ -418,9 +420,9 @@
         assert newops[0].getarg(1) == v_value
         assert newops[0].result is None
         wbdescr = newops[0].getdescr()
-        assert isinstance(wbdescr.jit_wb_if_flag, int)
-        assert isinstance(wbdescr.jit_wb_if_flag_byteofs, int)
-        assert isinstance(wbdescr.jit_wb_if_flag_singlebyte, int)
+        assert is_valid_int(wbdescr.jit_wb_if_flag)
+        assert is_valid_int(wbdescr.jit_wb_if_flag_byteofs)
+        assert is_valid_int(wbdescr.jit_wb_if_flag_singlebyte)
 
     def test_get_rid_of_debug_merge_point(self):
         operations = [
diff --git a/pypy/jit/backend/llsupport/test/test_regalloc.py b/pypy/jit/backend/llsupport/test/test_regalloc.py
--- a/pypy/jit/backend/llsupport/test/test_regalloc.py
+++ b/pypy/jit/backend/llsupport/test/test_regalloc.py
@@ -1,4 +1,4 @@
-
+import py
 from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT
 from pypy.jit.backend.llsupport.regalloc import FrameManager
 from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan
@@ -236,6 +236,16 @@
         assert isinstance(loc, FakeFramePos)
         assert len(asm.moves) == 1
 
+    def test_bogus_make_sure_var_in_reg(self):
+        b0, = newboxes(0)
+        longevity = {b0: (0, 1)}
+        fm = TFrameManager()
+        asm = MockAsm()
+        rm = RegisterManager(longevity, frame_manager=fm, assembler=asm)
+        rm.next_instruction()
+        # invalid call to make_sure_var_in_reg(): box unknown so far
+        py.test.raises(KeyError, rm.make_sure_var_in_reg, b0)
+
     def test_return_constant(self):
         asm = MockAsm()
         boxes, longevity = boxes_and_longevity(5)
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -16,15 +16,23 @@
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import intmask
+from pypy.rlib import longlong2float
+from pypy.rlib.rarithmetic import intmask, is_valid_int
 from pypy.jit.backend.detect_cpu import autodetect_main_model_and_size
 
+
 def boxfloat(x):
     return BoxFloat(longlong.getfloatstorage(x))
 
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
 
+def boxlonglong(ll):
+    if longlong.is_64_bit:
+        return BoxInt(ll)
+    else:
+        return BoxFloat(ll)
+
 
 class Runner(object):
 
@@ -493,7 +501,7 @@
         if cpu.supports_floats:
             def func(f, i):
                 assert isinstance(f, float)
-                assert isinstance(i, int)
+                assert is_valid_int(i)
                 return f - float(i)
             FPTR = self.Ptr(self.FuncType([lltype.Float, lltype.Signed],
                                           lltype.Float))
@@ -604,7 +612,7 @@
                                          [funcbox, BoxInt(arg1), BoxInt(arg2)],
                                          'int', descr=calldescr)
             assert res.getint() == f(arg1, arg2)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -1490,18 +1498,36 @@
     def test_noops(self):
         c_box = self.alloc_string("hi there").constbox()
         c_nest = ConstInt(0)
-        self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest], 'void')
+        c_id = ConstInt(0)
+        self.execute_operation(rop.DEBUG_MERGE_POINT, [c_box, c_nest, c_id], 'void')
         self.execute_operation(rop.JIT_DEBUG, [c_box, c_nest, c_nest,
                                                c_nest, c_nest], 'void')
 
     def test_read_timestamp(self):
+        if sys.platform == 'win32':
+            # windows quite often is very inexact (like the old Intel 8259 PIC),
+            # so we stretch the time a little bit.
+            # On my virtual Parallels machine in a 2GHz Core i7 Mac Mini,
+            # the test starts working at delay == 21670 and stops at 20600000.
+            # We take the geometric mean value.
+            from math import log, exp
+            delay_min = 21670
+            delay_max = 20600000
+            delay = int(exp((log(delay_min)+log(delay_max))/2))
+            def wait_a_bit():
+                for i in xrange(delay): pass
+        else:
+            def wait_a_bit():
+                pass
         if longlong.is_64_bit:
             got1 = self.execute_operation(rop.READ_TIMESTAMP, [], 'int')
+            wait_a_bit()
             got2 = self.execute_operation(rop.READ_TIMESTAMP, [], 'int')
             res1 = got1.getint()
             res2 = got2.getint()
         else:
             got1 = self.execute_operation(rop.READ_TIMESTAMP, [], 'float')
+            wait_a_bit()
             got2 = self.execute_operation(rop.READ_TIMESTAMP, [], 'float')
             res1 = got1.getlonglong()
             res2 = got2.getlonglong()
@@ -1597,6 +1623,17 @@
                                      [BoxPtr(x)], 'int').value
         assert res == -19
 
+    def test_convert_float_bytes(self):
+        t = 'int' if longlong.is_64_bit else 'float'
+        res = self.execute_operation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG,
+                                     [boxfloat(2.5)], t).value
+        assert res == longlong2float.float2longlong(2.5)
+
+        bytes = longlong2float.float2longlong(2.5)
+        res = self.execute_operation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT,
+                                     [boxlonglong(res)], 'float').value
+        assert longlong.getrealfloat(res) == 2.5
+
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
@@ -3061,7 +3098,7 @@
             ResOperation(rop.JUMP, [i2], None, descr=targettoken2),
             ]
         self.cpu.compile_bridge(faildescr, inputargs, operations, looptoken)
-        
+
         fail = self.cpu.execute_token(looptoken, 2)
         assert fail.identifier == 3
         res = self.cpu.get_latest_value_int(0)
@@ -3106,7 +3143,7 @@
             assert len(mc) == len(ops)
             for i in range(len(mc)):
                 assert mc[i].split("\t")[-1].startswith(ops[i])
-            
+
         data = ctypes.string_at(info.asmaddr, info.asmlen)
         mc = list(machine_code_dump(data, info.asmaddr, cpuname))
         lines = [line for line in mc if line.count('\t') == 2]
diff --git a/pypy/jit/backend/test/support.py b/pypy/jit/backend/test/support.py
--- a/pypy/jit/backend/test/support.py
+++ b/pypy/jit/backend/test/support.py
@@ -3,6 +3,7 @@
 from pypy.rlib.debug import debug_print
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.jit.metainterp.optimizeopt import ALL_OPTS_NAMES
+from pypy.rlib.rarithmetic import is_valid_int
 
 class BaseCompiledMixin(object):
 
@@ -24,7 +25,7 @@
         from pypy.annotation import model as annmodel
 
         for arg in args:
-            assert isinstance(arg, int)
+            assert is_valid_int(arg)
 
         self.pre_translation_hook()
         t = self._get_TranslationContext()
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -328,6 +328,15 @@
     def produce_into(self, builder, r):
         self.put(builder, [r.choice(builder.intvars)])
 
+class CastLongLongToFloatOperation(AbstractFloatOperation):
+    def produce_into(self, builder, r):
+        if longlong.is_64_bit:
+            self.put(builder, [r.choice(builder.intvars)])
+        else:
+            if not builder.floatvars:
+                raise CannotProduceOperation
+            self.put(builder, [r.choice(builder.floatvars)])
+
 class CastFloatToIntOperation(AbstractFloatOperation):
     def produce_into(self, builder, r):
         if not builder.floatvars:
@@ -449,6 +458,8 @@
 
 OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT))
 OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT))
+OPERATIONS.append(CastFloatToIntOperation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG))
+OPERATIONS.append(CastLongLongToFloatOperation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT))
 
 OperationBuilder.OPERATIONS = OPERATIONS
 
@@ -502,11 +513,11 @@
     else:
         assert 0, "unknown backend %r" % pytest.config.option.backend
 
-# ____________________________________________________________    
+# ____________________________________________________________
 
 class RandomLoop(object):
     dont_generate_more = False
-    
+
     def __init__(self, cpu, builder_factory, r, startvars=None):
         self.cpu = cpu
         if startvars is None:
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -88,7 +88,6 @@
         self._debug = False
         self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i')
         self.fail_boxes_count = 0
-        self._current_depths_cache = (0, 0)
         self.datablockwrapper = None
         self.stack_check_slowpath = 0
         self.propagate_exception_path = 0
@@ -442,10 +441,8 @@
         looppos = self.mc.get_relative_pos()
         looptoken._x86_loop_code = looppos
         clt.frame_depth = -1     # temporarily
-        clt.param_depth = -1     # temporarily
-        frame_depth, param_depth = self._assemble(regalloc, operations)
+        frame_depth = self._assemble(regalloc, operations)
         clt.frame_depth = frame_depth
-        clt.param_depth = param_depth
         #
         size_excluding_failure_stuff = self.mc.get_relative_pos()
         self.write_pending_failure_recoveries()
@@ -459,8 +456,7 @@
             rawstart + size_excluding_failure_stuff,
             rawstart))
         debug_stop("jit-backend-addr")
-        self._patch_stackadjust(rawstart + stackadjustpos,
-                                frame_depth + param_depth)
+        self._patch_stackadjust(rawstart + stackadjustpos, frame_depth)
         self.patch_pending_failure_recoveries(rawstart)
         #
         ops_offset = self.mc.ops_offset
@@ -500,14 +496,13 @@
             assert ([loc.assembler() for loc in arglocs] ==
                     [loc.assembler() for loc in faildescr._x86_debug_faillocs])
         regalloc = RegAlloc(self, self.cpu.translate_support_code)
-        fail_depths = faildescr._x86_current_depths
         startpos = self.mc.get_relative_pos()
-        operations = regalloc.prepare_bridge(fail_depths, inputargs, arglocs,
+        operations = regalloc.prepare_bridge(inputargs, arglocs,
                                              operations,
                                              self.current_clt.allgcrefs)
 
         stackadjustpos = self._patchable_stackadjust()
-        frame_depth, param_depth = self._assemble(regalloc, operations)
+        frame_depth = self._assemble(regalloc, operations)
         codeendpos = self.mc.get_relative_pos()
         self.write_pending_failure_recoveries()
         fullsize = self.mc.get_relative_pos()
@@ -517,19 +512,16 @@
         debug_print("bridge out of Guard %d has address %x to %x" %
                     (descr_number, rawstart, rawstart + codeendpos))
         debug_stop("jit-backend-addr")
-        self._patch_stackadjust(rawstart + stackadjustpos,
-                                frame_depth + param_depth)
+        self._patch_stackadjust(rawstart + stackadjustpos, frame_depth)
         self.patch_pending_failure_recoveries(rawstart)
         if not we_are_translated():
             # for the benefit of tests
             faildescr._x86_bridge_frame_depth = frame_depth
-            faildescr._x86_bridge_param_depth = param_depth
         # patch the jump from original guard
         self.patch_jump_for_descr(faildescr, rawstart)
         ops_offset = self.mc.ops_offset
         self.fixup_target_tokens(rawstart)
         self.current_clt.frame_depth = max(self.current_clt.frame_depth, frame_depth)
-        self.current_clt.param_depth = max(self.current_clt.param_depth, param_depth)
         self.teardown()
         # oprofile support
         if self.cpu.profile_agent is not None:
@@ -614,7 +606,7 @@
         else:
             assert token
             struct.number = compute_unique_id(token)
-        self.loop_run_counters.append(struct)            
+        self.loop_run_counters.append(struct)
         return struct
 
     def _find_failure_recovery_bytecode(self, faildescr):
@@ -673,7 +665,7 @@
                ResOperation(rop.SETFIELD_RAW, [c_adr, box2],
                             None, descr=self.debug_counter_descr)]
         operations.extend(ops)
-        
+
     @specialize.argtype(1)
     def _inject_debugging_code(self, looptoken, operations, tp, number):
         if self._debug:
@@ -700,15 +692,12 @@
         regalloc.walk_operations(operations)
         if we_are_translated() or self.cpu.dont_keepalive_stuff:
             self._regalloc = None   # else keep it around for debugging
-        frame_depth = regalloc.fm.get_frame_depth()
-        param_depth = regalloc.param_depth
+        frame_depth = regalloc.get_final_frame_depth()
         jump_target_descr = regalloc.jump_target_descr
         if jump_target_descr is not None:
             target_frame_depth = jump_target_descr._x86_clt.frame_depth
-            target_param_depth = jump_target_descr._x86_clt.param_depth
             frame_depth = max(frame_depth, target_frame_depth)
-            param_depth = max(param_depth, target_param_depth)
-        return frame_depth, param_depth
+        return frame_depth
 
     def _patchable_stackadjust(self):
         # stack adjustment LEA
@@ -847,8 +836,8 @@
             self.mc.MOVSD_sx(0, loc.value)
         elif WORD == 4 and isinstance(loc, StackLoc) and loc.get_width() == 8:
             # XXX evil trick
-            self.mc.PUSH_b(get_ebp_ofs(loc.position))
-            self.mc.PUSH_b(get_ebp_ofs(loc.position + 1))
+            self.mc.PUSH_b(loc.value + 4)
+            self.mc.PUSH_b(loc.value)
         else:
             self.mc.PUSH(loc)
 
@@ -858,8 +847,8 @@
             self.mc.ADD_ri(esp.value, 8)   # = size of doubles
         elif WORD == 4 and isinstance(loc, StackLoc) and loc.get_width() == 8:
             # XXX evil trick
-            self.mc.POP_b(get_ebp_ofs(loc.position + 1))
-            self.mc.POP_b(get_ebp_ofs(loc.position))
+            self.mc.POP_b(loc.value)
+            self.mc.POP_b(loc.value + 4)
         else:
             self.mc.POP(loc)
 
@@ -892,10 +881,9 @@
         genop_math_list[oopspecindex](self, op, arglocs, resloc)
 
     def regalloc_perform_with_guard(self, op, guard_op, faillocs,
-                                    arglocs, resloc, current_depths):
+                                    arglocs, resloc):
         faildescr = guard_op.getdescr()
         assert isinstance(faildescr, AbstractFailDescr)
-        faildescr._x86_current_depths = current_depths
         failargs = guard_op.getfailargs()
         guard_opnum = guard_op.getopnum()
         guard_token = self.implement_guard_recovery(guard_opnum,
@@ -911,10 +899,9 @@
             # must be added by the genop_guard_list[]()
             assert guard_token is self.pending_guard_tokens[-1]
 
-    def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc,
-                               current_depths):
+    def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc):
         self.regalloc_perform_with_guard(None, guard_op, faillocs, arglocs,
-                                         resloc, current_depths)
+                                         resloc)
 
     def load_effective_addr(self, sizereg, baseofs, scale, result, frm=imm0):
         self.mc.LEA(result, addr_add(frm, sizereg, baseofs, scale))
@@ -1038,13 +1025,14 @@
                     self.mc.MOV(tmp, loc)
                     self.mc.MOV_sr(p, tmp.value)
             p += loc.get_width()
-        self._regalloc.reserve_param(p//WORD)
         # x is a location
         self.mc.CALL(x)
         self.mark_gc_roots(force_index)
         #
         if callconv != FFI_DEFAULT_ABI:
             self._fix_stdcall(callconv, p)
+        #
+        self._regalloc.needed_extra_stack_locations(p//WORD)
 
     def _fix_stdcall(self, callconv, p):
         from pypy.rlib.clibffi import FFI_STDCALL
@@ -1127,9 +1115,9 @@
             x = r10
         remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG)
 
-        self._regalloc.reserve_param(len(pass_on_stack))
         self.mc.CALL(x)
         self.mark_gc_roots(force_index)
+        self._regalloc.needed_extra_stack_locations(len(pass_on_stack))
 
     def call(self, addr, args, res):
         force_index = self.write_new_force_index()
@@ -1254,6 +1242,24 @@
         self.mc.MOVD_xr(resloc.value, loc0.value)
         self.mc.CVTSS2SD_xx(resloc.value, resloc.value)
 
+    def genop_convert_float_bytes_to_longlong(self, op, arglocs, resloc):
+        loc0, = arglocs
+        if longlong.is_64_bit:
+            assert isinstance(resloc, RegLoc)
+            assert isinstance(loc0, RegLoc)
+            self.mc.MOVD(resloc, loc0)
+        else:
+            self.mov(loc0, resloc)
+
+    def genop_convert_longlong_bytes_to_float(self, op, arglocs, resloc):
+        loc0, = arglocs
+        if longlong.is_64_bit:
+            assert isinstance(resloc, RegLoc)
+            assert isinstance(loc0, RegLoc)
+            self.mc.MOVD(resloc, loc0)
+        else:
+            self.mov(loc0, resloc)
+
     def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
         guard_opnum = guard_op.getopnum()
         self.mc.CMP(arglocs[0], imm0)
@@ -1966,8 +1972,6 @@
             mc.PUSH_r(ebx.value)
         elif IS_X86_64:
             mc.MOV_rr(edi.value, ebx.value)
-            # XXX: Correct to only align the stack on 64-bit?
-            mc.AND_ri(esp.value, -16)
         else:
             raise AssertionError("Shouldn't happen")
 
@@ -2129,14 +2133,16 @@
         # First, we need to save away the registers listed in
         # 'save_registers' that are not callee-save.  XXX We assume that
         # the XMM registers won't be modified.  We store them in
-        # [ESP+4], [ESP+8], etc., leaving enough room in [ESP] for the
-        # single argument to closestack_addr below.
-        p = WORD
+        # [ESP+4], [ESP+8], etc.; on x86-32 we leave enough room in [ESP]
+        # for the single argument to closestack_addr below.
+        if IS_X86_32:
+            p = WORD
+        elif IS_X86_64:
+            p = 0
         for reg in self._regalloc.rm.save_around_call_regs:
             if reg in save_registers:
                 self.mc.MOV_sr(p, reg.value)
                 p += WORD
-        self._regalloc.reserve_param(p//WORD)
         #
         if gcrootmap.is_shadow_stack:
             args = []
@@ -2187,11 +2193,15 @@
         #
         self._emit_call(-1, imm(self.releasegil_addr), args)
         # Finally, restore the registers saved above.
-        p = WORD
+        if IS_X86_32:
+            p = WORD
+        elif IS_X86_64:
+            p = 0
         for reg in self._regalloc.rm.save_around_call_regs:
             if reg in save_registers:
                 self.mc.MOV_rs(reg.value, p)
                 p += WORD
+        self._regalloc.needed_extra_stack_locations(p//WORD)
 
     def call_reacquire_gil(self, gcrootmap, save_loc):
         # save the previous result (eax/xmm0) into the stack temporarily.
@@ -2199,7 +2209,6 @@
         # to save xmm0 in this case.
         if isinstance(save_loc, RegLoc) and not save_loc.is_xmm:
             self.mc.MOV_sr(WORD, save_loc.value)
-            self._regalloc.reserve_param(2)
         # call the reopenstack() function (also reacquiring the GIL)
         if gcrootmap.is_shadow_stack:
             args = []
@@ -2219,6 +2228,7 @@
         # restore the result from the stack
         if isinstance(save_loc, RegLoc) and not save_loc.is_xmm:
             self.mc.MOV_rs(save_loc.value, WORD)
+            self._regalloc.needed_extra_stack_locations(2)
 
     def genop_guard_call_assembler(self, op, guard_op, guard_token,
                                    arglocs, result_loc):
@@ -2495,11 +2505,6 @@
         # copy of heap(nursery_free_adr), so that the final MOV below is
         # a no-op.
 
-        # reserve room for the argument to the real malloc and the
-        # saved XMM regs (on 32 bit: 8 * 2 words; on 64 bit: 16 * 1
-        # word)
-        self._regalloc.reserve_param(1+16)
-
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         shadow_stack = (gcrootmap is not None and gcrootmap.is_shadow_stack)
         if not shadow_stack:
@@ -2510,6 +2515,11 @@
         slowpath_addr2 = self.malloc_slowpath2
         self.mc.CALL(imm(slowpath_addr2))
 
+        # reserve room for the argument to the real malloc and the
+        # saved XMM regs (on 32 bit: 8 * 2 words; on 64 bit: 16 * 1
+        # word)
+        self._regalloc.needed_extra_stack_locations(1+16)
+
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr-1, chr(offset))
diff --git a/pypy/jit/backend/x86/codebuf.py b/pypy/jit/backend/x86/codebuf.py
--- a/pypy/jit/backend/x86/codebuf.py
+++ b/pypy/jit/backend/x86/codebuf.py
@@ -19,8 +19,8 @@
 
 
 class MachineCodeBlockWrapper(BlockBuilderMixin,
-                              codebuilder_cls,
-                              LocationCodeBuilder):
+                              LocationCodeBuilder,
+                              codebuilder_cls):
     def __init__(self):
         self.init_block_builder()
         # a list of relative positions; for each position p, the bytes
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -168,7 +168,7 @@
 
     def _prepare(self, inputargs, operations, allgcrefs):
         self.fm = X86FrameManager()
-        self.param_depth = 0
+        self.min_frame_depth = 0
         cpu = self.assembler.cpu
         operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations,
                                                        allgcrefs)
@@ -193,11 +193,9 @@
             self.min_bytes_before_label = 13
         return operations
 
-    def prepare_bridge(self, prev_depths, inputargs, arglocs, operations,
-                       allgcrefs):
+    def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs):
         operations = self._prepare(inputargs, operations, allgcrefs)
         self._update_bindings(arglocs, inputargs)
-        self.param_depth = prev_depths[1]
         self.min_bytes_before_label = 0
         return operations
 
@@ -205,8 +203,15 @@
         self.min_bytes_before_label = max(self.min_bytes_before_label,
                                           at_least_position)
 
-    def reserve_param(self, n):
-        self.param_depth = max(self.param_depth, n)
+    def needed_extra_stack_locations(self, n):
+        # call *after* you needed extra stack locations: (%esp), (%esp+4)...
+        min_frame_depth = self.fm.get_frame_depth() + n
+        if min_frame_depth > self.min_frame_depth:
+            self.min_frame_depth = min_frame_depth
+
+    def get_final_frame_depth(self):
+        self.needed_extra_stack_locations(0)  # update min_frame_depth
+        return self.min_frame_depth
 
     def _set_initial_bindings(self, inputargs):
         if IS_X86_64:
@@ -376,25 +381,12 @@
     def locs_for_fail(self, guard_op):
         return [self.loc(v) for v in guard_op.getfailargs()]
 
-    def get_current_depth(self):
-        # return (self.fm.frame_depth, self.param_depth), but trying to share
-        # the resulting tuple among several calls
-        arg0 = self.fm.get_frame_depth()
-        arg1 = self.param_depth
-        result = self.assembler._current_depths_cache
-        if result[0] != arg0 or result[1] != arg1:
-            result = (arg0, arg1)
-            self.assembler._current_depths_cache = result
-        return result
-
     def perform_with_guard(self, op, guard_op, arglocs, result_loc):
         faillocs = self.locs_for_fail(guard_op)
         self.rm.position += 1
         self.xrm.position += 1
-        current_depths = self.get_current_depth()
         self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs,
-                                                   arglocs, result_loc,
-                                                   current_depths)
+                                                   arglocs, result_loc)
         if op.result is not None:
             self.possibly_free_var(op.result)
         self.possibly_free_vars(guard_op.getfailargs())
@@ -407,10 +399,8 @@
                                                       arglocs))
             else:
                 self.assembler.dump('%s(%s)' % (guard_op, arglocs))
-        current_depths = self.get_current_depth()
         self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs,
-                                              result_loc,
-                                              current_depths)
+                                              result_loc)
         self.possibly_free_vars(guard_op.getfailargs())
 
     def PerformDiscard(self, op, arglocs):
@@ -776,6 +766,32 @@
 
     consider_cast_singlefloat_to_float = consider_cast_int_to_float
 
+    def consider_convert_float_bytes_to_longlong(self, op):
+        if longlong.is_64_bit:
+            loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0))
+            loc1 = self.rm.force_allocate_reg(op.result)
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(op.getarg(0))
+        else:
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.loc(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
+
+    def consider_convert_longlong_bytes_to_float(self, op):
+        if longlong.is_64_bit:
+            loc0 = self.rm.make_sure_var_in_reg(op.getarg(0))
+            loc1 = self.xrm.force_allocate_reg(op.result)
+            self.Perform(op, [loc0], loc1)
+            self.rm.possibly_free_var(op.getarg(0))
+        else:
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.make_sure_var_in_reg(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
+
     def _consider_llong_binop_xx(self, op):
         # must force both arguments into xmm registers, because we don't
         # know if they will be suitably aligned.  Exception: if the second
@@ -1393,7 +1409,7 @@
         self.force_spill_var(op.getarg(0))
 
     def get_mark_gc_roots(self, gcrootmap, use_copy_area=False):
-        shape = gcrootmap.get_basic_shape(IS_X86_64)
+        shape = gcrootmap.get_basic_shape()
         for v, val in self.fm.bindings.items():
             if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)):
                 assert isinstance(val, StackLoc)
diff --git a/pypy/jit/backend/x86/rx86.py b/pypy/jit/backend/x86/rx86.py
--- a/pypy/jit/backend/x86/rx86.py
+++ b/pypy/jit/backend/x86/rx86.py
@@ -601,9 +601,12 @@
     CVTSS2SD_xb = xmminsn('\xF3', rex_nw, '\x0F\x5A',
                           register(1, 8), stack_bp(2))
 
-    MOVD_rx = xmminsn('\x66', rex_nw, '\x0F\x7E', register(2, 8), register(1), '\xC0')
-    MOVD_xr = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), register(2), '\xC0')
-    MOVD_xb = xmminsn('\x66', rex_nw, '\x0F\x6E', register(1, 8), stack_bp(2))
+    # These work on machine sized registers, so MOVD is actually MOVQ
+    # when running on 64 bits.  Note a bug in the Intel documentation:
+    # http://lists.gnu.org/archive/html/bug-binutils/2007-07/msg00095.html
+    MOVD_rx = xmminsn('\x66', rex_w, '\x0F\x7E', register(2, 8), register(1), '\xC0')
+    MOVD_xr = xmminsn('\x66', rex_w, '\x0F\x6E', register(1, 8), register(2), '\xC0')
+    MOVD_xb = xmminsn('\x66', rex_w, '\x0F\x6E', register(1, 8), stack_bp(2))
 
     PSRAD_xi = xmminsn('\x66', rex_nw, '\x0F\x72', register(1), '\xE0', immediate(2, 'b'))
 
diff --git a/pypy/jit/backend/x86/support.py b/pypy/jit/backend/x86/support.py
--- a/pypy/jit/backend/x86/support.py
+++ b/pypy/jit/backend/x86/support.py
@@ -36,15 +36,15 @@
 
 # ____________________________________________________________
 
-if sys.platform == 'win32':
-    ensure_sse2_floats = lambda : None
-    # XXX check for SSE2 on win32 too
+if WORD == 4:
+    extra = ['-DPYPY_X86_CHECK_SSE2']
 else:
-    if WORD == 4:
-        extra = ['-DPYPY_X86_CHECK_SSE2']
-    else:
-        extra = []
-    ensure_sse2_floats = rffi.llexternal_use_eci(ExternalCompilationInfo(
-        compile_extra = ['-msse2', '-mfpmath=sse',
-                         '-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra,
-        ))
+    extra = []
+
+if sys.platform != 'win32':
+    extra = ['-msse2', '-mfpmath=sse',
+             '-DPYPY_CPU_HAS_STANDARD_PRECISION'] + extra
+
+ensure_sse2_floats = rffi.llexternal_use_eci(ExternalCompilationInfo(
+    compile_extra = extra,
+))
diff --git a/pypy/jit/backend/x86/test/conftest.py b/pypy/jit/backend/x86/test/conftest.py
--- a/pypy/jit/backend/x86/test/conftest.py
+++ b/pypy/jit/backend/x86/test/conftest.py
@@ -1,4 +1,4 @@
-import py
+import py, os
 from pypy.jit.backend import detect_cpu
 
 cpu = detect_cpu.autodetect()
@@ -6,5 +6,7 @@
     if cpu not in ('x86', 'x86_64'):
         py.test.skip("x86/x86_64 tests skipped: cpu is %r" % (cpu,))
     if cpu == 'x86_64':
+        if os.name == "nt":
+            py.test.skip("Windows cannot allocate non-reserved memory")
         from pypy.rpython.lltypesystem import ll2ctypes
         ll2ctypes.do_allocation_in_far_regions()
diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py
--- a/pypy/jit/backend/x86/test/test_gc_integration.py
+++ b/pypy/jit/backend/x86/test/test_gc_integration.py
@@ -28,7 +28,7 @@
 
 class MockGcRootMap(object):
     is_shadow_stack = False
-    def get_basic_shape(self, is_64_bit):
+    def get_basic_shape(self):
         return ['shape']
     def add_frame_offset(self, shape, offset):
         shape.append(offset)
diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py
--- a/pypy/jit/backend/x86/test/test_recompilation.py
+++ b/pypy/jit/backend/x86/test/test_recompilation.py
@@ -34,7 +34,6 @@
         '''
         loop = self.interpret(ops, [0])
         previous = loop._jitcelltoken.compiled_loop_token.frame_depth
-        assert loop._jitcelltoken.compiled_loop_token.param_depth == 0
         assert self.getint(0) == 20
         ops = '''
         [i1]
@@ -51,7 +50,6 @@
         bridge = self.attach_bridge(ops, loop, -2)
         descr = loop.operations[3].getdescr()
         new = descr._x86_bridge_frame_depth
-        assert descr._x86_bridge_param_depth == 0
         # the force_spill() forces the stack to grow
         assert new > previous
         fail = self.run(loop, 0)
@@ -116,10 +114,8 @@
         loop_frame_depth = loop._jitcelltoken.compiled_loop_token.frame_depth
         bridge = self.attach_bridge(ops, loop, 6)
         guard_op = loop.operations[6]
-        assert loop._jitcelltoken.compiled_loop_token.param_depth == 0
         # the force_spill() forces the stack to grow
         assert guard_op.getdescr()._x86_bridge_frame_depth > loop_frame_depth
-        assert guard_op.getdescr()._x86_bridge_param_depth == 0
         self.run(loop, 0, 0, 0, 0, 0, 0)
         assert self.getint(0) == 1
         assert self.getint(1) == 20
diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py
--- a/pypy/jit/backend/x86/test/test_regalloc.py
+++ b/pypy/jit/backend/x86/test/test_regalloc.py
@@ -606,23 +606,37 @@
         assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1]
 
 class TestRegAllocCallAndStackDepth(BaseTestRegalloc):
-    def expected_param_depth(self, num_args):
+    def expected_frame_depth(self, num_call_args, num_pushed_input_args=0):
         # Assumes the arguments are all non-float
         if IS_X86_32:
-            return num_args
+            extra_esp = num_call_args
+            return extra_esp
         elif IS_X86_64:
-            return max(num_args - 6, 0)
+            # 'num_pushed_input_args' is for X86_64 only
+            extra_esp = max(num_call_args - 6, 0)
+            return num_pushed_input_args + extra_esp
 
     def test_one_call(self):
         ops = '''
-        [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9]
+        [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i9b]
         i10 = call(ConstClass(f1ptr), i0, descr=f1_calldescr)
-        finish(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9)
+        finish(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, i9b)
         '''
-        loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9])
-        assert self.getints(10) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9]
+        loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9, 8])
+        assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 8]
         clt = loop._jitcelltoken.compiled_loop_token
-        assert clt.param_depth == self.expected_param_depth(1)
+        assert clt.frame_depth == self.expected_frame_depth(1, 5)
+
+    def test_one_call_reverse(self):
+        ops = '''
+        [i1, i2, i3, i4, i5, i6, i7, i8, i9, i9b, i0]
+        i10 = call(ConstClass(f1ptr), i0, descr=f1_calldescr)
+        finish(i10, i1, i2, i3, i4, i5, i6, i7, i8, i9, i9b)
+        '''
+        loop = self.interpret(ops, [7, 9, 9 ,9, 9, 9, 9, 9, 9, 8, 4])
+        assert self.getints(11) == [5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 8]
+        clt = loop._jitcelltoken.compiled_loop_token
+        assert clt.frame_depth == self.expected_frame_depth(1, 6)
 
     def test_two_calls(self):
         ops = '''
@@ -634,7 +648,7 @@
         loop = self.interpret(ops, [4, 7, 9, 9 ,9, 9, 9, 9, 9, 9])
         assert self.getints(10) == [5*7, 7, 9, 9, 9, 9, 9, 9, 9, 9]
         clt = loop._jitcelltoken.compiled_loop_token
-        assert clt.param_depth == self.expected_param_depth(2)
+        assert clt.frame_depth == self.expected_frame_depth(2, 5)
 
     def test_call_many_arguments(self):
         # NB: The first and last arguments in the call are constants. This
@@ -648,25 +662,31 @@
         loop = self.interpret(ops, [2, 3, 4, 5, 6, 7, 8, 9])
         assert self.getint(0) == 55
         clt = loop._jitcelltoken.compiled_loop_token
-        assert clt.param_depth == self.expected_param_depth(10)
+        assert clt.frame_depth == self.expected_frame_depth(10)
 
     def test_bridge_calls_1(self):
         ops = '''
         [i0, i1]
         i2 = call(ConstClass(f1ptr), i0, descr=f1_calldescr)
-        guard_value(i2, 0, descr=fdescr1) [i2, i1]
+        guard_value(i2, 0, descr=fdescr1) [i2, i0, i1]
         finish(i1)
         '''
         loop = self.interpret(ops, [4, 7])
         assert self.getint(0) == 5
+        clt = loop._jitcelltoken.compiled_loop_token
+        orgdepth = clt.frame_depth
+        assert orgdepth == self.expected_frame_depth(1, 2)
+
         ops = '''
-        [i2, i1]
+        [i2, i0, i1]
         i3 = call(ConstClass(f2ptr), i2, i1, descr=f2_calldescr)        
-        finish(i3, descr=fdescr2)        
+        finish(i3, i0, descr=fdescr2)
         '''
         bridge = self.attach_bridge(ops, loop, -2)
 
-        assert loop.operations[-2].getdescr()._x86_bridge_param_depth == self.expected_param_depth(2)
+        assert clt.frame_depth == max(orgdepth, self.expected_frame_depth(2, 2))
+        assert loop.operations[-2].getdescr()._x86_bridge_frame_depth == \
+            self.expected_frame_depth(2, 2)
 
         self.run(loop, 4, 7)
         assert self.getint(0) == 5*7
@@ -676,10 +696,14 @@
         [i0, i1]
         i2 = call(ConstClass(f2ptr), i0, i1, descr=f2_calldescr)
         guard_value(i2, 0, descr=fdescr1) [i2]
-        finish(i1)
+        finish(i2)
         '''
         loop = self.interpret(ops, [4, 7])
         assert self.getint(0) == 4*7
+        clt = loop._jitcelltoken.compiled_loop_token
+        orgdepth = clt.frame_depth
+        assert orgdepth == self.expected_frame_depth(2)
+
         ops = '''
         [i2]
         i3 = call(ConstClass(f1ptr), i2, descr=f1_calldescr)        
@@ -687,7 +711,9 @@
         '''
         bridge = self.attach_bridge(ops, loop, -2)
 
-        assert loop.operations[-2].getdescr()._x86_bridge_param_depth == self.expected_param_depth(2)
+        assert clt.frame_depth == max(orgdepth, self.expected_frame_depth(1))
+        assert loop.operations[-2].getdescr()._x86_bridge_frame_depth == \
+            self.expected_frame_depth(1)
 
         self.run(loop, 4, 7)
         assert self.getint(0) == 29
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -371,7 +371,7 @@
 
         operations = [
             ResOperation(rop.LABEL, [i0], None, descr=targettoken),
-            ResOperation(rop.DEBUG_MERGE_POINT, [FakeString("hello"), 0], None),
+            ResOperation(rop.DEBUG_MERGE_POINT, [FakeString("hello"), 0, 0], None),
             ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
             ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
             ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1),
@@ -390,7 +390,7 @@
         bridge = [
             ResOperation(rop.INT_LE, [i1b, ConstInt(19)], i3),
             ResOperation(rop.GUARD_TRUE, [i3], None, descr=faildescr2),
-            ResOperation(rop.DEBUG_MERGE_POINT, [FakeString("bye"), 0], None),
+            ResOperation(rop.DEBUG_MERGE_POINT, [FakeString("bye"), 0, 0], None),
             ResOperation(rop.JUMP, [i1b], None, descr=targettoken),
         ]
         bridge[1].setfailargs([i1b])
@@ -531,12 +531,12 @@
         loop = """
         [i0]
         label(i0, descr=preambletoken)
-        debug_merge_point('xyz', 0)
+        debug_merge_point('xyz', 0, 0)
         i1 = int_add(i0, 1)
         i2 = int_ge(i1, 10)
         guard_false(i2) []
         label(i1, descr=targettoken)
-        debug_merge_point('xyz', 0)
+        debug_merge_point('xyz', 0, 0)
         i11 = int_add(i1, 1)
         i12 = int_ge(i11, 10)
         guard_false(i12) []
@@ -569,7 +569,7 @@
         loop = """
         [i0]
         label(i0, descr=targettoken)
-        debug_merge_point('xyz', 0)
+        debug_merge_point('xyz', 0, 0)
         i1 = int_add(i0, 1)
         i2 = int_ge(i1, 10)
         guard_false(i2) []
diff --git a/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -182,6 +182,12 @@
         filename  = str(testdir.join(FILENAME  % methname))
         g = open(inputname, 'w')
         g.write('\x09.string "%s"\n' % BEGIN_TAG)
+        #
+        if instrname == 'MOVD' and self.WORD == 8:
+            instrname = 'MOVQ'
+            if argmodes == 'xb':
+                py.test.skip('"as" uses an undocumented alternate encoding??')
+        #
         for args in args_lists:
             suffix = ""
     ##        all = instr.as_all_suffixes
@@ -229,9 +235,6 @@
                 # movq $xxx, %rax => movl $xxx, %eax
                 suffix = 'l'
                 ops[1] = reduce_to_32bit(ops[1])
-            if instrname.lower() == 'movd':
-                ops[0] = reduce_to_32bit(ops[0])
-                ops[1] = reduce_to_32bit(ops[1])
             #
             op = '\t%s%s %s%s' % (instrname.lower(), suffix,
                                   ', '.join(ops), following)
diff --git a/pypy/jit/backend/x86/test/test_zmath.py b/pypy/jit/backend/x86/test/test_zmath.py
--- a/pypy/jit/backend/x86/test/test_zmath.py
+++ b/pypy/jit/backend/x86/test/test_zmath.py
@@ -6,6 +6,8 @@
 from pypy.translator.c.test.test_genc import compile
 from pypy.jit.backend.x86.support import ensure_sse2_floats
 from pypy.rlib import rfloat
+from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.debug import debug_print
 
 
 def get_test_case((fnname, args, expected)):
@@ -16,16 +18,32 @@
     expect_valueerror = (expected == ValueError)
     expect_overflowerror = (expected == OverflowError)
     check = test_direct.get_tester(expected)
+    unroll_args = unrolling_iterable(args)
     #
     def testfn():
+        debug_print('calling', fnname, 'with arguments:')
+        for arg in unroll_args:
+            debug_print('\t', arg)
         try:
             got = fn(*args)
         except ValueError:
-            return expect_valueerror
+            if expect_valueerror:
+                return True
+            else:
+                debug_print('unexpected ValueError!')
+                return False
         except OverflowError:
-            return expect_overflowerror
+            if expect_overflowerror:
+                return True
+            else:
+                debug_print('unexpected OverflowError!')
+                return False
         else:
-            return check(got)
+            if check(got):
+                return True
+            else:
+                debug_print('unexpected result:', got)
+                return False
     #
     testfn.func_name = 'test_' + fnname
     return testfn
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -34,7 +34,7 @@
 # I am porting it in a lazy fashion...  See py-utils/xam.py
 
 if sys.platform == "win32":
-    XXX   # lots more in Psyco
+    pass   # lots more in Psyco
 
 def machine_code_dump(data, originaddr, backend_name, label_list=None):
     objdump_backend_option = {
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -291,6 +291,12 @@
         op1 = SpaceOperation('-live-', [], None)
         return [op, op1]
 
+    def _noop_rewrite(self, op):
+        return op
+
+    rewrite_op_convert_float_bytes_to_longlong = _noop_rewrite
+    rewrite_op_convert_longlong_bytes_to_float = _noop_rewrite
+
     # ----------
     # Various kinds of calls
 
@@ -365,7 +371,7 @@
     def handle_builtin_call(self, op):
         oopspec_name, args = support.decode_builtin_call(op)
         # dispatch to various implementations depending on the oopspec_name
-        if oopspec_name.startswith('list.') or oopspec_name == 'newlist':
+        if oopspec_name.startswith('list.') or oopspec_name.startswith('newlist'):
             prepare = self._handle_list_call
         elif oopspec_name.startswith('stroruni.'):
             prepare = self._handle_stroruni_call
@@ -1494,6 +1500,14 @@
                                arraydescr, v_length],
                               op.result)
 
+    def do_resizable_newlist_hint(self, op, args, arraydescr, lengthdescr,
+                                  itemsdescr, structdescr):
+        v_hint = self._get_initial_newlist_length(op, args)
+        return SpaceOperation('newlist_hint',
+                              [structdescr, lengthdescr, itemsdescr,
+                               arraydescr, v_hint],
+                              op.result)
+
     def do_resizable_list_getitem(self, op, args, arraydescr, lengthdescr,
                                   itemsdescr, structdescr):
         v_index, extraop = self._prepare_list_getset(op, lengthdescr, args,
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -144,6 +144,10 @@
 _ll_1_newlist.need_result_type = True
 _ll_2_newlist.need_result_type = True
 
+def _ll_1_newlist_hint(LIST, hint):
+    return LIST.ll_newlist_hint(hint)
+_ll_1_newlist_hint.need_result_type = True
+
 def _ll_1_list_len(l):
     return l.ll_length()
 def _ll_2_list_getitem(l, index):
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -968,6 +968,23 @@
             int_return %i2
         """, transform=True)
 
+    def test_convert_float_bytes(self):
+        from pypy.rlib.longlong2float import float2longlong, longlong2float
+        def f(x):
+            ll = float2longlong(x)
+            return longlong2float(ll)
+        if longlong.is_64_bit:
+            tmp_var = "%i0"
+            result_var = "%f1"
+        else:
+            tmp_var = "%f1"
+            result_var = "%f2"
+        self.encoding_test(f, [25.0], """
+            convert_float_bytes_to_longlong %%f0 -> %(tmp_var)s
+            convert_longlong_bytes_to_float %(tmp_var)s -> %(result_var)s
+            float_return %(result_var)s
+        """ % {"result_var": result_var, "tmp_var": tmp_var}, transform=True)
+
 
 def check_force_cast(FROM, TO, operations, value):
     """Check that the test is correctly written..."""
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -1,6 +1,6 @@
 import py, sys
 
-from pypy.rlib.rarithmetic import r_longlong, intmask
+from pypy.rlib.rarithmetic import r_longlong, intmask, is_valid_int
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.objspace.flow.model import Block, Link
 from pypy.translator.unsimplify import varoftype
@@ -32,7 +32,7 @@
 def test_functions():
     xll = longlong.getfloatstorage(3.5)
     assert longlong.getrealfloat(xll) == 3.5
-    assert isinstance(longlong.gethash(xll), int)
+    assert is_valid_int(longlong.gethash(xll))
 
 
 class TestLongLong:
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -1,15 +1,16 @@
+from pypy.jit.codewriter import heaptracker, longlong
+from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
+from pypy.jit.metainterp.compile import ResumeAtPositionDescr
+from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
+from pypy.rlib import longlong2float
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert, make_sure_not_resized
+from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
+from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.rtimer import read_timestamp
-from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
-from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop, ll_assert
-from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
-from pypy.jit.codewriter import heaptracker, longlong
-from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
-from pypy.jit.metainterp.compile import ResumeAtPositionDescr
+
 
 def arguments(*argtypes, **kwds):
     resulttype = kwds.pop('returns', None)
@@ -20,6 +21,9 @@
         return function
     return decorate
 
+LONGLONG_TYPECODE = 'i' if longlong.is_64_bit else 'f'
+
+
 class LeaveFrame(JitException):
     pass
 
@@ -663,6 +667,16 @@
         a = float(a)
         return longlong.getfloatstorage(a)
 
+    @arguments("f", returns=LONGLONG_TYPECODE)
+    def bhimpl_convert_float_bytes_to_longlong(a):
+        a = longlong.getrealfloat(a)
+        return longlong2float.float2longlong(a)
+
+    @arguments(LONGLONG_TYPECODE, returns="f")
+    def bhimpl_convert_longlong_bytes_to_float(a):
+        a = longlong2float.longlong2float(a)
+        return longlong.getfloatstorage(a)
+
     # ----------
     # control flow operations
 
@@ -982,6 +996,15 @@
         cpu.bh_setfield_gc_r(result, itemsdescr, items)
         return result
 
+    @arguments("cpu", "d", "d", "d", "d", "i", returns="r")
+    def bhimpl_newlist_hint(cpu, structdescr, lengthdescr, itemsdescr,
+                            arraydescr, lengthhint):
+        result = cpu.bh_new(structdescr)
+        cpu.bh_setfield_gc_i(result, lengthdescr, 0)
+        items = cpu.bh_new_array(arraydescr, lengthhint)
+        cpu.bh_setfield_gc_r(result, itemsdescr, items)
+        return result
+
     @arguments("cpu", "r", "d", "d", "i", returns="i")
     def bhimpl_getlistitem_gc_i(cpu, lst, itemsdescr, arraydescr, index):
         items = cpu.bh_getfield_gc_r(lst, itemsdescr)
@@ -1176,14 +1199,14 @@
     def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
         return cpu.bh_getinteriorfield_gc_f(array, index, descr)
 
-    @arguments("cpu", "r", "i", "d", "i")
-    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+    @arguments("cpu", "r", "i", "i", "d")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, value, descr):
         cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
-    @arguments("cpu", "r", "i", "d", "r")
-    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+    @arguments("cpu", "r", "i", "r", "d")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, value, descr):
         cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
-    @arguments("cpu", "r", "i", "d", "f")
-    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+    @arguments("cpu", "r", "i", "f", "d")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, value, descr):
         cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
 
     @arguments("cpu", "r", "d", returns="i")
@@ -1300,7 +1323,7 @@
     def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
         cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
 
-    @arguments(returns=(longlong.is_64_bit and "i" or "f"))
+    @arguments(returns=LONGLONG_TYPECODE)
     def bhimpl_ll_read_timestamp():
         return read_timestamp()
 
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -2,7 +2,7 @@
 """
 
 from pypy.rpython.lltypesystem import lltype, rstr
-from pypy.rlib.rarithmetic import ovfcheck, r_longlong
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong, is_valid_int
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -248,7 +248,7 @@
 def do_read_timestamp(cpu, _):
     x = read_timestamp()
     if longlong.is_64_bit:
-        assert isinstance(x, int)         # 64-bit
+        assert is_valid_int(x)            # 64-bit
         return BoxInt(x)
     else:
         assert isinstance(x, r_longlong)  # 32-bit
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -169,9 +169,9 @@
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
                 if jd_sd._get_printable_location_ptr:
-                    s = jd_sd.warmstate.get_location_str(op.getarglist()[2:])
+                    s = jd_sd.warmstate.get_location_str(op.getarglist()[3:])
                     s = s.replace(',', '.') # we use comma for argument splitting
-                    op_repr = "debug_merge_point(%d, '%s')" % (op.getarg(1).getint(), s)
+                    op_repr = "debug_merge_point(%d, %d, '%s')" % (op.getarg(1).getint(), op.getarg(2).getint(), s)
             lines.append(op_repr)
             if is_interesting_guard(op):
                 tgt = op.getdescr()._debug_suboperations[0]
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -4,7 +4,8 @@
 from pypy.rpython.ootypesystem import ootype
 from pypy.rlib.objectmodel import we_are_translated, Symbolic
 from pypy.rlib.objectmodel import compute_unique_id
-from pypy.rlib.rarithmetic import r_int64
+from pypy.rlib.rarithmetic import r_int64, is_valid_int
+
 from pypy.conftest import option
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
@@ -213,7 +214,7 @@
 
     def __init__(self, value):
         if not we_are_translated():
-            if isinstance(value, int):
+            if is_valid_int(value):
                 value = int(value)    # bool -> int
             else:
                 assert isinstance(value, Symbolic)
@@ -448,7 +449,7 @@
 
     def __init__(self, value=0):
         if not we_are_translated():
-            if isinstance(value, int):
+            if is_valid_int(value):
                 value = int(value)    # bool -> int
             else:
                 assert isinstance(value, Symbolic)
diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py
--- a/pypy/jit/metainterp/logger.py
+++ b/pypy/jit/metainterp/logger.py
@@ -110,9 +110,9 @@
     def repr_of_resop(self, op, ops_offset=None):
         if op.getopnum() == rop.DEBUG_MERGE_POINT:
             jd_sd = self.metainterp_sd.jitdrivers_sd[op.getarg(0).getint()]
-            s = jd_sd.warmstate.get_location_str(op.getarglist()[2:])
+            s = jd_sd.warmstate.get_location_str(op.getarglist()[3:])
             s = s.replace(',', '.') # we use comma for argument splitting
-            return "debug_merge_point(%d, '%s')" % (op.getarg(1).getint(), s)
+            return "debug_merge_point(%d, %d, '%s')" % (op.getarg(1).getint(), op.getarg(2).getint(), s)
         if ops_offset is None:
             offset = -1
         else:
@@ -149,7 +149,7 @@
             if target_token.exported_state:
                 for op in target_token.exported_state.inputarg_setup_ops:
                     debug_print('    ' + self.repr_of_resop(op))
-        
+
     def _log_operations(self, inputargs, operations, ops_offset):
         if not have_debug_prints():
             return
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -9,7 +9,7 @@
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
 from pypy.jit.metainterp.optimizeopt.pure import OptPure
 from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
-from pypy.rlib.jit import PARAMETERS
+from pypy.rlib.jit import PARAMETERS, ENABLE_ALL_OPTS
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.debug import debug_start, debug_stop, debug_print
 
@@ -30,6 +30,9 @@
 ALL_OPTS_LIST = [name for name, _ in ALL_OPTS]
 ALL_OPTS_NAMES = ':'.join([name for name, _ in ALL_OPTS])
 
+assert ENABLE_ALL_OPTS == ALL_OPTS_NAMES, (
+    'please fix rlib/jit.py to say ENABLE_ALL_OPTS = %r' % (ALL_OPTS_NAMES,))
+
 def build_opt_chain(metainterp_sd, enable_opts):
     config = metainterp_sd.config
     optimizations = []
diff --git a/pypy/jit/metainterp/optimizeopt/intutils.py b/pypy/jit/metainterp/optimizeopt/intutils.py
--- a/pypy/jit/metainterp/optimizeopt/intutils.py
+++ b/pypy/jit/metainterp/optimizeopt/intutils.py
@@ -1,10 +1,9 @@
-from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT
+from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, maxint, is_valid_int
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.jit.metainterp.resoperation import rop, ResOperation
 from pypy.jit.metainterp.history import BoxInt, ConstInt
-import sys
-MAXINT = sys.maxint
-MININT = -sys.maxint - 1
+MAXINT = maxint
+MININT = -maxint - 1
 
 class IntBound(object):
     _attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
@@ -16,8 +15,8 @@
         self.lower = lower
         # check for unexpected overflows:
         if not we_are_translated():
-            assert type(upper) is not long
-            assert type(lower) is not long
+            assert type(upper) is not long or is_valid_int(upper)
+            assert type(lower) is not long or is_valid_int(lower)
 
     # Returns True if the bound was updated
     def make_le(self, other):
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5031,6 +5031,42 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_str_copy_virtual(self):
+        ops = """
+        [i0]
+        p0 = newstr(8)
+        strsetitem(p0, 0, i0)
+        strsetitem(p0, 1, i0)
+        strsetitem(p0, 2, i0)
+        strsetitem(p0, 3, i0)
+        strsetitem(p0, 4, i0)
+        strsetitem(p0, 5, i0)
+        strsetitem(p0, 6, i0)
+        strsetitem(p0, 7, i0)
+        p1 = newstr(12)
+        copystrcontent(p0, p1, 0, 0, 8)
+        strsetitem(p1, 8, 3)
+        strsetitem(p1, 9, 0)
+        strsetitem(p1, 10, 0)
+        strsetitem(p1, 11, 0)
+        finish(p1)
+        """
+        expected = """
+        [i0]
+        p1 = newstr(12)
+        strsetitem(p1, 0, i0)
+        strsetitem(p1, 1, i0)
+        strsetitem(p1, 2, i0)
+        strsetitem(p1, 3, i0)
+        strsetitem(p1, 4, i0)
+        strsetitem(p1, 5, i0)
+        strsetitem(p1, 6, i0)
+        strsetitem(p1, 7, i0)
+        strsetitem(p1, 8, 3)
+        finish(p1)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -9,7 +9,6 @@
 from pypy.jit.metainterp.inliner import Inliner
 from pypy.jit.metainterp.resoperation import rop, ResOperation
 from pypy.jit.metainterp.resume import Snapshot
-from pypy.rlib.debug import debug_print
 import sys, os
 
 # FIXME: Introduce some VirtualOptimizer super class instead
@@ -121,9 +120,9 @@
                 limit = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.retrace_limit
                 if cell_token.retraced_count < limit:
                     cell_token.retraced_count += 1
-                    debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
+                    #debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
                 else:
-                    debug_print("Retrace count reached, jumping to preamble")
+                    #debug_print("Retrace count reached, jumping to preamble")
                     assert cell_token.target_tokens[0].virtual_state is None
                     jumpop.setdescr(cell_token.target_tokens[0])
                     self.optimizer.send_extra_operation(jumpop)
@@ -273,9 +272,9 @@
                    not newvalue.is_constant():
                     op = ResOperation(rop.SAME_AS, [op.result], newresult)
                     self.optimizer._newoperations.append(op)
-                    if self.optimizer.loop.logops:
-                        debug_print('  Falling back to add extra: ' +
-                                    self.optimizer.loop.logops.repr_of_resop(op))
+                    #if self.optimizer.loop.logops:
+                    #    debug_print('  Falling back to add extra: ' +
+                    #                self.optimizer.loop.logops.repr_of_resop(op))
                     
         self.optimizer.flush()
         self.optimizer.emitting_dissabled = False
@@ -341,8 +340,8 @@
             if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
-                    if self.optimizer.loop.logops:
-                        debug_print('J:  ' + self.optimizer.loop.logops.repr_of_arg(a))
+                    #if self.optimizer.loop.logops:
+                    #    debug_print('J:  ' + self.optimizer.loop.logops.repr_of_arg(a))
                     self.import_box(a, inputargs, short_jumpargs, jumpargs)
                     j += 1
             else:
@@ -353,11 +352,11 @@
                 if op.is_guard():
                     args = args + op.getfailargs()
 
-                if self.optimizer.loop.logops:
-                    debug_print('OP: ' + self.optimizer.loop.logops.repr_of_resop(op))
+                #if self.optimizer.loop.logops:
+                #    debug_print('OP: ' + self.optimizer.loop.logops.repr_of_resop(op))
                 for a in args:
-                    if self.optimizer.loop.logops:
-                        debug_print('A:  ' + self.optimizer.loop.logops.repr_of_arg(a))
+                    #if self.optimizer.loop.logops:
+                    #    debug_print('A:  ' + self.optimizer.loop.logops.repr_of_arg(a))
                     self.import_box(a, inputargs, short_jumpargs, jumpargs)
                 i += 1
             newoperations = self.optimizer.get_newoperations()
@@ -370,18 +369,18 @@
         # that is compatible with the virtual state at the start of the loop
         modifier = VirtualStateAdder(self.optimizer)
         final_virtual_state = modifier.get_virtual_state(original_jumpargs)
-        debug_start('jit-log-virtualstate')
-        virtual_state.debug_print('Closed loop with ')
+        #debug_start('jit-log-virtualstate')
+        #virtual_state.debug_print('Closed loop with ')
         bad = {}
         if not virtual_state.generalization_of(final_virtual_state, bad):
             # We ended up with a virtual state that is not compatible
             # and we are thus unable to jump to the start of the loop
-            final_virtual_state.debug_print("Bad virtual state at end of loop, ",
-                                            bad)
-            debug_stop('jit-log-virtualstate')
+            #final_virtual_state.debug_print("Bad virtual state at end of loop, ",
+            #                                bad)
+            #debug_stop('jit-log-virtualstate')
             raise InvalidLoop
             
-        debug_stop('jit-log-virtualstate')
+        #debug_stop('jit-log-virtualstate')
 
         maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
         if self.optimizer.emitted_guards > maxguards:
@@ -444,9 +443,9 @@
                 self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer,
                                              seen)
 
-        if self.optimizer.loop.logops:
-            debug_print('  Emitting short op: ' +
-                        self.optimizer.loop.logops.repr_of_resop(op))
+        #if self.optimizer.loop.logops:
+        #    debug_print('  Emitting short op: ' +
+        #                self.optimizer.loop.logops.repr_of_resop(op))
 
         optimizer.send_extra_operation(op)
         seen[op.result] = True
@@ -527,8 +526,8 @@
         args = jumpop.getarglist()
         modifier = VirtualStateAdder(self.optimizer)
         virtual_state = modifier.get_virtual_state(args)
-        debug_start('jit-log-virtualstate')
-        virtual_state.debug_print("Looking for ")
+        #debug_start('jit-log-virtualstate')
+        #virtual_state.debug_print("Looking for ")
 
         for target in cell_token.target_tokens:
             if not target.virtual_state:
@@ -537,10 +536,10 @@
             extra_guards = []
 
             bad = {}
-            debugmsg = 'Did not match '
+            #debugmsg = 'Did not match '
             if target.virtual_state.generalization_of(virtual_state, bad):
                 ok = True
-                debugmsg = 'Matched '
+                #debugmsg = 'Matched '
             else:
                 try:
                     cpu = self.optimizer.cpu
@@ -549,13 +548,13 @@
                                                          extra_guards)
 
                     ok = True
-                    debugmsg = 'Guarded to match '
+                    #debugmsg = 'Guarded to match '
                 except InvalidLoop:
                     pass
-            target.virtual_state.debug_print(debugmsg, bad)
+            #target.virtual_state.debug_print(debugmsg, bad)
 
             if ok:
-                debug_stop('jit-log-virtualstate')
+                #debug_stop('jit-log-virtualstate')
 
                 values = [self.getvalue(arg)
                           for arg in jumpop.getarglist()]
@@ -576,13 +575,13 @@
                         newop = inliner.inline_op(shop)
                         self.optimizer.send_extra_operation(newop)
                 except InvalidLoop:
-                    debug_print("Inlining failed unexpectedly",
-                                "jumping to preamble instead")
+                    #debug_print("Inlining failed unexpectedly",
+                    #            "jumping to preamble instead")
                     assert cell_token.target_tokens[0].virtual_state is None
                     jumpop.setdescr(cell_token.target_tokens[0])
                     self.optimizer.send_extra_operation(jumpop)
                 return True
-        debug_stop('jit-log-virtualstate')
+        #debug_stop('jit-log-virtualstate')
         return False
 
 class ValueImporter(object):
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -681,13 +681,14 @@
             self.synthetic[op] = True
 
     def debug_print(self, logops):
-        debug_start('jit-short-boxes')
-        for box, op in self.short_boxes.items():
-            if op:
-                debug_print(logops.repr_of_arg(box) + ': ' + logops.repr_of_resop(op))
-            else:
-                debug_print(logops.repr_of_arg(box) + ': None')
-        debug_stop('jit-short-boxes')
+        if 0:
+            debug_start('jit-short-boxes')
+            for box, op in self.short_boxes.items():
+                if op:
+                    debug_print(logops.repr_of_arg(box) + ': ' + logops.repr_of_resop(op))
+                else:
+                    debug_print(logops.repr_of_arg(box) + ': None')
+            debug_stop('jit-short-boxes')
 
     def operations(self):
         if not we_are_translated(): # For tests
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -10,6 +10,8 @@
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython import annlowlevel
 from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import is_valid_int
+
 
 
 class StrOrUnicode(object):
@@ -505,14 +507,23 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self,
-            src.force_box(self),
-            dst.force_box(self),
-            srcstart.force_box(self),
-            dststart.force_box(self),
-            length.force_box(self),
-            mode, need_next_offset=False
-        )
+        elif (src.is_virtual() and dst.is_virtual() and srcstart.is_constant() and
+            dststart.is_constant() and length.is_constant()):
+
+            src_start = srcstart.force_box(self).getint()
+            dst_start = dststart.force_box(self).getint()
+            for index in range(length.force_box(self).getint()):
+                vresult = self.strgetitem(src, optimizer.ConstantValue(ConstInt(index + src_start)), mode)
+                dst.setitem(index + dst_start, vresult)
+        else:
+            copy_str_content(self,
+                src.force_box(self),
+                dst.force_box(self),
+                srcstart.force_box(self),
+                dststart.force_box(self),
+                length.force_box(self),
+                mode, need_next_offset=False
+            )
 
     def optimize_CALL(self, op):
         # dispatch based on 'oopspecindex' to a method that handles
@@ -721,7 +732,7 @@
     for name in dir(OptString):
         if name.startswith(prefix):
             value = getattr(EffectInfo, 'OS_' + name[len(prefix):])
-            assert isinstance(value, int) and value != 0
+            assert is_valid_int(value) and value != 0
             result.append((value, getattr(OptString, name)))
     return unrolling_iterable(result)
 opt_call_oopspec_ops = _findall_call_oopspec()
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -223,6 +223,8 @@
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
                     'cast_ptr_to_int', 'cast_int_to_ptr',
+                    'convert_float_bytes_to_longlong',
+                    'convert_longlong_bytes_to_float',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -509,6 +511,15 @@
         self._opimpl_setfield_gc_any(sbox, itemsdescr, abox)
         return sbox
 
+    @arguments("descr", "descr", "descr", "descr", "box")
+    def opimpl_newlist_hint(self, structdescr, lengthdescr, itemsdescr,
+                            arraydescr, sizehintbox):
+        sbox = self.opimpl_new(structdescr)
+        self._opimpl_setfield_gc_any(sbox, lengthdescr, history.CONST_FALSE)
+        abox = self.opimpl_new_array(arraydescr, sizehintbox)
+        self._opimpl_setfield_gc_any(sbox, itemsdescr, abox)
+        return sbox
+
     @arguments("box", "descr", "descr", "box")
     def _opimpl_getlistitem_gc_any(self, listbox, itemsdescr, arraydescr,
                                    indexbox):
@@ -974,9 +985,11 @@
         any_operation = len(self.metainterp.history.operations) > 0
         jitdriver_sd = self.metainterp.staticdata.jitdrivers_sd[jdindex]
         self.verify_green_args(jitdriver_sd, greenboxes)
-        self.debug_merge_point(jitdriver_sd, jdindex, self.metainterp.portal_call_depth,
+        self.debug_merge_point(jitdriver_sd, jdindex,
+                               self.metainterp.portal_call_depth,
+                               self.metainterp.call_ids[-1],
                                greenboxes)
-        
+
         if self.metainterp.seen_loop_header_for_jdindex < 0:
             if not any_operation:
                 return
@@ -1028,11 +1041,11 @@
                                     assembler_call=True)
             raise ChangeFrame
 
-    def debug_merge_point(self, jitdriver_sd, jd_index, portal_call_depth, greenkey):
+    def debug_merge_point(self, jitdriver_sd, jd_index, portal_call_depth, current_call_id, greenkey):
         # debugging: produce a DEBUG_MERGE_POINT operation
         loc = jitdriver_sd.warmstate.get_location_str(greenkey)
         debug_print(loc)
-        args = [ConstInt(jd_index), ConstInt(portal_call_depth)] + greenkey
+        args = [ConstInt(jd_index), ConstInt(portal_call_depth), ConstInt(current_call_id)] + greenkey
         self.metainterp.history.record(rop.DEBUG_MERGE_POINT, args, None)
 
     @arguments("box", "label")
@@ -1574,11 +1587,14 @@
         self.call_pure_results = args_dict_box()
         self.heapcache = HeapCache()
 
+        self.call_ids = []
+        self.current_call_id = 0
+
     def retrace_needed(self, trace):
         self.partial_trace = trace
         self.retracing_from = len(self.history.operations) - 1
         self.heapcache.reset()
-        
+
 
     def perform_call(self, jitcode, boxes, greenkey=None):
         # causes the metainterp to enter the given subfunction
@@ -1592,6 +1608,8 @@
     def newframe(self, jitcode, greenkey=None):
         if jitcode.is_portal:
             self.portal_call_depth += 1
+            self.call_ids.append(self.current_call_id)
+            self.current_call_id += 1
         if greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
                     (greenkey, len(self.history.operations)))
@@ -1608,6 +1626,7 @@
         jitcode = frame.jitcode
         if jitcode.is_portal:
             self.portal_call_depth -= 1
+            self.call_ids.pop()
         if frame.greenkey is not None and self.is_main_jitcode(jitcode):
             self.portal_trace_positions.append(
                     (None, len(self.history.operations)))
@@ -1976,7 +1995,7 @@
                 # Found!  Compile it as a loop.
                 # raises in case it works -- which is the common case
                 if self.partial_trace:
-                    if  start != self.retracing_from: 
+                    if  start != self.retracing_from:
                         raise SwitchToBlackhole(ABORT_BAD_LOOP) # For now
                 self.compile_loop(original_boxes, live_arg_boxes, start, resumedescr)
                 # creation of the loop was cancelled!
@@ -2085,7 +2104,7 @@
             if not token.target_tokens:
                 return None
         return token
-        
+
     def compile_loop(self, original_boxes, live_arg_boxes, start, resume_at_jump_descr):
         num_green_args = self.jitdriver_sd.num_green_args
         greenkey = original_boxes[:num_green_args]
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -419,6 +419,8 @@
     'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
+    'CONVERT_FLOAT_BYTES_TO_LONGLONG/1',
+    'CONVERT_LONGLONG_BYTES_TO_FLOAT/1',
     #
     'INT_LT/2b',
     'INT_LE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,8 +1,10 @@
+import math
 import sys
 
 import py
 
 from pypy import conftest
+from pypy.jit.codewriter import longlong
 from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
 from pypy.jit.metainterp import pyjitpl, history
 from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
@@ -14,7 +16,8 @@
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
     isconstant, isvirtual, promote_string, set_param, record_known_class)
-from pypy.rlib.rarithmetic import ovfcheck
+from pypy.rlib.longlong2float import float2longlong, longlong2float
+from pypy.rlib.rarithmetic import ovfcheck, is_valid_int
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
 
@@ -292,7 +295,7 @@
         assert res == f(6, sys.maxint, 32, 48)
         res = self.meta_interp(f, [sys.maxint, 6, 32, 48])
         assert res == f(sys.maxint, 6, 32, 48)
-        
+
 
     def test_loop_invariant_intbox(self):
         myjitdriver = JitDriver(greens = [], reds = ['y', 'res', 'x'])
@@ -953,7 +956,7 @@
         self.meta_interp(f, [20], repeat=7)
         # the loop and the entry path as a single trace
         self.check_jitcell_token_count(1)
-        
+
         # we get:
         #    ENTER             - compile the new loop and the entry bridge
         #    ENTER             - compile the leaving path
@@ -1470,7 +1473,7 @@
         assert res == f(299)
         self.check_resops(guard_class=0, guard_nonnull=4,
                           guard_nonnull_class=4, guard_isnull=2)
-        
+
 
     def test_merge_guardnonnull_guardvalue(self):
         from pypy.rlib.objectmodel import instantiate
@@ -1499,7 +1502,7 @@
         assert res == f(299)
         self.check_resops(guard_value=4, guard_class=0, guard_nonnull=4,
                           guard_nonnull_class=0, guard_isnull=2)
-        
+
 
     def test_merge_guardnonnull_guardvalue_2(self):
         from pypy.rlib.objectmodel import instantiate
@@ -1528,7 +1531,7 @@
         assert res == f(299)
         self.check_resops(guard_value=4, guard_class=0, guard_nonnull=4,
                           guard_nonnull_class=0, guard_isnull=2)
-        
+
 
     def test_merge_guardnonnull_guardclass_guardvalue(self):
         from pypy.rlib.objectmodel import instantiate
@@ -2296,7 +2299,7 @@
             self.check_resops(int_rshift=3)
 
             bigval = 1
-            while (bigval << 3).__class__ is int:
+            while is_valid_int(bigval << 3):
                 bigval = bigval << 1
 
             assert self.meta_interp(f, [bigval, 5]) == 0
@@ -2341,7 +2344,7 @@
             self.check_resops(int_rshift=3)
 
             bigval = 1
-            while (bigval << 3).__class__ is int:
+            while is_valid_int(bigval << 3):
                 bigval = bigval << 1
 
             assert self.meta_interp(f, [bigval, 5]) == 0
@@ -2636,7 +2639,7 @@
             return sa
         assert self.meta_interp(f, [20]) == f(20)
         self.check_resops(int_lt=6, int_le=2, int_ge=4, int_gt=3)
-        
+
 
     def test_intbounds_not_generalized2(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'sa', 'node'])
@@ -2677,7 +2680,7 @@
         assert self.meta_interp(f, [20, 3]) == f(20, 3)
         self.check_jitcell_token_count(1)
         self.check_target_token_count(5)
-        
+
     def test_max_retrace_guards(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'sa', 'a'])
 
@@ -2815,7 +2818,7 @@
         for cell in get_stats().get_all_jitcell_tokens():
             # Initialal trace with two labels and 5 retraces
             assert len(cell.target_tokens) <= 7
-            
+
     def test_nested_retrace(self):
 
         myjitdriver = JitDriver(greens = ['pc'], reds = ['n', 'a', 'i', 'j', 'sa'])
@@ -3784,6 +3787,25 @@
         assert res == 11 * 12 * 13
         self.check_operations_history(int_add=3, int_mul=2)
 
+    def test_setinteriorfield(self):
+        A = lltype.GcArray(lltype.Struct('S', ('x', lltype.Signed)))
+        a = lltype.malloc(A, 5, immortal=True)
+        def g(n):
+            a[n].x = n + 2
+            return a[n].x
+        res = self.interp_operations(g, [1])
+        assert res == 3
+
+    def test_float_bytes(self):
+        def f(n):
+            ll = float2longlong(n)
+            return longlong2float(ll)
+
+        for x in [2.5, float("nan"), -2.5, float("inf")]:
+            # There are tests elsewhere to verify the correctness of this.
+            res = self.interp_operations(f, [x])
+            assert res == x or math.isnan(x) and math.isnan(res)
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py
--- a/pypy/jit/metainterp/test/test_compile.py
+++ b/pypy/jit/metainterp/test/test_compile.py
@@ -14,7 +14,7 @@
     ts = typesystem.llhelper
     def __init__(self):
         self.seen = []
-    def compile_loop(self, inputargs, operations, token, name=''):
+    def compile_loop(self, inputargs, operations, token, log=True, name=''):
         self.seen.append((inputargs, operations, token))
 
 class FakeLogger(object):
diff --git a/pypy/jit/metainterp/test/test_list.py b/pypy/jit/metainterp/test/test_list.py
--- a/pypy/jit/metainterp/test/test_list.py
+++ b/pypy/jit/metainterp/test/test_list.py
@@ -1,4 +1,5 @@
 import py
+from pypy.rlib.objectmodel import newlist_hint
 from pypy.rlib.jit import JitDriver
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 
@@ -228,6 +229,28 @@
         self.check_resops({'jump': 1, 'int_gt': 2, 'int_add': 2,
                            'guard_true': 2, 'int_sub': 2})
 
+    def test_newlist_hint(self):
+        def f(i):
+            l = newlist_hint(i)
+            l[0] = 55
+            return len(l)
+
+        r = self.interp_operations(f, [3])
+        assert r == 0
+
+    def test_newlist_hint_optimized(self):
+        driver = JitDriver(greens = [], reds = ['i'])
+
+        def f(i):
+            while i > 0:
+                driver.jit_merge_point(i=i)
+                l = newlist_hint(5)
+                l.append(1)
+                i -= l[0]
+
+        self.meta_interp(f, [10], listops=True)
+        self.check_resops(new_array=0, call=0)
+
 class TestOOtype(ListTests, OOJitMixin):
     pass
 
diff --git a/pypy/jit/metainterp/test/test_logger.py b/pypy/jit/metainterp/test/test_logger.py
--- a/pypy/jit/metainterp/test/test_logger.py
+++ b/pypy/jit/metainterp/test/test_logger.py
@@ -54,7 +54,7 @@
         class FakeJitDriver(object):
             class warmstate(object):
                 get_location_str = staticmethod(lambda args: "dupa")
-        
+
         class FakeMetaInterpSd:
             cpu = AbstractCPU()
             cpu.ts = self.ts
@@ -77,7 +77,7 @@
             equaloplists(loop.operations, oloop.operations)
             assert oloop.inputargs == loop.inputargs
         return logger, loop, oloop
-    
+
     def test_simple(self):
         inp = '''
         [i0, i1, i2, p3, p4, p5]
@@ -116,12 +116,13 @@
     def test_debug_merge_point(self):
         inp = '''
         []
-        debug_merge_point(0, 0)
+        debug_merge_point(0, 0, 0)
         '''
         _, loop, oloop = self.reparse(inp, check_equal=False)
         assert loop.operations[0].getarg(1).getint() == 0
-        assert oloop.operations[0].getarg(1)._get_str() == "dupa"
-        
+        assert loop.operations[0].getarg(2).getint() == 0
+        assert oloop.operations[0].getarg(2)._get_str() == "dupa"
+
     def test_floats(self):
         inp = '''
         [f0]
@@ -142,7 +143,7 @@
         output = logger.log_loop(loop)
         assert output.splitlines()[-1] == "jump(i0, descr=<Loop3>)"
         pure_parse(output)
-        
+
     def test_guard_descr(self):
         namespace = {'fdescr': BasicFailDescr()}
         inp = '''
@@ -154,7 +155,7 @@
         output = logger.log_loop(loop)
         assert output.splitlines()[-1] == "guard_true(i0, descr=<Guard0>) [i0]"
         pure_parse(output)
-        
+
         logger = Logger(self.make_metainterp_sd(), guard_number=False)
         output = logger.log_loop(loop)
         lastline = output.splitlines()[-1]
diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py
--- a/pypy/jit/metainterp/test/test_warmspot.py
+++ b/pypy/jit/metainterp/test/test_warmspot.py
@@ -13,7 +13,7 @@
 
 
 class WarmspotTests(object):
-    
+
     def test_basic(self):
         mydriver = JitDriver(reds=['a'],
                              greens=['i'])
@@ -77,16 +77,16 @@
         self.meta_interp(f, [123, 10])
         assert len(get_stats().locations) >= 4
         for loc in get_stats().locations:
-            assert loc == (0, 123)
+            assert loc == (0, 0, 123)
 
     def test_set_param_enable_opts(self):
         from pypy.rpython.annlowlevel import llstr, hlstr
-        
+
         myjitdriver = JitDriver(greens = [], reds = ['n'])
         class A(object):
             def m(self, n):
                 return n-1
-            
+
         def g(n):
             while n > 0:
                 myjitdriver.can_enter_jit(n=n)
@@ -332,7 +332,7 @@
             ts = llhelper
             translate_support_code = False
             stats = "stats"
-            
+
             def get_fail_descr_number(self, d):
                 return -1
 
@@ -352,7 +352,7 @@
                 return "not callable"
 
         driver = JitDriver(reds = ['red'], greens = ['green'])
-        
+
         def f(green):
             red = 0
             while red < 10:
diff --git a/pypy/jit/tl/tlc.py b/pypy/jit/tl/tlc.py
--- a/pypy/jit/tl/tlc.py
+++ b/pypy/jit/tl/tlc.py
@@ -6,6 +6,8 @@
 from pypy.jit.tl.tlopcode import *
 from pypy.jit.tl import tlopcode
 from pypy.rlib.jit import JitDriver, elidable
+from pypy.rlib.rarithmetic import is_valid_int
+
 
 class Obj(object):
 
@@ -219,7 +221,7 @@
 class Frame(object):
 
     def __init__(self, args, pc):
-        assert isinstance(pc, int)
+        assert is_valid_int(pc)
         self.args  = args
         self.pc    = pc
         self.stack = []
@@ -239,7 +241,7 @@
         return interp_eval(code, pc, args, pool).int_o()
 
     def interp_eval(code, pc, args, pool):
-        assert isinstance(pc, int)
+        assert is_valid_int(pc)
         frame = Frame(args, pc)
         pc = frame.pc
 
diff --git a/pypy/jit/tool/test/test_oparser.py b/pypy/jit/tool/test/test_oparser.py
--- a/pypy/jit/tool/test/test_oparser.py
+++ b/pypy/jit/tool/test/test_oparser.py
@@ -146,16 +146,18 @@
     def test_debug_merge_point(self):
         x = '''
         []
-        debug_merge_point(0, "info")
-        debug_merge_point(0, 'info')
-        debug_merge_point(1, '<some ('other.')> info')
-        debug_merge_point(0, '(stuff) #1')
+        debug_merge_point(0, 0, "info")
+        debug_merge_point(0, 0, 'info')
+        debug_merge_point(1, 1, '<some ('other.')> info')
+        debug_merge_point(0, 0, '(stuff) #1')
         '''
         loop = self.parse(x)
-        assert loop.operations[0].getarg(1)._get_str() == 'info'
-        assert loop.operations[1].getarg(1)._get_str() == 'info'
-        assert loop.operations[2].getarg(1)._get_str() == "<some ('other.')> info"
-        assert loop.operations[3].getarg(1)._get_str() == "(stuff) #1"
+        assert loop.operations[0].getarg(2)._get_str() == 'info'
+        assert loop.operations[0].getarg(1).value == 0
+        assert loop.operations[1].getarg(2)._get_str() == 'info'
+        assert loop.operations[2].getarg(2)._get_str() == "<some ('other.')> info"
+        assert loop.operations[2].getarg(1).value == 1
+        assert loop.operations[3].getarg(2)._get_str() == "(stuff) #1"
 
 
     def test_descr_with_obj_print(self):
diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py
--- a/pypy/module/__builtin__/app_inspect.py
+++ b/pypy/module/__builtin__/app_inspect.py
@@ -8,8 +8,6 @@
 from __pypy__ import lookup_special
 
 def _caller_locals(): 
-    # note: the reason why this is working is because the functions in here are
-    # compiled by geninterp, so they don't have a frame
     return sys._getframe(0).f_locals 
 
 def vars(*obj):
@@ -26,17 +24,6 @@
         except AttributeError:
             raise TypeError, "vars() argument must have __dict__ attribute"
 
-# Replaced by the interp-level helper space.callable(): 
-##def callable(ob):
-##    import __builtin__ # XXX this is insane but required for now for geninterp
-##    for c in type(ob).__mro__:
-##        if '__call__' in c.__dict__:
-##            if isinstance(ob, __builtin__._instance): # old style instance!
-##                return getattr(ob, '__call__', None) is not None
-##            return True
-##    else:
-##        return False
-
 def dir(*args):
     """dir([object]) -> list of strings
 
diff --git a/pypy/module/__builtin__/interp_memoryview.py b/pypy/module/__builtin__/interp_memoryview.py
--- a/pypy/module/__builtin__/interp_memoryview.py
+++ b/pypy/module/__builtin__/interp_memoryview.py
@@ -69,6 +69,10 @@
         return W_MemoryView(buf)
 
     def descr_buffer(self, space):
+        """Note that memoryview() objects in PyPy support buffer(), whereas
+        not in CPython; but CPython supports passing memoryview() to most
+        built-in functions that accept buffers, with the notable exception
+        of the buffer() built-in."""
         return space.wrap(self.buf)
 
     def descr_tobytes(self, space):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -1,5 +1,5 @@
+import sys
 
-# Package initialisation
 from pypy.interpreter.mixedmodule import MixedModule
 from pypy.module.imp.importing import get_pyc_magic
 
@@ -12,6 +12,21 @@
         "UnicodeBuilder": "interp_builders.W_UnicodeBuilder",
     }
 
+class TimeModule(MixedModule):
+    appleveldefs = {}
+    interpleveldefs = {}
+    if sys.platform.startswith("linux"):
+        from pypy.module.__pypy__ import interp_time
+        interpleveldefs["clock_gettime"] = "interp_time.clock_gettime"
+        interpleveldefs["clock_getres"] = "interp_time.clock_getres"
+        for name in [
+            "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW",
+            "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID"
+        ]:
+            if getattr(interp_time, name) is not None:
+                interpleveldefs[name] = "space.wrap(interp_time.%s)" % name
+
+
 class Module(MixedModule):
     appleveldefs = {
     }
@@ -32,6 +47,7 @@
 
     submodules = {
         "builders": BuildersModule,
+        "time": TimeModule,
     }
 
     def setup_after_space_initialization(self):
diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/interp_time.py
@@ -0,0 +1,65 @@
+from __future__ import with_statement
+import sys
+
+from pypy.interpreter.error import exception_from_errno
+from pypy.interpreter.gateway import unwrap_spec
+from pypy.rpython.lltypesystem import rffi, lltype
+from pypy.rpython.tool import rffi_platform
+from pypy.translator.tool.cbuild import ExternalCompilationInfo
+
+
+class CConfig:
+    _compilation_info_ = ExternalCompilationInfo(
+        includes=["time.h"],
+        libraries=["rt"],
+    )
+
+    HAS_CLOCK_GETTIME = rffi_platform.Has('clock_gettime')
+
+    CLOCK_REALTIME = rffi_platform.DefinedConstantInteger("CLOCK_REALTIME")
+    CLOCK_MONOTONIC = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC")
+    CLOCK_MONOTONIC_RAW = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC_RAW")
+    CLOCK_PROCESS_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_PROCESS_CPUTIME_ID")
+    CLOCK_THREAD_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_THREAD_CPUTIME_ID")
+
+    TIMESPEC = rffi_platform.Struct("struct timespec", [
+        ("tv_sec", rffi.TIME_T),
+        ("tv_nsec", rffi.LONG),
+    ])
+
+cconfig = rffi_platform.configure(CConfig)
+
+HAS_CLOCK_GETTIME = cconfig["HAS_CLOCK_GETTIME"]
+
+CLOCK_REALTIME = cconfig["CLOCK_REALTIME"]
+CLOCK_MONOTONIC = cconfig["CLOCK_MONOTONIC"]
+CLOCK_MONOTONIC_RAW = cconfig["CLOCK_MONOTONIC_RAW"]
+CLOCK_PROCESS_CPUTIME_ID = cconfig["CLOCK_PROCESS_CPUTIME_ID"]
+CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"]
+
+TIMESPEC = cconfig["TIMESPEC"]
+
+c_clock_gettime = rffi.llexternal("clock_gettime",
+    [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
+    compilation_info=CConfig._compilation_info_, threadsafe=False
+)
+c_clock_getres = rffi.llexternal("clock_getres",
+    [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
+    compilation_info=CConfig._compilation_info_, threadsafe=False
+)
+
+ at unwrap_spec(clk_id="c_int")
+def clock_gettime(space, clk_id):
+    with lltype.scoped_alloc(TIMESPEC) as tp:
+        ret = c_clock_gettime(clk_id, tp)
+        if ret != 0:
+            raise exception_from_errno(space, space.w_IOError)
+        return space.wrap(tp.c_tv_sec + tp.c_tv_nsec * 1e-9)
+
+ at unwrap_spec(clk_id="c_int")
+def clock_getres(space, clk_id):
+    with lltype.scoped_alloc(TIMESPEC) as tp:
+        ret = c_clock_getres(clk_id, tp)
+        if ret != 0:
+            raise exception_from_errno(space, space.w_IOError)
+        return space.wrap(tp.c_tv_sec + tp.c_tv_nsec * 1e-9)
diff --git a/pypy/module/__pypy__/test/test_time.py b/pypy/module/__pypy__/test/test_time.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_time.py
@@ -0,0 +1,26 @@
+import py
+
+from pypy.module.__pypy__.interp_time import HAS_CLOCK_GETTIME
+
+
+class AppTestTime(object):
+    def setup_class(cls):
+        if not HAS_CLOCK_GETTIME:
+            py.test.skip("need time.clock_gettime")
+
+    def test_clock_realtime(self):
+        from __pypy__ import time
+        res = time.clock_gettime(time.CLOCK_REALTIME)
+        assert isinstance(res, float)
+
+    def test_clock_monotonic(self):
+        from __pypy__ import time
+        a = time.clock_gettime(time.CLOCK_MONOTONIC)
+        b = time.clock_gettime(time.CLOCK_MONOTONIC)
+        assert a <= b
+
+    def test_clock_getres(self):
+        from __pypy__ import time
+        res = time.clock_getres(time.CLOCK_REALTIME)
+        assert res > 0.0
+        assert res <= 1.0
diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -1,9 +1,10 @@
 import py
-
+from pypy.conftest import gettestobjspace
 
 class AppTestAST:
 
     def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['struct'])
         cls.w_ast = cls.space.appexec([], """():
     import _ast
     return _ast""")
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -4,7 +4,7 @@
 
 class AppTestCodecs:
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('unicodedata',))
+        space = gettestobjspace(usemodules=('unicodedata', 'struct'))
         cls.space = space
 
     def test_register_noncallable(self):
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
--- a/pypy/module/_continuation/test/test_zpickle.py
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -106,8 +106,9 @@
     version = 0
 
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=('_continuation',),
+        cls.space = gettestobjspace(usemodules=('_continuation', 'struct'),
                                     CALL_METHOD=True)
+        cls.space.config.translation.continuation = True
         cls.space.appexec([], """():
             global continulet, A, __name__
 
diff --git a/pypy/module/_ffi/test/test__ffi.py b/pypy/module/_ffi/test/test__ffi.py
--- a/pypy/module/_ffi/test/test__ffi.py
+++ b/pypy/module/_ffi/test/test__ffi.py
@@ -100,7 +100,10 @@
         from _ffi import CDLL, types
         libm = CDLL(self.libm_name)
         pow_addr = libm.getaddressindll('pow')
-        assert pow_addr == self.pow_addr & (sys.maxint*2-1)
+        fff = sys.maxint*2-1
+        if sys.platform == 'win32':
+            fff = sys.maxint*2+1
+        assert pow_addr == self.pow_addr & fff
 
     def test_func_fromaddr(self):
         import sys
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -3,7 +3,7 @@
 
 class AppTestHashlib:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_hashlib'])
+        cls.space = gettestobjspace(usemodules=['_hashlib', 'array', 'struct'])
 
     def test_simple(self):
         import _hashlib
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -158,7 +158,7 @@
 
 class AppTestOpen:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_io', '_locale'])
+        cls.space = gettestobjspace(usemodules=['_io', '_locale', 'array', 'struct'])
         tmpfile = udir.join('tmpfile').ensure()
         cls.w_tmpfile = cls.space.wrap(str(tmpfile))
 
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -22,7 +22,7 @@
 eci = ExternalCompilationInfo(
     separate_module_files=[srcdir.join('profiling.c')],
     export_symbols=['pypy_setup_profiling', 'pypy_teardown_profiling'])
-                                                     
+
 c_setup_profiling = rffi.llexternal('pypy_setup_profiling',
                                   [], lltype.Void,
                                   compilation_info = eci)
@@ -228,7 +228,7 @@
         if w_self.builtins:
             key = create_spec(space, w_arg)
             w_self._enter_builtin_call(key)
-    elif event == 'c_return':
+    elif event == 'c_return' or event == 'c_exception':
         if w_self.builtins:
             key = create_spec(space, w_arg)
             w_self._enter_builtin_return(key)
@@ -237,7 +237,7 @@
         pass
 
 class W_Profiler(Wrappable):
-    
+
     def __init__(self, space, w_callable, time_unit, subcalls, builtins):
         self.subcalls = subcalls
         self.builtins = builtins
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -117,6 +117,20 @@
             assert 0.9 < subentry.totaltime < 2.9
             #assert 0.9 < subentry.inlinetime < 2.9
 
+    def test_builtin_exception(self):
+        import math
+        import _lsprof
+
+        prof = _lsprof.Profiler()
+        prof.enable()
+        try:
+            math.sqrt("a")
+        except TypeError:
+            pass
+        prof.disable()
+        stats = prof.getstats()
+        assert len(stats) == 2
+
     def test_use_cprofile(self):
         import sys, os
         # XXX this is evil trickery to walk around the fact that we don't
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -28,7 +28,7 @@
         assert self.md5.digest_size == 16
         #assert self.md5.digestsize == 16        -- not on CPython
         assert self.md5.md5().digest_size == 16
-        if sys.version >= (2, 5):
+        if sys.version_info >= (2, 5):
             assert self.md5.blocksize == 1
             assert self.md5.md5().digestsize == 16
 
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -92,7 +92,8 @@
 
 class AppTestSocketConnection(BaseConnectionTest):
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('_multiprocessing', 'thread', 'signal'))
+        space = gettestobjspace(usemodules=('_multiprocessing', 'thread', 'signal',
+                                            'struct', 'array'))
         cls.space = space
         cls.w_connections = space.newlist([])
 
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -2,6 +2,7 @@
 from pypy.module._multiprocessing.interp_semaphore import (
     RECURSIVE_MUTEX, SEMAPHORE)
 
+
 class AppTestSemaphore:
     def setup_class(cls):
         space = gettestobjspace(usemodules=('_multiprocessing', 'thread'))
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -6,7 +6,7 @@
 from pypy.rpython.lltypesystem import lltype, rffi
 
 def setup_module(mod):
-    mod.space = gettestobjspace(usemodules=['_socket', 'array'])
+    mod.space = gettestobjspace(usemodules=['_socket', 'array', 'struct'])
     global socket
     import socket
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
@@ -372,10 +372,9 @@
     def test_socket_connect(self):
         import _socket, os
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
-        # XXX temporarily we use python.org to test, will have more robust tests
-        # in the absence of a network connection later when more parts of the
-        # socket API are implemented.  Currently skip the test if there is no
-        # connection.
+        # it would be nice to have a test which works even if there is no
+        # network connection. However, this one is "good enough" for now. Skip
+        # it if there is no connection.
         try:
             s.connect(("www.python.org", 80))
         except _socket.gaierror, ex:
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -432,7 +432,8 @@
                     raise _ssl_seterror(self.space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return self.space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return self.space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                              length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -2,6 +2,7 @@
 import os
 import py
 
+
 class AppTestSSL:
     def setup_class(cls):
         space = gettestobjspace(usemodules=('_ssl', '_socket'))
@@ -29,7 +30,6 @@
         assert isinstance(_ssl.SSL_ERROR_EOF, int)
         assert isinstance(_ssl.SSL_ERROR_INVALID_ERROR_CODE, int)
 
-        assert isinstance(_ssl.OPENSSL_VERSION_NUMBER, (int, long))
         assert isinstance(_ssl.OPENSSL_VERSION_INFO, tuple)
         assert len(_ssl.OPENSSL_VERSION_INFO) == 5
         assert isinstance(_ssl.OPENSSL_VERSION, str)
@@ -90,7 +90,7 @@
 
 class AppTestConnectedSSL:
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('_ssl', '_socket'))
+        space = gettestobjspace(usemodules=('_ssl', '_socket', 'struct'))
         cls.space = space
 
     def setup_method(self, method):
@@ -179,7 +179,7 @@
     # to exercise the poll() calls
 
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('_ssl', '_socket'))
+        space = gettestobjspace(usemodules=('_ssl', '_socket', 'struct'))
         cls.space = space
         cls.space.appexec([], """():
             import socket; socket.setdefaulttimeout(1)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -11,6 +11,7 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.unroll import unrolling_iterable
+from pypy.rlib.objectmodel import specialize, keepalive_until_here
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
@@ -144,28 +145,36 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(RWBuffer):
-    def __init__(self, data, bytes):
-        self.data = data
-        self.len = bytes
+    def __init__(self, array):
+        self.array = array
 
     def getlength(self):
-        return self.len
+        return self.array.len * self.array.itemsize
 
     def getitem(self, index):
-        return self.data[index]
+        array = self.array
+        data = array._charbuf_start()
+        char = data[index]
+        array._charbuf_stop()
+        return char
 
     def setitem(self, index, char):
-        self.data[index] = char
+        array = self.array
+        data = array._charbuf_start()
+        data[index] = char
+        array._charbuf_stop()
 
 
 def make_array(mytype):
+    W_ArrayBase = globals()['W_ArrayBase']
+
     class W_Array(W_ArrayBase):
         itemsize = mytype.bytes
         typecode = mytype.typecode
 
         @staticmethod
         def register(typeorder):
-            typeorder[W_Array] = []
+            typeorder[W_Array] = [(W_ArrayBase, None)]
 
         def __init__(self, space):
             self.space = space
@@ -275,9 +284,10 @@
             oldlen = self.len
             new = len(s) / mytype.bytes
             self.setlen(oldlen + new)
-            cbuf = self.charbuf()
+            cbuf = self._charbuf_start()
             for i in range(len(s)):
                 cbuf[oldlen * mytype.bytes + i] = s[i]
+            self._charbuf_stop()
 
         def fromlist(self, w_lst):
             s = self.len
@@ -307,8 +317,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def charbuf(self):
-            return  rffi.cast(rffi.CCHARP, self.buffer)
+        def _charbuf_start(self):
+            return rffi.cast(rffi.CCHARP, self.buffer)
+
+        def _charbuf_stop(self):
+            keepalive_until_here(self)
 
         def w_getitem(self, space, idx):
             item = self.buffer[idx]
@@ -527,8 +540,10 @@
         self.fromstring(space.str_w(w_s))
 
     def array_tostring__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
+        self._charbuf_stop()
+        return self.space.wrap(s)
 
     def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
         if not isinstance(w_f, W_File):
@@ -583,37 +598,34 @@
             raise OperationError(space.w_ValueError, space.wrap(msg))
 
     # Compare methods
+    @specialize.arg(3)
     def _cmp_impl(space, self, other, space_fn):
-        if isinstance(other, W_ArrayBase):
-            w_lst1 = array_tolist__Array(space, self)
-            w_lst2 = space.call_method(other, 'tolist')
-            return space_fn(w_lst1, w_lst2)
-        else:
-            return space.w_NotImplemented
+        w_lst1 = array_tolist__Array(space, self)
+        w_lst2 = space.call_method(other, 'tolist')
+        return space_fn(w_lst1, w_lst2)
 
-    def eq__Array_ANY(space, self, other):
+    def eq__Array_ArrayBase(space, self, other):
         return _cmp_impl(space, self, other, space.eq)
 
-    def ne__Array_ANY(space, self, other):
+    def ne__Array_ArrayBase(space, self, other):
         return _cmp_impl(space, self, other, space.ne)
 
-    def lt__Array_ANY(space, self, other):
+    def lt__Array_ArrayBase(space, self, other):
         return _cmp_impl(space, self, other, space.lt)
 
-    def le__Array_ANY(space, self, other):
+    def le__Array_ArrayBase(space, self, other):
         return _cmp_impl(space, self, other, space.le)
 
-    def gt__Array_ANY(space, self, other):
+    def gt__Array_ArrayBase(space, self, other):
         return _cmp_impl(space, self, other, space.gt)
 
-    def ge__Array_ANY(space, self, other):
+    def ge__Array_ArrayBase(space, self, other):
         return _cmp_impl(space, self, other, space.ge)
 
     # Misc methods
 
     def buffer__Array(space, self):
-        b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes)
-        return space.wrap(b)
+        return space.wrap(ArrayBuffer(self))
 
     def array_buffer_info__Array(space, self):
         w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer))
@@ -648,7 +660,7 @@
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
-        bytes = self.charbuf()
+        bytes = self._charbuf_start()
         tmp = [bytes[0]] * mytype.bytes
         for start in range(0, self.len * mytype.bytes, mytype.bytes):
             stop = start + mytype.bytes - 1
@@ -656,6 +668,7 @@
                 tmp[i] = bytes[start + i]
             for i in range(mytype.bytes):
                 bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
 
     def repr__Array(space, self):
         if self.len == 0:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -433,7 +433,25 @@
         a = self.array('h', 'Hi')
         buf = buffer(a)
         assert buf[1] == 'i'
-        #raises(TypeError, buf.__setitem__, 1, 'o')
+
+    def test_buffer_write(self):
+        a = self.array('c', 'hello')
+        buf = buffer(a)
+        print repr(buf)
+        try:
+            buf[3] = 'L'
+        except TypeError:
+            skip("buffer(array) returns a read-only buffer on CPython")
+        assert a.tostring() == 'helLo'
+
+    def test_buffer_keepalive(self):
+        buf = buffer(self.array('c', 'text'))
+        assert buf[2] == 'x'
+        #
+        a = self.array('c', 'foobarbaz')
+        buf = buffer(a)
+        a.fromstring('some extra text')
+        assert buf[:] == 'foobarbazsome extra text'
 
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
@@ -845,8 +863,11 @@
         cls.maxint = sys.maxint
 
 class AppTestArray(BaseArrayTests):
+    OPTIONS = {}
+
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=('array', 'struct', '_rawffi'))
+        cls.space = gettestobjspace(usemodules=('array', 'struct', '_rawffi'),
+                                    **cls.OPTIONS)
         cls.w_array = cls.space.appexec([], """():
             import array
             return array.array
@@ -868,3 +889,7 @@
         a = self.array('b', range(4))
         a[::-1] = a
         assert a == self.array('b', [3, 2, 1, 0])
+
+
+class AppTestArrayBuiltinShortcut(AppTestArray):
+    OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -352,6 +352,9 @@
     'PyObject_AsReadBuffer', 'PyObject_AsWriteBuffer', 'PyObject_CheckReadBuffer',
 
     'PyOS_getsig', 'PyOS_setsig',
+    'PyThread_create_key', 'PyThread_delete_key', 'PyThread_set_key_value',
+    'PyThread_get_key_value', 'PyThread_delete_key_value',
+    'PyThread_ReInitTLS',
 
     'PyStructSequence_InitType', 'PyStructSequence_New',
 ]
@@ -617,6 +620,10 @@
         lambda space: init_pycobject(),
         lambda space: init_capsule(),
     ])
+    from pypy.module.posix.interp_posix import add_fork_hook
+    reinit_tls = rffi.llexternal('PyThread_ReInitTLS', [], lltype.Void,
+                                 compilation_info=eci)    
+    add_fork_hook('child', reinit_tls)
 
 def init_function(func):
     INIT_FUNCTIONS.append(func)
@@ -817,6 +824,8 @@
     pypy_decls.append("#ifdef __cplusplus")
     pypy_decls.append("extern \"C\" {")
     pypy_decls.append("#endif\n")
+    pypy_decls.append('#define Signed   long           /* xxx temporary fix */\n')
+    pypy_decls.append('#define Unsigned unsigned long  /* xxx temporary fix */\n')
 
     for decl in FORWARD_DECLS:
         pypy_decls.append("%s;" % (decl,))
@@ -848,6 +857,8 @@
             typ = 'PyObject*'
         pypy_decls.append('PyAPI_DATA(%s) %s;' % (typ, name))
 
+    pypy_decls.append('#undef Signed    /* xxx temporary fix */\n')
+    pypy_decls.append('#undef Unsigned  /* xxx temporary fix */\n')
     pypy_decls.append("#ifdef __cplusplus")
     pypy_decls.append("}")
     pypy_decls.append("#endif")
@@ -926,6 +937,7 @@
                                source_dir / "structseq.c",
                                source_dir / "capsule.c",
                                source_dir / "pysignals.c",
+                               source_dir / "thread.c",
                                ],
         separate_module_sources=separate_module_sources,
         export_symbols=export_symbols_eci,
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,8 +2,10 @@
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
     PyObjectFields, PyObject)
-from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
+from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
+from pypy.interpreter.error import OperationError
+from pypy.module.array.interp_array import ArrayBuffer
 
 
 PyBufferObjectStruct = lltype.ForwardReference()
@@ -41,26 +43,38 @@
         py_buf.c_b_offset = w_obj.offset
         w_obj = w_obj.buffer
 
+    # If w_obj already allocated a fixed buffer, use it, and keep a
+    # reference to w_obj.
+    # Otherwise, b_base stays NULL, and we own the b_ptr.
+
     if isinstance(w_obj, StringBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.as_str()))
+        py_buf.c_b_base = lltype.nullptr(PyObject.TO)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
+        py_buf.c_b_size = w_obj.getlength()
+    elif isinstance(w_obj, ArrayBuffer):
+        w_base = w_obj.array
+        py_buf.c_b_base = make_ref(space, w_base)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start())
         py_buf.c_b_size = w_obj.getlength()
     else:
-        raise Exception("Fail fail fail fail fail")
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "buffer flavor not supported"))
 
 
 def buffer_realize(space, py_obj):
     """
     Creates the buffer in the PyPy interpreter from a cpyext representation.
     """
-    raise Exception("realize fail fail fail")
-
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "Don't know how to realize a buffer"))
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
-    Py_DecRef(space, py_buf.c_b_base)
-    rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
+    if py_buf.c_b_base:
+        Py_DecRef(space, py_buf.c_b_base)
+    else:
+        rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/include/pythread.h b/pypy/module/cpyext/include/pythread.h
--- a/pypy/module/cpyext/include/pythread.h
+++ b/pypy/module/cpyext/include/pythread.h
@@ -3,8 +3,26 @@
 
 #define WITH_THREAD
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef void *PyThread_type_lock;
 #define WAIT_LOCK	1
 #define NOWAIT_LOCK	0
 
+/* Thread Local Storage (TLS) API */
+PyAPI_FUNC(int) PyThread_create_key(void);
+PyAPI_FUNC(void) PyThread_delete_key(int);
+PyAPI_FUNC(int) PyThread_set_key_value(int, void *);
+PyAPI_FUNC(void *) PyThread_get_key_value(int);
+PyAPI_FUNC(void) PyThread_delete_key_value(int key);
+
+/* Cleanup after a fork */
+PyAPI_FUNC(void) PyThread_ReInitTLS(void);
+
+#ifdef __cplusplus
+}
 #endif
+
+#endif
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -315,3 +316,65 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py
--- a/pypy/module/cpyext/pystate.py
+++ b/pypy/module/cpyext/pystate.py
@@ -10,7 +10,7 @@
     [('next', PyInterpreterState)],
     PyInterpreterStateStruct)
 PyThreadState = lltype.Ptr(cpython_struct(
-    "PyThreadState", 
+    "PyThreadState",
     [('interp', PyInterpreterState),
      ('dict', PyObject),
      ]))
@@ -19,12 +19,15 @@
 def PyEval_SaveThread(space):
     """Release the global interpreter lock (if it has been created and thread
     support is enabled) and reset the thread state to NULL, returning the
-    previous thread state (which is not NULL except in PyPy).  If the lock has been created,
+    previous thread state.  If the lock has been created,
     the current thread must have acquired it.  (This function is available even
     when thread support is disabled at compile time.)"""
+    state = space.fromcache(InterpreterState)
     if rffi.aroundstate.before:
         rffi.aroundstate.before()
-    return lltype.nullptr(PyThreadState.TO)
+    tstate = state.swap_thread_state(
+        space, lltype.nullptr(PyThreadState.TO))
+    return tstate
 
 @cpython_api([PyThreadState], lltype.Void)
 def PyEval_RestoreThread(space, tstate):
@@ -35,6 +38,8 @@
     when thread support is disabled at compile time.)"""
     if rffi.aroundstate.after:
         rffi.aroundstate.after()
+    state = space.fromcache(InterpreterState)
+    state.swap_thread_state(space, tstate)
 
 @cpython_api([], lltype.Void)
 def PyEval_InitThreads(space):
@@ -67,28 +72,91 @@
                                   dealloc=ThreadState_dealloc)
 
 from pypy.interpreter.executioncontext import ExecutionContext
+
+# Keep track of the ThreadStateCapsule for a particular execution context.  The
+# default is for new execution contexts not to have one; it is allocated on the
+# first cpyext-based request for it.
 ExecutionContext.cpyext_threadstate = ThreadStateCapsule(None)
 
+# Also keep track of whether it has been initialized yet or not (None is a valid
+# PyThreadState for an execution context to have, when the GIL has been
+# released, so a check against that can't be used to determine the need for
+# initialization).
+ExecutionContext.cpyext_initialized_threadstate = False
+
+def cleanup_cpyext_state(self):
+    try:
+        del self.cpyext_threadstate
+    except AttributeError:
+        pass
+    self.cpyext_initialized_threadstate = False
+ExecutionContext.cleanup_cpyext_state = cleanup_cpyext_state
+
 class InterpreterState(object):
     def __init__(self, space):
         self.interpreter_state = lltype.malloc(
             PyInterpreterState.TO, flavor='raw', zero=True, immortal=True)
 
     def new_thread_state(self, space):
+        """
+        Create a new ThreadStateCapsule to hold the PyThreadState for a
+        particular execution context.
+
+        :param space: A space.
+
+        :returns: A new ThreadStateCapsule holding a newly allocated
+            PyThreadState and referring to this interpreter state.
+        """
         capsule = ThreadStateCapsule(space)
         ts = capsule.memory
         ts.c_interp = self.interpreter_state
         ts.c_dict = make_ref(space, space.newdict())
         return capsule
 
+
     def get_thread_state(self, space):
+        """
+        Get the current PyThreadState for the current execution context.
+
+        :param space: A space.
+
+        :returns: The current PyThreadState for the current execution context,
+            or None if it does not have one.
+        """
         ec = space.getexecutioncontext()
         return self._get_thread_state(space, ec).memory
 
+
+    def swap_thread_state(self, space, tstate):
+        """
+        Replace the current thread state of the current execution context with a
+        new thread state.
+
+        :param space: The space.
+
+        :param tstate: The new PyThreadState for the current execution context.
+
+        :returns: The old thread state for the current execution context, either
+            None or a PyThreadState.
+        """
+        ec = space.getexecutioncontext()
+        capsule = self._get_thread_state(space, ec)
+        old_tstate = capsule.memory
+        capsule.memory = tstate
+        return old_tstate
+
     def _get_thread_state(self, space, ec):
-        if ec.cpyext_threadstate.memory == lltype.nullptr(PyThreadState.TO):
+        """
+        Get the ThreadStateCapsule for the given execution context, possibly
+        creating a new one if it does not already have one.
+
+        :param space: The space.
+        :param ec: The ExecutionContext of which to get the thread state.
+        :returns: The ThreadStateCapsule for the given execution context.
+        """
+        if not ec.cpyext_initialized_threadstate:
             ec.cpyext_threadstate = self.new_thread_state(space)
-
+            ec.cpyext_initialized_threadstate = True
         return ec.cpyext_threadstate
 
 @cpython_api([], PyThreadState, error=CANNOT_FAIL)
@@ -105,13 +173,8 @@
 def PyThreadState_Swap(space, tstate):
     """Swap the current thread state with the thread state given by the argument
     tstate, which may be NULL.  The global interpreter lock must be held."""
-    # All cpyext calls release and acquire the GIL, so this function has no
-    # side-effects
-    if tstate:
-        return lltype.nullptr(PyThreadState.TO)
-    else:
-        state = space.fromcache(InterpreterState)
-        return state.get_thread_state(space)
+    state = space.fromcache(InterpreterState)
+    return state.swap_thread_state(space, tstate)
 
 @cpython_api([PyThreadState], lltype.Void)
 def PyEval_AcquireThread(space, tstate):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,14 +167,16 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -183,7 +185,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/src/getargs.c b/pypy/module/cpyext/src/getargs.c
--- a/pypy/module/cpyext/src/getargs.c
+++ b/pypy/module/cpyext/src/getargs.c
@@ -23,16 +23,33 @@
 #define FLAG_COMPAT 1
 #define FLAG_SIZE_T 2
 
+typedef int (*destr_t)(PyObject *, void *);
+
+
+/* Keep track of "objects" that have been allocated or initialized and
+   which will need to be deallocated or cleaned up somehow if overall
+   parsing fails.
+*/
+typedef struct {
+  void *item;
+  destr_t destructor;
+} freelistentry_t;
+
+typedef struct {
+  int first_available;
+  freelistentry_t *entries;
+} freelist_t;
+
 
 /* Forward */
 static int vgetargs1(PyObject *, const char *, va_list *, int);
 static void seterror(int, const char *, int *, const char *, const char *);
 static char *convertitem(PyObject *, const char **, va_list *, int, int *, 
-                         char *, size_t, PyObject **);
+                         char *, size_t, freelist_t *);
 static char *converttuple(PyObject *, const char **, va_list *, int,
-			  int *, char *, size_t, int, PyObject **);
+			  int *, char *, size_t, int, freelist_t *);
 static char *convertsimple(PyObject *, const char **, va_list *, int, char *,
-			   size_t, PyObject **);
+			   size_t, freelist_t *);
 static Py_ssize_t convertbuffer(PyObject *, void **p, char **);
 static int getbuffer(PyObject *, Py_buffer *, char**);
 
@@ -129,57 +146,56 @@
 
 /* Handle cleanup of allocated memory in case of exception */
 
-static void
-cleanup_ptr(void *ptr)
+static int
+cleanup_ptr(PyObject *self, void *ptr)
 {
-	PyMem_FREE(ptr);
-}
-
-static void
-cleanup_buffer(void *ptr)
-{
-	PyBuffer_Release((Py_buffer *) ptr);
+    if (ptr) {
+        PyMem_FREE(ptr);
+    }
+    return 0;
 }
 
 static int
-addcleanup(void *ptr, PyObject **freelist, void (*destr)(void *))
+cleanup_buffer(PyObject *self, void *ptr)
 {
-	PyObject *cobj;
-	if (!*freelist) {
-		*freelist = PyList_New(0);
-		if (!*freelist) {
-			destr(ptr);
-			return -1;
-		}
-	}
-	cobj = PyCObject_FromVoidPtr(ptr, destr);
-	if (!cobj) {
-		destr(ptr);
-		return -1;
-	}
-	if (PyList_Append(*freelist, cobj)) {
-		Py_DECREF(cobj);
-		return -1;
-	}
-        Py_DECREF(cobj);
-	return 0;
+    Py_buffer *buf = (Py_buffer *)ptr;
+    if (buf) {
+        PyBuffer_Release(buf);
+    }
+    return 0;
 }
 
 static int
-cleanreturn(int retval, PyObject *freelist)
+addcleanup(void *ptr, freelist_t *freelist, destr_t destructor)
 {
-	if (freelist && retval != 0) {
-		/* We were successful, reset the destructors so that they
-		   don't get called. */
-		Py_ssize_t len = PyList_GET_SIZE(freelist), i;
-		for (i = 0; i < len; i++)
-			((PyCObject *) PyList_GET_ITEM(freelist, i))
-				->destructor = NULL;
-	}
-	Py_XDECREF(freelist);
-	return retval;
+    int index;
+
+    index = freelist->first_available;
+    freelist->first_available += 1;
+
+    freelist->entries[index].item = ptr;
+    freelist->entries[index].destructor = destructor;
+
+    return 0;
 }
 
+static int
+cleanreturn(int retval, freelist_t *freelist)
+{
+    int index;
+
+    if (retval == 0) {
+      /* A failure occurred, therefore execute all of the cleanup
+	 functions.
+      */
+      for (index = 0; index < freelist->first_available; ++index) {
+          freelist->entries[index].destructor(NULL,
+                                              freelist->entries[index].item);
+      }
+    }
+    PyMem_Free(freelist->entries);
+    return retval;
+}
 
 static int
 vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags)
@@ -195,7 +211,7 @@
 	const char *formatsave = format;
 	Py_ssize_t i, len;
 	char *msg;
-	PyObject *freelist = NULL;
+	freelist_t freelist = {0, NULL};
 	int compat = flags & FLAG_COMPAT;
 
 	assert(compat || (args != (PyObject*)NULL));
@@ -251,16 +267,18 @@
 	
 	format = formatsave;
 	
+	freelist.entries = PyMem_New(freelistentry_t, max);
+
 	if (compat) {
 		if (max == 0) {
 			if (args == NULL)
-				return 1;
+			    return cleanreturn(1, &freelist);
 			PyOS_snprintf(msgbuf, sizeof(msgbuf),
 				      "%.200s%s takes no arguments",
 				      fname==NULL ? "function" : fname,
 				      fname==NULL ? "" : "()");
 			PyErr_SetString(PyExc_TypeError, msgbuf);
-			return 0;
+			return cleanreturn(0, &freelist);
 		}
 		else if (min == 1 && max == 1) {
 			if (args == NULL) {
@@ -269,26 +287,26 @@
 					      fname==NULL ? "function" : fname,
 					      fname==NULL ? "" : "()");
 				PyErr_SetString(PyExc_TypeError, msgbuf);
-				return 0;
+				return cleanreturn(0, &freelist);
 			}
 			msg = convertitem(args, &format, p_va, flags, levels, 
 					  msgbuf, sizeof(msgbuf), &freelist);
 			if (msg == NULL)
-				return cleanreturn(1, freelist);
+				return cleanreturn(1, &freelist);
 			seterror(levels[0], msg, levels+1, fname, message);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 		else {
 			PyErr_SetString(PyExc_SystemError,
 			    "old style getargs format uses new features");
-			return 0;
+			return cleanreturn(0, &freelist);
 		}
 	}
 	
 	if (!PyTuple_Check(args)) {
 		PyErr_SetString(PyExc_SystemError,
 		    "new style getargs format but argument is not a tuple");
-		return 0;
+		return cleanreturn(0, &freelist);
 	}
 	
 	len = PyTuple_GET_SIZE(args);
@@ -308,7 +326,7 @@
 			message = msgbuf;
 		}
 		PyErr_SetString(PyExc_TypeError, message);
-		return 0;
+		return cleanreturn(0, &freelist);
 	}
 	
 	for (i = 0; i < len; i++) {
@@ -319,7 +337,7 @@
 				  sizeof(msgbuf), &freelist);
 		if (msg) {
 			seterror(i+1, msg, levels, fname, message);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 	}
 
@@ -328,10 +346,10 @@
 	    *format != '|' && *format != ':' && *format != ';') {
 		PyErr_Format(PyExc_SystemError,
 			     "bad format string: %.200s", formatsave);
-		return cleanreturn(0, freelist);
+		return cleanreturn(0, &freelist);
 	}
 	
-	return cleanreturn(1, freelist);
+	return cleanreturn(1, &freelist);
 }
 
 
@@ -395,7 +413,7 @@
 static char *
 converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
              int *levels, char *msgbuf, size_t bufsize, int toplevel, 
-             PyObject **freelist)
+             freelist_t *freelist)
 {
 	int level = 0;
 	int n = 0;
@@ -472,7 +490,7 @@
 
 static char *
 convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags,
-            int *levels, char *msgbuf, size_t bufsize, PyObject **freelist)
+            int *levels, char *msgbuf, size_t bufsize, freelist_t *freelist)
 {
 	char *msg;
 	const char *format = *p_format;
@@ -539,7 +557,7 @@
 
 static char *
 convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
-              char *msgbuf, size_t bufsize, PyObject **freelist)
+              char *msgbuf, size_t bufsize, freelist_t *freelist)
 {
 	/* For # codes */
 #define FETCH_SIZE	int *q=NULL;Py_ssize_t *q2=NULL;\
@@ -1501,7 +1519,9 @@
 	const char *fname, *msg, *custom_msg, *keyword;
 	int min = INT_MAX;
 	int i, len, nargs, nkeywords;
-	PyObject *freelist = NULL, *current_arg;
+	PyObject *current_arg;
+	freelist_t freelist = {0, NULL};
+
 
 	assert(args != NULL && PyTuple_Check(args));
 	assert(keywords == NULL || PyDict_Check(keywords));
@@ -1525,6 +1545,8 @@
 	for (len=0; kwlist[len]; len++)
 		continue;
 
+	freelist.entries = PyMem_New(freelistentry_t, len);
+
 	nargs = PyTuple_GET_SIZE(args);
 	nkeywords = (keywords == NULL) ? 0 : PyDict_Size(keywords);
 	if (nargs + nkeywords > len) {
@@ -1535,7 +1557,7 @@
 			     len,
 			     (len == 1) ? "" : "s",
 			     nargs + nkeywords);
-		return 0;
+		return cleanreturn(0, &freelist);
 	}
 
 	/* convert tuple args and keyword args in same loop, using kwlist to drive process */
@@ -1549,7 +1571,7 @@
 			PyErr_Format(PyExc_RuntimeError,
 				     "More keyword list entries (%d) than "
 				     "format specifiers (%d)", len, i);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 		current_arg = NULL;
 		if (nkeywords) {
@@ -1563,11 +1585,11 @@
 					     "Argument given by name ('%s') "
 					     "and position (%d)",
 					     keyword, i+1);
-				return cleanreturn(0, freelist);
+				return cleanreturn(0, &freelist);
 			}
 		}
 		else if (nkeywords && PyErr_Occurred())
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		else if (i < nargs)
 			current_arg = PyTuple_GET_ITEM(args, i);
 			
@@ -1576,7 +1598,7 @@
 				levels, msgbuf, sizeof(msgbuf), &freelist);
 			if (msg) {
 				seterror(i+1, msg, levels, fname, custom_msg);
-				return cleanreturn(0, freelist);
+				return cleanreturn(0, &freelist);
 			}
 			continue;
 		}
@@ -1585,14 +1607,14 @@
 			PyErr_Format(PyExc_TypeError, "Required argument "
 				     "'%s' (pos %d) not found",
 				     keyword, i+1);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 		/* current code reports success when all required args
 		 * fulfilled and no keyword args left, with no further
 		 * validation. XXX Maybe skip this in debug build ?
 		 */
 		if (!nkeywords)
-			return cleanreturn(1, freelist);
+			return cleanreturn(1, &freelist);
 
 		/* We are into optional args, skip thru to any remaining
 		 * keyword args */
@@ -1600,7 +1622,7 @@
 		if (msg) {
 			PyErr_Format(PyExc_RuntimeError, "%s: '%s'", msg,
 				     format);
-			return cleanreturn(0, freelist);
+			return cleanreturn(0, &freelist);
 		}
 	}
 
@@ -1608,7 +1630,7 @@
 		PyErr_Format(PyExc_RuntimeError,
 			"more argument specifiers than keyword list entries "
 			"(remaining format:'%s')", format);
-		return cleanreturn(0, freelist);
+		return cleanreturn(0, &freelist);
 	}
 
 	/* make sure there are no extraneous keyword arguments */
@@ -1621,7 +1643,7 @@
 			if (!PyString_Check(key)) {
                             PyErr_SetString(PyExc_TypeError, 
 					        "keywords must be strings");
-				return cleanreturn(0, freelist);
+				return cleanreturn(0, &freelist);
 			}
 			ks = PyString_AsString(key);
 			for (i = 0; i < len; i++) {
@@ -1635,12 +1657,12 @@
 					     "'%s' is an invalid keyword "
 					     "argument for this function",
 					     ks);
-				return cleanreturn(0, freelist);
+				return cleanreturn(0, &freelist);
 			}
 		}
 	}
 
-	return cleanreturn(1, freelist);
+	return cleanreturn(1, &freelist);
 }
 
 
diff --git a/pypy/module/cpyext/src/thread.c b/pypy/module/cpyext/src/thread.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/thread.c
@@ -0,0 +1,313 @@
+#include <Python.h>
+#include "pythread.h"
+
+/* ------------------------------------------------------------------------
+Per-thread data ("key") support.
+
+Use PyThread_create_key() to create a new key.  This is typically shared
+across threads.
+
+Use PyThread_set_key_value(thekey, value) to associate void* value with
+thekey in the current thread.  Each thread has a distinct mapping of thekey
+to a void* value.  Caution:  if the current thread already has a mapping
+for thekey, value is ignored.
+
+Use PyThread_get_key_value(thekey) to retrieve the void* value associated
+with thekey in the current thread.  This returns NULL if no value is
+associated with thekey in the current thread.
+
+Use PyThread_delete_key_value(thekey) to forget the current thread's associated
+value for thekey.  PyThread_delete_key(thekey) forgets the values associated
+with thekey across *all* threads.
+
+While some of these functions have error-return values, none set any
+Python exception.
+
+None of the functions does memory management on behalf of the void* values.
+You need to allocate and deallocate them yourself.  If the void* values
+happen to be PyObject*, these functions don't do refcount operations on
+them either.
+
+The GIL does not need to be held when calling these functions; they supply
+their own locking.  This isn't true of PyThread_create_key(), though (see
+next paragraph).
+
+There's a hidden assumption that PyThread_create_key() will be called before
+any of the other functions are called.  There's also a hidden assumption
+that calls to PyThread_create_key() are serialized externally.
+------------------------------------------------------------------------ */
+
+#ifdef MS_WINDOWS
+#include <windows.h>
+
+/* use native Windows TLS functions */
+#define Py_HAVE_NATIVE_TLS
+
+int
+PyThread_create_key(void)
+{
+    return (int) TlsAlloc();
+}
+
+void
+PyThread_delete_key(int key)
+{
+    TlsFree(key);
+}
+
+/* We must be careful to emulate the strange semantics implemented in thread.c,
+ * where the value is only set if it hasn't been set before.
+ */
+int
+PyThread_set_key_value(int key, void *value)
+{
+    BOOL ok;
+    void *oldvalue;
+
+    assert(value != NULL);
+    oldvalue = TlsGetValue(key);
+    if (oldvalue != NULL)
+        /* ignore value if already set */
+        return 0;
+    ok = TlsSetValue(key, value);
+    if (!ok)
+        return -1;
+    return 0;
+}
+
+void *
+PyThread_get_key_value(int key)
+{
+    /* because TLS is used in the Py_END_ALLOW_THREAD macro,
+     * it is necessary to preserve the windows error state, because
+     * it is assumed to be preserved across the call to the macro.
+     * Ideally, the macro should be fixed, but it is simpler to
+     * do it here.
+     */
+    DWORD error = GetLastError();
+    void *result = TlsGetValue(key);
+    SetLastError(error);
+    return result;
+}
+
+void
+PyThread_delete_key_value(int key)
+{
+    /* NULL is used as "key missing", and it is also the default
+     * given by TlsGetValue() if nothing has been set yet.
+     */
+    TlsSetValue(key, NULL);
+}
+
+/* reinitialization of TLS is not necessary after fork when using
+ * the native TLS functions.  And forking isn't supported on Windows either.
+ */
+void
+PyThread_ReInitTLS(void)
+{}
+
+#else  /* MS_WINDOWS */
+
+/* A singly-linked list of struct key objects remembers all the key->value
+ * associations.  File static keyhead heads the list.  keymutex is used
+ * to enforce exclusion internally.
+ */
+struct key {
+    /* Next record in the list, or NULL if this is the last record. */
+    struct key *next;
+
+    /* The thread id, according to PyThread_get_thread_ident(). */
+    long id;
+
+    /* The key and its associated value. */
+    int key;
+    void *value;
+};
+
+static struct key *keyhead = NULL;
+static PyThread_type_lock keymutex = NULL;
+static int nkeys = 0;  /* PyThread_create_key() hands out nkeys+1 next */
+
+/* Internal helper.
+ * If the current thread has a mapping for key, the appropriate struct key*
+ * is returned.  NB:  value is ignored in this case!
+ * If there is no mapping for key in the current thread, then:
+ *     If value is NULL, NULL is returned.
+ *     Else a mapping of key to value is created for the current thread,
+ *     and a pointer to a new struct key* is returned; except that if
+ *     malloc() can't find room for a new struct key*, NULL is returned.
+ * So when value==NULL, this acts like a pure lookup routine, and when
+ * value!=NULL, this acts like dict.setdefault(), returning an existing
+ * mapping if one exists, else creating a new mapping.
+ *
+ * Caution:  this used to be too clever, trying to hold keymutex only
+ * around the "p->next = keyhead; keyhead = p" pair.  That allowed
+ * another thread to mutate the list, via key deletion, concurrent with
+ * find_key() crawling over the list.  Hilarity ensued.  For example, when
+ * the for-loop here does "p = p->next", p could end up pointing at a
+ * record that PyThread_delete_key_value() was concurrently free()'ing.
+ * That could lead to anything, from failing to find a key that exists, to
+ * segfaults.  Now we lock the whole routine.
+ */
+static struct key *
+find_key(int key, void *value)
+{
+    struct key *p, *prev_p;
+    long id = PyThread_get_thread_ident();
+
+    if (!keymutex)
+        return NULL;
+    PyThread_acquire_lock(keymutex, 1);
+    prev_p = NULL;
+    for (p = keyhead; p != NULL; p = p->next) {
+        if (p->id == id && p->key == key)
+            goto Done;
+        /* Sanity check.  These states should never happen but if
+         * they do we must abort.  Otherwise we'll end up spinning in
+         * in a tight loop with the lock held.  A similar check is done
+         * in pystate.c tstate_delete_common().  */
+        if (p == prev_p)
+            Py_FatalError("tls find_key: small circular list(!)");
+        prev_p = p;
+        if (p->next == keyhead)
+            Py_FatalError("tls find_key: circular list(!)");
+    }
+    if (value == NULL) {
+        assert(p == NULL);
+        goto Done;
+    }
+    p = (struct key *)malloc(sizeof(struct key));
+    if (p != NULL) {
+        p->id = id;
+        p->key = key;
+        p->value = value;
+        p->next = keyhead;
+        keyhead = p;
+    }
+ Done:
+    PyThread_release_lock(keymutex);
+    return p;
+}
+
+/* Return a new key.  This must be called before any other functions in
+ * this family, and callers must arrange to serialize calls to this
+ * function.  No violations are detected.
+ */
+int
+PyThread_create_key(void)
+{
+    /* All parts of this function are wrong if it's called by multiple
+     * threads simultaneously.
+     */
+    if (keymutex == NULL)
+        keymutex = PyThread_allocate_lock();
+    return ++nkeys;
+}
+
+/* Forget the associations for key across *all* threads. */
+void
+PyThread_delete_key(int key)
+{
+    struct key *p, **q;
+
+    PyThread_acquire_lock(keymutex, 1);
+    q = &keyhead;
+    while ((p = *q) != NULL) {
+        if (p->key == key) {
+            *q = p->next;
+            free((void *)p);
+            /* NB This does *not* free p->value! */
+        }
+        else
+            q = &p->next;
+    }
+    PyThread_release_lock(keymutex);
+}
+
+/* Confusing:  If the current thread has an association for key,
+ * value is ignored, and 0 is returned.  Else an attempt is made to create
+ * an association of key to value for the current thread.  0 is returned
+ * if that succeeds, but -1 is returned if there's not enough memory
+ * to create the association.  value must not be NULL.
+ */
+int
+PyThread_set_key_value(int key, void *value)
+{
+    struct key *p;
+
+    assert(value != NULL);
+    p = find_key(key, value);
+    if (p == NULL)
+        return -1;
+    else
+        return 0;
+}
+
+/* Retrieve the value associated with key in the current thread, or NULL
+ * if the current thread doesn't have an association for key.
+ */
+void *
+PyThread_get_key_value(int key)
+{
+    struct key *p = find_key(key, NULL);
+
+    if (p == NULL)
+        return NULL;
+    else
+        return p->value;
+}
+
+/* Forget the current thread's association for key, if any. */
+void
+PyThread_delete_key_value(int key)
+{
+    long id = PyThread_get_thread_ident();
+    struct key *p, **q;
+
+    PyThread_acquire_lock(keymutex, 1);
+    q = &keyhead;
+    while ((p = *q) != NULL) {
+        if (p->key == key && p->id == id) {
+            *q = p->next;
+            free((void *)p);
+            /* NB This does *not* free p->value! */
+            break;
+        }
+        else
+            q = &p->next;
+    }
+    PyThread_release_lock(keymutex);
+}
+
+/* Forget everything not associated with the current thread id.
+ * This function is called from PyOS_AfterFork().  It is necessary
+ * because other thread ids which were in use at the time of the fork
+ * may be reused for new threads created in the forked process.
+ */
+void
+PyThread_ReInitTLS(void)
+{
+    long id = PyThread_get_thread_ident();
+    struct key *p, **q;
+
+    if (!keymutex)
+        return;
+
+    /* As with interpreter_lock in PyEval_ReInitThreads()
+       we just create a new lock without freeing the old one */
+    keymutex = PyThread_allocate_lock();
+
+    /* Delete all keys which do not match the current thread id */
+    q = &keyhead;
+    while ((p = *q) != NULL) {
+        if (p->id != id) {
+            *q = p->next;
+            free((void *)p);
+            /* NB This does *not* free p->value! */
+        }
+        else
+            q = &p->next;
+    }
+}
+
+#endif  /* !MS_WINDOWS */
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -130,6 +130,11 @@
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyString_AsString(space, ref):
+    if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_str:
+        pass    # typecheck returned "ok" without forcing 'ref' at all
+    elif not PyString_Check(space, ref):   # otherwise, use the alternate way
+        raise OperationError(space.w_TypeError, space.wrap(
+            "PyString_AsString only support strings"))
     ref_str = rffi.cast(PyStringObject, ref)
     if not ref_str.c_buffer:
         # copy string buffer
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -10,7 +10,7 @@
     return False
 
 def pytest_funcarg__space(request):
-    return gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+    return gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
 
 def pytest_funcarg__api(request):
     return request.cls.api
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -19,7 +19,8 @@
 
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
-        cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+        cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi',
+                                                        'array'])
 
         # warm up reference counts:
         # - the posix module allocates a HCRYPTPROV on Windows
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -1,3 +1,4 @@
+from pypy.conftest import gettestobjspace
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 import py
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -48,3 +48,17 @@
             ])
         b = module.buffer_new()
         raises(AttributeError, getattr, b, 'x')
+
+    def test_array_buffer(self):
+        module = self.import_extension('foo', [
+            ("roundtrip", "METH_O",
+             """
+                 PyBufferObject *buf = (PyBufferObject *)args;
+                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
+             """),
+            ])
+        import array
+        a = array.array('c', 'text')
+        b = buffer(a)
+        assert module.roundtrip(b) == 'text'
+        
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -35,7 +35,7 @@
 
 class AppTestApi:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
         from pypy.rlib.libffi import get_libc_name
         cls.w_libc = cls.space.wrap(get_libc_name())
 
@@ -106,10 +106,7 @@
             del obj
         import gc; gc.collect()
 
-        try:
-            del space.getexecutioncontext().cpyext_threadstate
-        except AttributeError:
-            pass
+        space.getexecutioncontext().cleanup_cpyext_state()
 
         for w_obj in state.non_heaptypes_w:
             Py_DecRef(space, w_obj)
@@ -168,8 +165,9 @@
         return leaking
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
+    
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
         cls.space.getbuiltinmodule("cpyext")
         from pypy.module.imp.importing import importhook
         importhook(cls.space, "os") # warm up reference counts
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -19,7 +19,7 @@
                                          space.wrap('__name__'))) == 'foobar'
 
     def test_getmoduledict(self, space, api):
-        testmod = "binascii"
+        testmod = "_functools"
         w_pre_dict = api.PyImport_GetModuleDict()
         assert not space.is_true(space.contains(w_pre_dict, space.wrap(testmod)))
 
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -101,9 +101,9 @@
                                   space.wrap((2, 7)))):
             py.test.skip("unsupported before Python 2.7")
 
-        assert api._PyLong_Sign(space.wrap(0L)) == 0
-        assert api._PyLong_Sign(space.wrap(2L)) == 1
-        assert api._PyLong_Sign(space.wrap(-2L)) == -1
+        assert api._PyLong_Sign(space.wraplong(0L)) == 0
+        assert api._PyLong_Sign(space.wraplong(2L)) == 1
+        assert api._PyLong_Sign(space.wraplong(-2L)) == -1
 
         assert api._PyLong_NumBits(space.wrap(0)) == 0
         assert api._PyLong_NumBits(space.wrap(1)) == 1
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py
--- a/pypy/module/cpyext/test/test_number.py
+++ b/pypy/module/cpyext/test/test_number.py
@@ -6,12 +6,12 @@
 class TestIterator(BaseApiTest):
     def test_check(self, space, api):
         assert api.PyIndex_Check(space.wrap(12))
-        assert api.PyIndex_Check(space.wrap(-12L))
+        assert api.PyIndex_Check(space.wraplong(-12L))
         assert not api.PyIndex_Check(space.wrap(12.1))
         assert not api.PyIndex_Check(space.wrap('12'))
 
         assert api.PyNumber_Check(space.wrap(12))
-        assert api.PyNumber_Check(space.wrap(-12L))
+        assert api.PyNumber_Check(space.wraplong(-12L))
         assert api.PyNumber_Check(space.wrap(12.1))
         assert not api.PyNumber_Check(space.wrap('12'))
         assert not api.PyNumber_Check(space.wrap(1+3j))
@@ -21,7 +21,7 @@
         assert api.PyLong_CheckExact(w_l)
 
     def test_number_int(self, space, api):
-        w_l = api.PyNumber_Int(space.wrap(123L))
+        w_l = api.PyNumber_Int(space.wraplong(123L))
         assert api.PyInt_CheckExact(w_l)
         w_l = api.PyNumber_Int(space.wrap(2 << 65))
         assert api.PyLong_CheckExact(w_l)
@@ -29,7 +29,7 @@
         assert api.PyInt_CheckExact(w_l)
 
     def test_number_index(self, space, api):
-        w_l = api.PyNumber_Index(space.wrap(123L))
+        w_l = api.PyNumber_Index(space.wraplong(123L))
         assert api.PyLong_CheckExact(w_l)
         w_l = api.PyNumber_Index(space.wrap(42.3))
         assert w_l is None
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info
diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py
--- a/pypy/module/cpyext/test/test_pystate.py
+++ b/pypy/module/cpyext/test/test_pystate.py
@@ -3,6 +3,10 @@
 from pypy.rpython.lltypesystem.lltype import nullptr
 from pypy.module.cpyext.pystate import PyInterpreterState, PyThreadState
 from pypy.module.cpyext.pyobject import from_ref
+from pypy.rpython.lltypesystem import lltype
+from pypy.module.cpyext.test.test_cpyext import LeakCheckingTest, freeze_refcnts
+from pypy.module.cpyext.pystate import PyThreadState_Get, PyInterpreterState_Head
+from pypy.tool import leakfinder
 
 class AppTestThreads(AppTestCpythonExtensionBase):
     def test_allow_threads(self):
@@ -21,6 +25,93 @@
         # Should compile at least
         module.test()
 
+
+    def test_thread_state_get(self):
+        module = self.import_extension('foo', [
+                ("get", "METH_NOARGS",
+                 """
+                     PyThreadState *tstate = PyThreadState_Get();
+                     if (tstate == NULL) {
+                         return PyLong_FromLong(0);
+                     }
+                     if (tstate->interp != PyInterpreterState_Head()) {
+                         return PyLong_FromLong(1);
+                     }
+                     if (tstate->interp->next != NULL) {
+                         return PyLong_FromLong(2);
+                     }
+                     return PyLong_FromLong(3);
+                 """),
+                ])
+        assert module.get() == 3
+
+    def test_basic_threadstate_dance(self):
+        module = self.import_extension('foo', [
+                ("dance", "METH_NOARGS",
+                 """
+                     PyThreadState *old_tstate, *new_tstate;
+
+                     old_tstate = PyThreadState_Swap(NULL);
+                     if (old_tstate == NULL) {
+                         return PyLong_FromLong(0);
+                     }
+
+                     new_tstate = PyThreadState_Get();
+                     if (new_tstate != NULL) {
+                         return PyLong_FromLong(1);
+                     }
+
+                     new_tstate = PyThreadState_Swap(old_tstate);
+                     if (new_tstate != NULL) {
+                         return PyLong_FromLong(2);
+                     }
+
+                     new_tstate = PyThreadState_Get();
+                     if (new_tstate != old_tstate) {
+                         return PyLong_FromLong(3);
+                     }
+
+                     return PyLong_FromLong(4);
+                 """),
+                ])
+        assert module.dance() == 4
+
+    def test_threadstate_dict(self):
+        module = self.import_extension('foo', [
+                ("getdict", "METH_NOARGS",
+                 """
+                 PyObject *dict = PyThreadState_GetDict();
+                 Py_INCREF(dict);
+                 return dict;
+                 """),
+                ])
+        assert isinstance(module.getdict(), dict)
+
+    def test_savethread(self):
+        module = self.import_extension('foo', [
+                ("bounce", "METH_NOARGS",
+                 """
+                 PyThreadState *tstate = PyEval_SaveThread();
+                 if (tstate == NULL) {
+                     return PyLong_FromLong(0);
+                 }
+
+                 if (PyThreadState_Get() != NULL) {
+                     return PyLong_FromLong(1);
+                 }
+
+                 PyEval_RestoreThread(tstate);
+
+                 if (PyThreadState_Get() != tstate) {
+                     return PyLong_FromLong(2);
+                 }
+
+                 return PyLong_FromLong(3);
+                                  """),
+                ])
+
+
+
 class TestInterpreterState(BaseApiTest):
     def test_interpreter_head(self, space, api):
         state = api.PyInterpreterState_Head()
@@ -29,31 +120,3 @@
     def test_interpreter_next(self, space, api):
         state = api.PyInterpreterState_Head()
         assert nullptr(PyInterpreterState.TO) == api.PyInterpreterState_Next(state)
-
-class TestThreadState(BaseApiTest):
-    def test_thread_state_get(self, space, api):
-        ts = api.PyThreadState_Get()
-        assert ts != nullptr(PyThreadState.TO)
-
-    def test_thread_state_interp(self, space, api):
-        ts = api.PyThreadState_Get()
-        assert ts.c_interp == api.PyInterpreterState_Head()
-        assert ts.c_interp.c_next == nullptr(PyInterpreterState.TO)
-
-    def test_basic_threadstate_dance(self, space, api):
-        # Let extension modules call these functions,
-        # Not sure of the semantics in pypy though.
-        # (cpyext always acquires and releases the GIL around calls)
-        tstate = api.PyThreadState_Swap(None)
-        assert tstate is not None
-        assert not api.PyThreadState_Swap(tstate)
-
-        api.PyEval_AcquireThread(tstate)
-        api.PyEval_ReleaseThread(tstate)
-
-    def test_threadstate_dict(self, space, api):
-        ts = api.PyThreadState_Get()
-        ref = ts.c_dict
-        assert ref == api.PyThreadState_GetDict()
-        w_obj = from_ref(space, ref)
-        assert space.isinstance_w(w_obj, space.w_dict)
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -105,6 +105,15 @@
             )])
         assert module.string_as_string("huheduwe") == "huhe"
 
+    def test_py_string_as_string_None(self):
+        module = self.import_extension('foo', [
+            ("string_None", "METH_VARARGS",
+             '''
+             return PyString_AsString(Py_None);
+             '''
+            )])
+        raises(TypeError, module.string_None)
+
     def test_AsStringAndSize(self):
         module = self.import_extension('foo', [
             ("getstring", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_thread.py b/pypy/module/cpyext/test/test_thread.py
--- a/pypy/module/cpyext/test/test_thread.py
+++ b/pypy/module/cpyext/test/test_thread.py
@@ -5,6 +5,7 @@
 
 from pypy.module.thread.ll_thread import allocate_ll_lock
 from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 
 class TestPyThread(BaseApiTest):
@@ -38,3 +39,51 @@
         api.PyThread_release_lock(lock)
         assert api.PyThread_acquire_lock(lock, 0) == 1
         api.PyThread_free_lock(lock)
+
+
+class AppTestThread(AppTestCpythonExtensionBase):
+    def test_tls(self):
+        module = self.import_extension('foo', [
+            ("create_key", "METH_NOARGS",
+             """
+                 return PyInt_FromLong(PyThread_create_key());
+             """),
+            ("test_key", "METH_O",
+             """
+                 int key = PyInt_AsLong(args);
+                 if (PyThread_get_key_value(key) != NULL) {
+                     PyErr_SetNone(PyExc_ValueError);
+                     return NULL;
+                 }
+                 if (PyThread_set_key_value(key, (void*)123) < 0) {
+                     PyErr_SetNone(PyExc_ValueError);
+                     return NULL;
+                 }
+                 if (PyThread_get_key_value(key) != (void*)123) {
+                     PyErr_SetNone(PyExc_ValueError);
+                     return NULL;
+                 }
+                 Py_RETURN_NONE;
+             """),
+            ])
+        key = module.create_key()
+        assert key > 0
+        # Test value in main thread.
+        module.test_key(key)
+        raises(ValueError, module.test_key, key)
+        # Same test, in another thread.
+        result = []
+        import thread, time
+        def in_thread():
+            try:
+                module.test_key(key)
+                raises(ValueError, module.test_key, key)
+            except Exception, e:
+                result.append(e)
+            else:
+                result.append(True)
+        thread.start_new_thread(in_thread, ())
+        while not result:
+            print "."
+            time.sleep(.5)
+        assert result == [True]
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
@@ -414,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -425,6 +429,36 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
+
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -307,6 +307,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -624,7 +626,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -178,7 +178,7 @@
 @cpython_api([], Py_UNICODE, error=CANNOT_FAIL)
 def PyUnicode_GetMax(space):
     """Get the maximum ordinal for a Unicode character."""
-    return unichr(runicode.MAXUNICODE)
+    return runicode.UNICHR(runicode.MAXUNICODE)
 
 @cpython_api([PyObject], rffi.CCHARP, error=CANNOT_FAIL)
 def PyUnicode_AS_DATA(space, ref):
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -13,7 +13,7 @@
 
 class AppTestFcntl:
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('fcntl', 'array'))
+        space = gettestobjspace(usemodules=('fcntl', 'array', 'struct'))
         cls.space = space
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
         cls.w_tmp = space.wrap(tmpprefix)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -987,6 +987,10 @@
             os.environ['LANG'] = oldlang
 
 class AppTestImportHooks(object):
+
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('struct',))
+    
     def test_meta_path(self):
         tried_imports = []
         class Importer(object):
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -891,7 +891,7 @@
 
 class AppTestItertools27:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['itertools'])
+        cls.space = gettestobjspace(usemodules=['itertools', 'struct'])
         if cls.space.is_true(cls.space.appexec([], """():
             import sys; return sys.version_info < (2, 7)
             """)):
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -327,8 +327,10 @@
     # %r not supported in rpython
     #u.raise_exc('invalid typecode in unmarshal: %r' % tc)
     c = ord(tc)
-    if c < 32 or c > 126:
-        s = '\\x' + hex(c)
+    if c < 16:
+        s = '\\x0%x' % c
+    elif c < 32 or c > 126:
+        s = '\\x%x' % c
     elif tc == '\\':
         s = r'\\'
     else:
diff --git a/pypy/module/marshal/test/make_test_marshal.py b/pypy/module/marshal/test/make_test_marshal.py
deleted file mode 100644
--- a/pypy/module/marshal/test/make_test_marshal.py
+++ /dev/null
@@ -1,78 +0,0 @@
-
-TESTCASES = """\
-    None
-    False
-    True
-    StopIteration
-    Ellipsis
-    42
-    -17
-    sys.maxint
-    -1.25
-    -1.25 #2
-    2+5j
-    2+5j #2
-    42L
-    -1234567890123456789012345678901234567890L
-    hello   # not interned
-    "hello"
-    ()
-    (1, 2)
-    []
-    [3, 4]
-    {}
-    {5: 6, 7: 8}
-    func.func_code
-    scopefunc.func_code
-    u'hello'
-    set()
-    set([1, 2])
-    frozenset()
-    frozenset([3, 4])
-""".strip().split('\n')
-
-def readable(s):
-    for c, repl in (
-        ("'", '_quote_'), ('"', '_Quote_'), (':', '_colon_'), ('.', '_dot_'),
-        ('[', '_list_'), (']', '_tsil_'), ('{', '_dict_'), ('}', '_tcid_'),
-        ('-', '_minus_'), ('+', '_plus_'),
-        (',', '_comma_'), ('(', '_brace_'), (')', '_ecarb_') ):
-        s = s.replace(c, repl)
-    lis = list(s)
-    for i, c in enumerate(lis):
-        if c.isalnum() or c == '_':
-            continue
-        lis[i] = '_'
-    return ''.join(lis)
-
-print """class AppTestMarshal:
-"""
-for line in TESTCASES:
-    line = line.strip()
-    name = readable(line)
-    version = ''
-    extra = ''
-    if line.endswith('#2'):
-        version = ', 2'
-        extra = '; assert len(s) in (9, 17)'
-    src = '''\
-    def test_%(name)s(self):
-        import sys
-        hello = "he"
-        hello += "llo"
-        def func(x):
-            return lambda y: x+y
-        scopefunc = func(42)
-        import marshal, StringIO
-        case = %(line)s
-        print "case: %%-30s   func=%(name)s" %% (case, )
-        s = marshal.dumps(case%(version)s)%(extra)s
-        x = marshal.loads(s)
-        assert x == case
-        f = StringIO.StringIO()
-        marshal.dump(case, f)
-        f.seek(0)
-        x = marshal.load(f)
-        assert x == case
-''' % {'name': name, 'line': line, 'version' : version, 'extra': extra}
-    print src
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -174,6 +174,11 @@
                 pass
             raises(ValueError, marshal.dumps, subtype)
 
+    def test_bad_typecode(self):
+        import marshal
+        exc = raises(ValueError, marshal.loads, chr(1))
+        assert r"'\x01'" in exc.value.message
+
 
 class AppTestRope(AppTestMarshal):
     def setup_class(cls):
diff --git a/pypy/module/math/test/test_direct.py b/pypy/module/math/test/test_direct.py
--- a/pypy/module/math/test/test_direct.py
+++ b/pypy/module/math/test/test_direct.py
@@ -55,6 +55,15 @@
         ('frexp', (-1.25,), lambda x: x == (-0.625, 1)),
         ('modf',  (4.25,), lambda x: x == (0.25, 4.0)),
         ('modf',  (-4.25,), lambda x: x == (-0.25, -4.0)),
+        ('copysign', (1.5, 0.0), 1.5),
+        ('copysign', (1.5, -0.0), -1.5),
+        ('copysign', (1.5, INFINITY), 1.5),
+        ('copysign', (1.5, -INFINITY), -1.5),
+        ]
+    if sys.platform != 'win32':    # all NaNs seem to be negative there...?
+        IRREGCASES += [
+        ('copysign', (1.5, NAN), 1.5),
+        ('copysign', (1.75, -NAN), -1.75),      # special case for -NAN here
         ]
 
     OVFCASES = [
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import sys
 from pypy.conftest import gettestobjspace
 from pypy.module.math.test import test_direct
@@ -5,7 +6,7 @@
 
 class AppTestMath:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['math'])
+        cls.space = gettestobjspace(usemodules=['math', 'struct'])
         cls.w_cases = cls.space.wrap(test_direct.MathTests.TESTCASES)
         cls.w_consistent_host = cls.space.wrap(test_direct.consistent_host)
 
@@ -268,3 +269,7 @@
             def __trunc__(self):
                 return "truncated"
         assert math.trunc(foo()) == "truncated"
+
+    def test_copysign_nan(self):
+        import math
+        assert math.copysign(1.0, float('-nan')) == -1.0
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
@@ -37,26 +38,44 @@
         'True_': 'types.Bool.True',
         'False_': 'types.Bool.False',
 
+        'typeinfo': 'interp_dtype.get_dtype_cache(space).w_typeinfo',
+
         'generic': 'interp_boxes.W_GenericBox',
         'number': 'interp_boxes.W_NumberBox',
         'integer': 'interp_boxes.W_IntegerBox',
         'signedinteger': 'interp_boxes.W_SignedIntegerBox',
         'unsignedinteger': 'interp_boxes.W_UnsignedIntegerBox',
         'bool_': 'interp_boxes.W_BoolBox',
+        'bool8': 'interp_boxes.W_BoolBox',
         'int8': 'interp_boxes.W_Int8Box',
+        'byte': 'interp_boxes.W_Int8Box',
         'uint8': 'interp_boxes.W_UInt8Box',
+        'ubyte': 'interp_boxes.W_UInt8Box',
         'int16': 'interp_boxes.W_Int16Box',
+        'short': 'interp_boxes.W_Int16Box',
         'uint16': 'interp_boxes.W_UInt16Box',
+        'ushort': 'interp_boxes.W_UInt16Box',
         'int32': 'interp_boxes.W_Int32Box',
+        'intc': 'interp_boxes.W_Int32Box',
         'uint32': 'interp_boxes.W_UInt32Box',
+        'uintc': 'interp_boxes.W_UInt32Box',
         'int64': 'interp_boxes.W_Int64Box',
         'uint64': 'interp_boxes.W_UInt64Box',
+        'longlong': 'interp_boxes.W_LongLongBox',
+        'ulonglong': 'interp_boxes.W_ULongLongBox',
         'int_': 'interp_boxes.W_LongBox',
         'inexact': 'interp_boxes.W_InexactBox',
         'floating': 'interp_boxes.W_FloatingBox',
         'float_': 'interp_boxes.W_Float64Box',
         'float32': 'interp_boxes.W_Float32Box',
         'float64': 'interp_boxes.W_Float64Box',
+        'intp': 'types.IntP.BoxType',
+        'uintp': 'types.UIntP.BoxType',
+        'flexible': 'interp_boxes.W_FlexibleBox',
+        'character': 'interp_boxes.W_CharacterBox',
+        'str_': 'interp_boxes.W_StringBox',
+        'unicode_': 'interp_boxes.W_UnicodeBox',
+        'void': 'interp_boxes.W_VoidBox',
     }
 
     # ufuncs
@@ -67,6 +86,7 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arctan2", "arctan2"),
         ("arccosh", "arccosh"),
         ("arcsinh", "arcsinh"),
         ("arctanh", "arctanh"),
@@ -77,9 +97,15 @@
         ("true_divide", "true_divide"),
         ("equal", "equal"),
         ("exp", "exp"),
+        ("exp2", "exp2"),
+        ("expm1", "expm1"),
         ("fabs", "fabs"),
+        ("fmax", "fmax"),
+        ("fmin", "fmin"),
+        ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -89,24 +115,44 @@
         ("multiply", "multiply"),
         ("negative", "negative"),
         ("not_equal", "not_equal"),
+        ("radians", "radians"),
+        ("degrees", "degrees"),
+        ("deg2rad", "radians"),
+        ("rad2deg", "degrees"),
         ("reciprocal", "reciprocal"),
         ("sign", "sign"),
+        ("signbit", "signbit"),
         ("sin", "sin"),
         ("sinh", "sinh"),
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
+        ('square', 'square'),
         ("tan", "tan"),
         ("tanh", "tanh"),
         ('bitwise_and', 'bitwise_and'),
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
+        ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
+        ('isneginf', 'isneginf'),
+        ('isposinf', 'isposinf'),
+        ('isfinite', 'isfinite'),
         ('logical_and', 'logical_and'),
         ('logical_xor', 'logical_xor'),
         ('logical_not', 'logical_not'),
         ('logical_or', 'logical_or'),
+        ('log', 'log'),
+        ('log2', 'log2'),
+        ('log10', 'log10'),
+        ('log1p', 'log1p'),
+        ('power', 'power'),
+        ('floor_divide', 'floor_divide'),
+        ('logaddexp', 'logaddexp'),
+        ('logaddexp2', 'logaddexp2'),
     ]:
         interpleveldefs[exposed] = "interp_ufuncs.get(space).%s" % impl
 
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -16,7 +16,7 @@
         a[i][i] = 1
     return a
 
-def sum(a,axis=None):
+def sum(a,axis=None, out=None):
     '''sum(a, axis=None)
     Sum of array elements over a given axis.
 
@@ -43,17 +43,17 @@
     # TODO: add to doc (once it's implemented): cumsum : Cumulative sum of array elements.
     if not hasattr(a, "sum"):
         a = _numpypy.array(a)
-    return a.sum(axis)
+    return a.sum(axis=axis, out=out)
 

From noreply at buildbot.pypy.org  Mon Apr  9 13:40:42 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  9 Apr 2012 13:40:42 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: hopefully the minimal change to
 make benchmark run again
Message-ID: <20120409114042.A032646E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r178:d92ba6094dd0
Date: 2012-04-09 13:40 +0200
http://bitbucket.org/pypy/benchmarks/changeset/d92ba6094dd0/

Log:	hopefully the minimal change to make benchmark run again

diff --git a/lib/pypy/lib-python/modified-2.7/opcode.py b/lib/pypy/lib-python/modified-2.7/opcode.py
--- a/lib/pypy/lib-python/modified-2.7/opcode.py
+++ b/lib/pypy/lib-python/modified-2.7/opcode.py
@@ -192,5 +192,6 @@
 def_op('LOOKUP_METHOD', 201)          # Index in name list
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
+def_op('BUILD_LIST_FROM_ARG', 203)
 
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib/pypy/pypy/interpreter/pyopcode.py b/lib/pypy/pypy/interpreter/pyopcode.py
--- a/lib/pypy/pypy/interpreter/pyopcode.py
+++ b/lib/pypy/pypy/interpreter/pyopcode.py
@@ -713,6 +713,17 @@
         w_list = self.space.newlist(items)
         self.pushvalue(w_list)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # this is a little dance, because list has to be before the
+        # value
+        last_val = self.popvalue()
+        try:
+            lgt = self.space.int_w(self.space.len(last_val))
+        except OperationError:
+            lgt = 0 # oh well
+        self.pushvalue(self.space.newlist([]))
+        self.pushvalue(last_val)
+
     def LOAD_ATTR(self, nameindex, next_instr):
         "obj.attributename"
         w_obj = self.popvalue()
diff --git a/lib/pypy/pypy/objspace/flow/test/test_objspace.py b/lib/pypy/pypy/objspace/flow/test/test_objspace.py
--- a/lib/pypy/pypy/objspace/flow/test/test_objspace.py
+++ b/lib/pypy/pypy/objspace/flow/test/test_objspace.py
@@ -1,6 +1,6 @@
 from __future__ import with_statement
 import new
-import py
+import py, sys
 from pypy.objspace.flow.model import Constant, Block, Link, Variable
 from pypy.objspace.flow.model import mkentrymap, c_last_exception
 from pypy.interpreter.argument import Arguments
@@ -831,16 +831,25 @@
                         c.co_filename, c.co_name, c.co_firstlineno,
                         c.co_lnotab)
 
+    def patch_opcodes(self, *opcodes):
+        flow_meth_names = flowcontext.FlowSpaceFrame.opcode_method_names
+        pyframe_meth_names = PyFrame.opcode_method_names
+        for name in opcodes:
+            num = bytecode_spec.opmap[name]
+            setattr(self, 'old_' + name, flow_meth_names[num])
+            flow_meth_names[num] = pyframe_meth_names[num]
+
+    def unpatch_opcodes(self, *opcodes):
+        flow_meth_names = flowcontext.FlowSpaceFrame.opcode_method_names
+        for name in opcodes:
+            num = bytecode_spec.opmap[name]
+            flow_meth_names[num] = getattr(self, 'old_' + name)
+
     def test_callmethod_opcode(self):
         """ Tests code generated by pypy-c compiled with CALL_METHOD
         bytecode
         """
-        flow_meth_names = flowcontext.FlowSpaceFrame.opcode_method_names
-        pyframe_meth_names = PyFrame.opcode_method_names
-        for name in ['CALL_METHOD', 'LOOKUP_METHOD']:
-            num = bytecode_spec.opmap[name]
-            locals()['old_' + name] = flow_meth_names[num]
-            flow_meth_names[num] = pyframe_meth_names[num]
+        self.patch_opcodes('CALL_METHOD', 'LOOKUP_METHOD')
         try:
             class X:
                 def m(self):
@@ -860,9 +869,31 @@
             assert all_ops['simple_call'] == 2
             assert all_ops['getattr'] == 1
         finally:
-            for name in ['CALL_METHOD', 'LOOKUP_METHOD']:
-                num = bytecode_spec.opmap[name]
-                flow_meth_names[num] = locals()['old_' + name]
+            self.unpatch_opcodes('CALL_METHOD', 'LOOKUP_METHOD')
+
+    def test_build_list_from_arg_opcode(self):
+        """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG
+        bytecode
+        """
+        if sys.version_info < (2, 7):
+            py.test.skip("2.7 only test")
+        self.patch_opcodes('BUILD_LIST_FROM_ARG')
+        try:
+            def f():
+                return [i for i in "abc"]
+
+            # this code is generated by pypy-c when compiling above f
+            pypy_code = 'd\x01\x00\xcb\x00\x00D]\x0c\x00}\x00\x00|\x00\x00^\x02\x00q\x07\x00S'
+            new_c = self.monkey_patch_code(f.func_code, 3, 67, pypy_code, (),
+                                           ('i',))
+            f2 = new.function(new_c, locals(), 'f')
+
+            graph = self.codetest(f2)
+            all_ops = self.all_operations(graph)
+            assert all_ops == {'newlist': 1, 'getattr': 1, 'simple_call': 1,
+                               'iter': 1, 'next': 1}
+        finally:
+            self.unpatch_opcodes('BUILD_LIST_FROM_ARG')
 
     def test_generator(self):
         def f():

From noreply at buildbot.pypy.org  Mon Apr  9 14:54:14 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon,  9 Apr 2012 14:54:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge pylib/pytest updates & bugfixes
Message-ID: <20120409125414.8103A46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54266:e87a047ee66f
Date: 2012-04-09 14:53 +0200
http://bitbucket.org/pypy/pypy/changeset/e87a047ee66f/

Log:	merge pylib/pytest updates & bugfixes

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return pyc
+def pytest_unconfigure(config):
+    hook = config._assertstate.hook
+    if hook is not None:
+        sys.meta_path.remove(hook)
 
-def before_module_import(mod):
-    if mod.config._assertstate.mode != "on":
-        return
-    # Some deep magic: load the source, rewrite the asserts, and write a
-    # fake pyc, so that it'll be loaded when the module is imported.
-    source = mod.fspath.read()
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
-        return
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, str(mod.fspath), "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the assertion
-        # rewriting, but I don't know of a fast way to tell.
-        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
-        return
-    mod._pyc = _write_pyc(co, mod.fspath)
-    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+def pytest_collection(session):
+    # this hook is only called when test modules are collected
+    # so for example not in the master process of pytest-xdist
+    # (which does not collect test modules)
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.set_session(session)
 
-def after_module_import(mod):
-    if not hasattr(mod, "_pyc"):
-        return
-    state = mod.config._assertstate
-    try:
-        mod._pyc.remove()
-    except py.error.ENOENT:
-        state.trace("couldn't find pyc: %r" % (mod._pyc,))
-    else:
-        state.trace("removed pyc: %r" % (mod._pyc,))
+def pytest_runtest_setup(item):
+    def callbinrepr(op, left, right):
+        hook_result = item.ihook.pytest_assertrepr_compare(
+            config=item.config, op=op, left=left, right=right)
+        for new_expl in hook_result:
+            if new_expl:
+                res = '\n~'.join(new_expl)
+                if item.config.getvalue("assertmode") == "rewrite":
+                    # The result will be fed back a python % formatting
+                    # operation, which will fail if there are extraneous
+                    # '%'s in the string. Escape them here.
+                    res = res.replace("%", "%%")
+                return res
+    util._reprcompare = callbinrepr
 
-def warn_about_missing_assertion():
+def pytest_runtest_teardown(item):
+    util._reprcompare = None
+
+def pytest_sessionfinish(session):
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.session = None
+
+def _load_modules(mode):
+    """Lazily import assertion related code."""
+    global rewrite, reinterpret
+    from _pytest.assertion import reinterpret
+    if mode == "rewrite":
+        from _pytest.assertion import rewrite
+
+def warn_about_missing_assertion(mode):
     try:
         assert False
     except AssertionError:
         pass
     else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
+        if mode == "rewrite":
+            specifically = ("assertions which are not in test modules "
+                            "will be ignored")
+        else:
+            specifically = "failing tests may report as passing"
+
+        sys.stderr.write("WARNING: " + specifically +
+                        " because assert statements are not executed "
+                        "by the underlying Python interpreter "
+                        "(are you using python -O?)\n")
 
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -53,7 +53,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
+                "compute assert expression before the assert or use --assert=plain)")
 
 def run(offending_line, frame=None):
     if frame is None:
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -482,7 +482,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --nomagic)")
+                "compute assert expression before the assert or use --assert=plain)")
     else:
         return None
 
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,14 +1,258 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
+import errno
 import itertools
+import imp
+import marshal
+import os
+import struct
 import sys
+import types
 
 import py
 from _pytest.assertion import util
 
 
+# Windows gives ENOENT in places *nix gives ENOTDIR.
+if sys.platform.startswith("win"):
+    PATH_COMPONENT_NOT_DIR = errno.ENOENT
+else:
+    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
+
+# py.test caches rewritten pycs in __pycache__.
+if hasattr(imp, "get_tag"):
+    PYTEST_TAG = imp.get_tag() + "-PYTEST"
+else:
+    if hasattr(sys, "pypy_version_info"):
+        impl = "pypy"
+    elif sys.platform == "java":
+        impl = "jython"
+    else:
+        impl = "cpython"
+    ver = sys.version_info
+    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
+    del ver, impl
+
+PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
+
+REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+
+class AssertionRewritingHook(object):
+    """Import hook which rewrites asserts."""
+
+    def __init__(self):
+        self.session = None
+        self.modules = {}
+
+    def set_session(self, session):
+        self.fnpats = session.config.getini("python_files")
+        self.session = session
+
+    def find_module(self, name, path=None):
+        if self.session is None:
+            return None
+        sess = self.session
+        state = sess.config._assertstate
+        state.trace("find_module called for: %s" % name)
+        names = name.rsplit(".", 1)
+        lastname = names[-1]
+        pth = None
+        if path is not None and len(path) == 1:
+            pth = path[0]
+        if pth is None:
+            try:
+                fd, fn, desc = imp.find_module(lastname, path)
+            except ImportError:
+                return None
+            if fd is not None:
+                fd.close()
+            tp = desc[2]
+            if tp == imp.PY_COMPILED:
+                if hasattr(imp, "source_from_cache"):
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
+            elif tp != imp.PY_SOURCE:
+                # Don't know what this is.
+                return None
+        else:
+            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
+        fn_pypath = py.path.local(fn)
+        # Is this a test file?
+        if not sess.isinitpath(fn):
+            # We have to be very careful here because imports in this code can
+            # trigger a cycle.
+            self.session = None
+            try:
+                for pat in self.fnpats:
+                    if fn_pypath.fnmatch(pat):
+                        state.trace("matched test file %r" % (fn,))
+                        break
+                else:
+                    return None
+            finally:
+                self.session = sess
+        else:
+            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+        # The requested module looks like a test file, so rewrite it. This is
+        # the most magical part of the process: load the source, rewrite the
+        # asserts, and load the rewritten source. We also cache the rewritten
+        # module code in a special pyc. We must be aware of the possibility of
+        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # tricky race conditions, we maintain the following invariant: The
+        # cached pyc is always a complete, valid pyc. Operations on it must be
+        # atomic. POSIX's atomic rename comes in handy.
+        write = not sys.dont_write_bytecode
+        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
+        if write:
+            try:
+                os.mkdir(cache_dir)
+            except OSError:
+                e = sys.exc_info()[1].errno
+                if e == errno.EEXIST:
+                    # Either the __pycache__ directory already exists (the
+                    # common case) or it's blocked by a non-dir node. In the
+                    # latter case, we'll ignore it in _write_pyc.
+                    pass
+                elif e == PATH_COMPONENT_NOT_DIR:
+                    # One of the path components was not a directory, likely
+                    # because we're in a zip file.
+                    write = False
+                elif e == errno.EACCES:
+                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    write = False
+                else:
+                    raise
+        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
+        pyc = os.path.join(cache_dir, cache_name)
+        # Notice that even if we're in a read-only directory, I'm going to check
+        # for a cached pyc. This may not be optimal...
+        co = _read_pyc(fn_pypath, pyc)
+        if co is None:
+            state.trace("rewriting %r" % (fn,))
+            co = _rewrite_test(state, fn_pypath)
+            if co is None:
+                # Probably a SyntaxError in the test.
+                return None
+            if write:
+                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+        else:
+            state.trace("found cached rewritten pyc for %r" % (fn,))
+        self.modules[name] = co, pyc
+        return self
+
+    def load_module(self, name):
+        co, pyc = self.modules.pop(name)
+        # I wish I could just call imp.load_compiled here, but __file__ has to
+        # be set properly. In Python 3.2+, this all would be handled correctly
+        # by load_compiled.
+        mod = sys.modules[name] = imp.new_module(name)
+        try:
+            mod.__file__ = co.co_filename
+            # Normally, this attribute is 3.2+.
+            mod.__cached__ = pyc
+            py.builtin.exec_(co, mod.__dict__)
+        except:
+            del sys.modules[name]
+            raise
+        return sys.modules[name]
+
+def _write_pyc(co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as (C)Python, since
+    # these "pycs" should never be seen by builtin import. However, there's
+    # little reason deviate, and I hope sometime to be able to use
+    # imp.load_compiled to load them. (See the comment in load_module above.)
+    mtime = int(source_path.mtime())
+    try:
+        fp = open(pyc, "wb")
+    except IOError:
+        err = sys.exc_info()[1].errno
+        if err == PATH_COMPONENT_NOT_DIR:
+            # This happens when we get a EEXIST in find_module creating the
+            # __pycache__ directory and __pycache__ is by some non-dir node.
+            return False
+        raise
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return True
+
+RN = "\r\n".encode("utf-8")
+N = "\n".encode("utf-8")
+
+def _rewrite_test(state, fn):
+    """Try to read and rewrite *fn* and return the code object."""
+    try:
+        source = fn.read("rb")
+    except EnvironmentError:
+        return None
+    # On Python versions which are not 2.7 and less than or equal to 3.1, the
+    # parser expects *nix newlines.
+    if REWRITE_NEWLINES:
+        source = source.replace(RN, N) + N
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        state.trace("failed to parse: %r" % (fn,))
+        return None
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, fn.strpath, "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the
+        # assertion rewriting, but I don't know of a fast way to tell.
+        state.trace("failed to compile: %r" % (fn,))
+        return None
+    return co
+
+def _make_rewritten_pyc(state, fn, pyc, co):
+    """Try to dump rewritten code to *pyc*."""
+    if sys.platform.startswith("win"):
+        # Windows grants exclusive access to open files and doesn't have atomic
+        # rename, so just write into the final file.
+        _write_pyc(co, fn, pyc)
+    else:
+        # When not on windows, assume rename is atomic. Dump the code object
+        # into a file specific to this process and atomically replace it.
+        proc_pyc = pyc + "." + str(os.getpid())
+        if _write_pyc(co, fn, proc_pyc):
+            os.rename(proc_pyc, pyc)
+
+def _read_pyc(source, pyc):
+    """Possibly read a py.test pyc containing rewritten code.
+
+    Return rewritten code if successful or None if not.
+    """
+    try:
+        fp = open(pyc, "rb")
+    except IOError:
+        return None
+    try:
+        try:
+            mtime = int(source.mtime())
+            data = fp.read(8)
+        except EnvironmentError:
+            return None
+        # Check for invalid or out of date pyc file.
+        if (len(data) != 8 or
+            data[:4] != imp.get_magic() or
+            struct.unpack("<l", data[4:])[0] != mtime):
+            return None
+        co = marshal.load(fp)
+        if not isinstance(co, types.CodeType):
+            # That's interesting....
+            return None
+        return co
+    finally:
+        fp.close()
+
+
 def rewrite_asserts(mod):
     """Rewrite the assert statements in mod."""
     AssertionRewriter().run(mod)
@@ -17,13 +261,8 @@
 _saferepr = py.io.saferepr
 from _pytest.assertion.util import format_explanation as _format_explanation
 
-def _format_boolop(operands, explanations, is_or):
-    show_explanations = []
-    for operand, expl in zip(operands, explanations):
-        show_explanations.append(expl)
-        if operand == is_or:
-            break
-    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+def _format_boolop(explanations, is_or):
+    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -109,8 +348,8 @@
                     return
                 lineno += len(doc) - 1
                 expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +357,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +382,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +437,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +460,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +480,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +545,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,21 +25,39 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +86,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("</testcase>")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="xfail-marked test passes unexpectedly"/>')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('<failure message="test failure">%s</failure>',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<failure message="collection failure">%s</failure>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<skipped message="collection skipped">%s</skipped>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('<error message="test setup failure">%s</error>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="expected test failure">%s</skipped>',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('<skipped type="pytest.skip" '
-                           'message="%s">%s</skipped>',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +185,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n<testcase classname="pytest" name="internal">'
-            '    <error message="internal error">'
-            '%s</error></testcase>' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +203,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write('<testsuite ')
-        logfile.write('name="" ')
-        logfile.write('errors="%i" ' % self.errors)
-        logfile.write('failures="%i" ' % self.failed)
-        logfile.write('skips="%i" ' % self.skipped)
-        logfile.write('tests="%i" ' % numtests)
-        logfile.write('time="%.3f"' % suite_time_delta)
-        logfile.write(' >')
-        logfile.writelines(self.test_logs)
-        logfile.write('</testsuite>')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -399,6 +410,7 @@
         self._notfound = []
         self._initialpaths = set()
         self._initialparts = []
+        self.items = items = []
         for arg in args:
             parts = self._parsearg(arg)
             self._initialparts.append(parts)
@@ -414,7 +426,6 @@
         if not genitems:
             return rep.result
         else:
-            self.items = items = []
             if rep.passed:
                 for node in rep.result:
                     self.items.extend(self.genitems(node))
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "<CallSpec id=%r param=%r funcargs=%r>" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,71 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        if not indirect:
+            #XXX should we also check for the opposite case?
+            for arg in argnames:
+                if arg not in self.funcargnames:
+                    raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +678,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +769,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +781,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +830,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +956,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         <ExceptionInfo ...>
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         <ExceptionInfo ...>
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,60 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
-
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
+def pytest_sessionfinish(session):
+    session._setupstate.teardown_all()
 
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +76,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +93,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +114,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +143,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +166,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +188,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +208,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "<TestReport %r when=%r outcome=%r>" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +234,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +259,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +319,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -117,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
@@ -169,21 +192,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -289,10 +282,18 @@
         # we take care to leave out Instances aka ()
         # because later versions are going to get rid of them anyway
         if self.config.option.verbose < 0:
-            for item in items:
-                nodeid = item.nodeid
-                nodeid = nodeid.replace("::()::", "::")
-                self._tw.line(nodeid)
+            if self.config.option.verbose < -1:
+                counts = {}
+                for item in items:
+                    name = item.nodeid.split('::', 1)[0]
+                    counts[name] = counts.get(name, 0) + 1
+                for name, count in sorted(counts.items()):
+                    self._tw.line("%s: %d" % (name, count))
+            else:
+                for item in items:
+                    nodeid = item.nodeid
+                    nodeid = nodeid.replace("::()::", "::")
+                    self._tw.line(nodeid)
             return
         stack = []
         indent = ""
@@ -318,12 +319,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +394,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +412,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +431,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +445,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -83,28 +91,37 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +137,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.4.dev1'
+__version__ = '1.4.7'
 
 from py import _apipkg
 
@@ -70,6 +70,11 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
+        '_AssertionError'   : '._code.assertion:AssertionError',
+        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
+        '_reinterpret'      : '._code.assertion:reinterpret',
+        '_reprcompare'      : '._code.assertion:_reprcompare',
+        '_format_explanation' : '._code.assertion:_format_explanation',
     },
 
     # backports and additions of builtins
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -113,9 +113,12 @@
 
     # some backward compatibility helpers
     _basestring = str
-    def _totext(obj, encoding=None):
+    def _totext(obj, encoding=None, errors=None):
         if isinstance(obj, bytes):
-            obj = obj.decode(encoding)
+            if errors is None:
+                obj = obj.decode(encoding)
+            else:
+                obj = obj.decode(encoding, errors)
         elif not isinstance(obj, str):
             obj = str(obj)
         return obj
@@ -142,7 +145,7 @@
             del back
         elif locs is None:
             locs = globs
-        fp = open(fn, "rb")
+        fp = open(fn, "r")
         try:
             source = fp.read()
         finally:
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionnew.py
@@ -0,0 +1,339 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace _assertionold.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from py._code.assertion import _format_explanation, BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(failure):
+    explanation = _format_explanation(failure.explanation)
+    value = failure.cause[1]
+    if str(value):
+        lines = explanation.splitlines()
+        if not lines:
+            lines.append("")
+        lines[0] += " << %s" % (value,)
+        explanation = "\n".join(lines)
+    text = "%s: %s" % (failure.cause[0].__name__, explanation)
+    if text.startswith("AssertionError: assert "):
+        text = text[16:]
+    return text
+
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = False
+        if not local:
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not result:
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        rcomp = py.code._reprcompare
+        if rcomp:
+            res = rcomp(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = True
+        if from_instance:
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        if test_explanation.startswith("False\n{False =") and \
+                test_explanation.endswith("\n"):
+            test_explanation = test_explanation[15:-2]
+        explanation = "assert %s" % (test_explanation,)
+        if not test_result:
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionold.py
@@ -0,0 +1,555 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from py._code.assertion import BuiltinAssertionError, _format_explanation
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = '<eval>'
+            self.__obj__.filename = '<eval>'
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = '<run>'
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return _format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # simplify 'assert False where False = ...'
+        if (test.explanation.startswith('False\n{False = ') and
+            test.explanation.endswith('\n}')):
+            test.explanation = test.explanation[15:-2]
+        # print the result as  'assert <explanation>'
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = '<run>'
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
new file mode 100644
--- /dev/null
+++ b/py/_code/assertion.py
@@ -0,0 +1,94 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+_reprcompare = None # if set, will be called by assert reinterp for comparison ops
+
+def _format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "<run>".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = "<could not determine information>"
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from py._code._assertionold import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from py._code._assertionnew import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,6 +145,17 @@
         return self.frame.f_locals
     locals = property(getlocals, None, None, "locals of underlaying frame")
 
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        if self.exprinfo is None:
+            source = str(self.statement).strip()
+            x = py.code._reinterpret(source, self.frame, should_fail=True)
+            if not isinstance(x, str):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
     def getfirstlinesource(self):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
@@ -158,13 +169,12 @@
         end = self.lineno
         try:
             _, end = source.getstatementrange(end)
-        except IndexError:
+        except (IndexError, ValueError):
             end = self.lineno + 1
         # heuristic to stop displaying source on e.g.
         #   if something:  # assume this causes a NameError
         #      # _this_ lines and the one
                #        below we don't want from entry.getsource()
-        end = min(end, len(source))
         for i in range(self.lineno, end):
             if source[i].rstrip().endswith(':'):
                 end = i + 1
@@ -273,7 +283,11 @@
         """
         cache = {}
         for i, entry in enumerate(self):
-            key = entry.frame.code.path, entry.lineno
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
             #print "checking for recursion at", key
             l = cache.setdefault(key, [])
             if l:
@@ -308,7 +322,7 @@
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
         self.type, self.value, tb = self._excinfo
-        self.typename = getattr(self.type, "__name__", "???")
+        self.typename = self.type.__name__
         self.traceback = py.code.Traceback(tb)
 
     def __repr__(self):
@@ -347,14 +361,16 @@
             showlocals: show locals per traceback entry
             style: long|short|no|native traceback style
             tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
         """
         if style == 'native':
-            import traceback
-            return ''.join(traceback.format_exception(
-                self.type,
-                self.value,
-                self.traceback[0]._rawentry,
-                ))
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
 
         fmt = FormattedExcinfo(showlocals=showlocals, style=style,
             abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
@@ -452,7 +468,7 @@
     def repr_locals(self, locals):
         if self.showlocals:
             lines = []
-            keys = list(locals)
+            keys = [loc for loc in locals if loc[0] != "@"]
             keys.sort()
             for name in keys:
                 value = locals[name]
@@ -506,7 +522,10 @@
 
     def _makepath(self, path):
         if not self.abspath:
-            np = py.path.local().bestrelpath(path)
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
             if len(np) < len(str(path)):
                 path = np
         return path
@@ -595,6 +614,19 @@
         if self.extraline:
             tw.line(self.extraline)
 
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
 class ReprEntry(TerminalRepr):
     localssep = "_ "
 
@@ -680,19 +712,26 @@
 
 oldbuiltins = {}
 
-def patch_builtins(compile=True):
-    """ put compile builtins to Python's builtins. """
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from py._code import assertion
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = assertion.AssertionError
     if compile:
         l = oldbuiltins.setdefault('compile', [])
         l.append(py.builtin.builtins.compile)
         py.builtin.builtins.compile = py.code.compile
 
-def unpatch_builtins(compile=True):
+def unpatch_builtins(assertion=True, compile=True):
     """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
     if compile:
         py.builtin.builtins.compile = oldbuiltins['compile'].pop()
 
-def getrawcode(obj):
+def getrawcode(obj, trycall=True):
     """ return code object for given function. """
     try:
         return obj.__code__
@@ -701,5 +740,10 @@
         obj = getattr(obj, 'func_code', obj)
         obj = getattr(obj, 'f_code', obj)
         obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
         return obj
 
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,6 +108,7 @@
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
+            raise an IndexError if no such statementrange can be found.
         """
         # XXX there must be a better than these heuristic ways ...
         # XXX there may even be better heuristics :-)
@@ -116,6 +117,7 @@
 
         # 1. find the start of the statement
         from codeop import compile_command
+        end = None
         for start in range(lineno, -1, -1):
             if assertion:
                 line = self.lines[start]
@@ -139,7 +141,9 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, len(self)
+        if end is None:
+            raise IndexError("no valid source range around line %d " % (lineno,))
+        return start, end
 
     def getblockend(self, lineno):
         # XXX
@@ -257,23 +261,29 @@
 
 
 def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
     try:
         code = py.code.Code(obj)
     except TypeError:
-        # fallback to
-        fn = (py.std.inspect.getsourcefile(obj) or
-              py.std.inspect.getfile(obj))
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
         fspath = fn and py.path.local(fn) or None
+        lineno = -1
         if fspath:
             try:
                 _, lineno = findsource(obj)
             except IOError:
-                lineno = None
-        else:
-            lineno = None
+                pass
     else:
         fspath = code.path
         lineno = code.firstlineno
+    assert isinstance(lineno, int)
     return fspath, lineno
 
 #
@@ -286,7 +296,7 @@
     except py.builtin._sysex:
         raise
     except:
-        return None, None
+        return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
diff --git a/py/_error.py b/py/_error.py
--- a/py/_error.py
+++ b/py/_error.py
@@ -23,6 +23,7 @@
     2: errno.ENOENT,
     3: errno.ENOENT,
     17: errno.EEXIST,
+    13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -103,6 +103,7 @@
     def _parseline(self, line, lineno):
         # comments
         line = line.split('#')[0].rstrip()
+        line = line.split(';')[0].rstrip()
         # blank lines
         if not line:
             return None, None
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -12,7 +12,7 @@
     class TextIO(StringIO):
         def write(self, data):
             if not isinstance(data, unicode):
-                data = unicode(data, getattr(self, '_encoding', 'UTF-8'))
+                data = unicode(data, getattr(self, '_encoding', 'UTF-8'), 'replace')
             StringIO.write(self, data)
 else:
     TextIO = StringIO
@@ -258,6 +258,9 @@
                 f = getattr(self, name).tmpfile
                 f.seek(0)
                 res = f.read()
+                enc = getattr(f, 'encoding', None)
+                if enc:
+                    res = py.builtin._totext(res, enc, 'replace')
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -105,6 +105,8 @@
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
+    _newline = None   # the last line printed
+
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -112,11 +114,9 @@
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-                if hasattr(file, 'encoding'):
-                    encoding = file.encoding
         elif hasattr(file, '__call__'):
             file = WriteFile(file, encoding=encoding)
-        self.encoding = encoding
+        self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
@@ -182,8 +182,31 @@
         return s
 
     def line(self, s='', **kw):
+        if self._newline == False:
+            self.write("\n")
         self.write(s, **kw)
         self.write('\n')
+        self._newline = True
+
+    def reline(self, line, **opts):
+        if not self.hasmarkup:
+            raise ValueError("cannot use rewrite-line without terminal")
+        if not self._newline:
+            self.write("\r")
+        self.write(line, **opts)
+        # see if we need to fill up some spaces at the end
+        # xxx have a more exact lastlinelen working from self.write?
+        lenline = len(line)
+        try:
+            lastlen = self._lastlinelen
+        except AttributeError:
+            pass
+        else:
+            if lenline < lastlen:
+                self.write(" " * (lastlen - lenline + 1))
+        self._lastlinelen = lenline
+        self._newline = False
+
 
 class Win32ConsoleWriter(TerminalWriter):
     def write(self, s, **kw):
@@ -280,10 +303,10 @@
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
     SetConsoleTextAttribute.argtypes = [wintypes.HANDLE, wintypes.WORD]
     SetConsoleTextAttribute.restype = wintypes.BOOL
-        
+
     _GetConsoleScreenBufferInfo = \
         ctypes.windll.kernel32.GetConsoleScreenBufferInfo
-    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, 
+    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE,
                                 ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
     _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
     def GetConsoleInfo(handle):
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -64,7 +64,10 @@
                 else:
                     if bool(value) ^ bool(meth()) ^ invert:
                         return False
-            except (py.error.ENOENT, py.error.ENOTDIR):
+            except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
+                # EBUSY feels not entirely correct,
+                # but its kind of necessary since ENOMEDIUM
+                # is not accessible in python     
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
@@ -368,6 +371,5 @@
         else:
             name = str(path) # path.strpath # XXX svn?
             pattern = '*' + path.sep + pattern
-        from fnmatch import fnmatch
-        return fnmatch(name, pattern)
+        return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -157,14 +157,16 @@
         return str(self) < str(other)
 
     def samefile(self, other):
-        """ return True if 'other' references the same file as 'self'. """
-        if not iswin32:
-            return py.error.checked_call(
-                    os.path.samefile, str(self), str(other))
+        """ return True if 'other' references the same file as 'self'.
+        """
+        if not isinstance(other, py.path.local):
+            other = os.path.abspath(str(other))
         if self == other:
             return True
-        other = os.path.abspath(str(other))
-        return self == other
+        if iswin32:
+            return False # ther is no samefile
+        return py.error.checked_call(
+                os.path.samefile, str(self), str(other))
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -539,7 +541,11 @@
                 if self.basename != "__init__.py":
                     modfile = modfile[:-12]
 
-            if not self.samefile(modfile):
+            try:
+                issame = self.samefile(modfile)
+            except py.error.ENOENT:
+                issame = False
+            if not issame:
                 raise self.ImportMismatchError(modname, modfile, self)
             return mod
         else:
diff --git a/py/_path/svnurl.py b/py/_path/svnurl.py
--- a/py/_path/svnurl.py
+++ b/py/_path/svnurl.py
@@ -233,6 +233,8 @@
                 e = sys.exc_info()[1]
                 if e.err.find('non-existent in that revision') != -1:
                     raise py.error.ENOENT(self, e.err)
+                elif e.err.find("E200009:") != -1:
+                    raise py.error.ENOENT(self, e.err)
                 elif e.err.find('File not found') != -1:
                     raise py.error.ENOENT(self, e.err)
                 elif e.err.find('not part of a repository')!=-1:
diff --git a/py/_path/svnwc.py b/py/_path/svnwc.py
--- a/py/_path/svnwc.py
+++ b/py/_path/svnwc.py
@@ -482,10 +482,13 @@
         except py.process.cmdexec.Error:
             e = sys.exc_info()[1]
             strerr = e.err.lower()
-            if strerr.find('file not found') != -1:
+            if strerr.find('not found') != -1:
+                raise py.error.ENOENT(self)
+            elif strerr.find("E200009:") != -1:
                 raise py.error.ENOENT(self)
             if (strerr.find('file exists') != -1 or
                 strerr.find('file already exists') != -1 or
+                strerr.find('w150002:') != -1 or
                 strerr.find("can't create directory") != -1):
                 raise py.error.EEXIST(self)
             raise
@@ -593,7 +596,7 @@
         out = self._authsvn('lock').strip()
         if not out:
             # warning or error, raise exception
-            raise Exception(out[4:])
+            raise ValueError("unknown error in svn lock command")
 
     def unlock(self):
         """ unset a previously set lock """
@@ -1066,6 +1069,8 @@
                 modrev = '?'
                 author = '?'
                 date = ''
+            elif itemstatus == "replaced":
+                pass
             else:
                 #print entryel.toxml()
                 commitel = entryel.getElementsByTagName('commit')[0]
@@ -1148,7 +1153,11 @@
             raise  ValueError("Not a versioned resource")
             #raise ValueError, "Not a versioned resource %r" % path
         self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind']
-        self.rev = int(d['revision'])
+        try:
+            self.rev = int(d['revision'])
+        except KeyError:
+            self.rev = None
+
         self.path = py.path.local(d['path'])
         self.size = self.path.size()
         if 'lastchangedrev' in d:
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -52,7 +52,7 @@
     def unicode(self, indent=2):
         l = []
         SimpleUnicodeVisitor(l.append, indent).visit(self)
-        return "".join(l)
+        return u("").join(l)
 
     def __repr__(self):
         name = self.__class__.__name__
@@ -122,11 +122,13 @@
                 if visitmethod is not None:
                     break
             else:
-                visitmethod = self.object
+                visitmethod = self.__object
             self.cache[cls] = visitmethod
         visitmethod(node)
 
-    def object(self, obj):
+    # the default fallback handler is marked private
+    # to avoid clashes with the tag name object
+    def __object(self, obj):
         #self.write(obj)
         self.write(escape(unicode(obj)))
 
@@ -136,7 +138,8 @@
     def list(self, obj):
         assert id(obj) not in self.visited
         self.visited[id(obj)] = 1
-        map(self.visit, obj)
+        for elem in obj:
+            self.visit(elem)
 
     def Tag(self, tag):
         assert id(tag) not in self.visited
@@ -181,7 +184,11 @@
             value = getattr(attrs, name)
             if name.endswith('_'):
                 name = name[:-1]
-            return ' %s="%s"' % (name, escape(unicode(value)))
+            if isinstance(value, raw):
+                insert = value.uniobj
+            else:
+                insert = escape(unicode(value))
+            return ' %s="%s"' % (name, insert)
 
     def getstyle(self, tag):
         """ return attribute list suitable for styling. """
diff --git a/py/bin/_findpy.py b/py/bin/_findpy.py
deleted file mode 100644
--- a/py/bin/_findpy.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-#
-# find and import a version of 'py'
-#
-import sys
-import os
-from os.path import dirname as opd, exists, join, basename, abspath
-
-def searchpy(current):
-    while 1:
-        last = current
-        initpy = join(current, '__init__.py')
-        if not exists(initpy):
-            pydir = join(current, 'py')
-            # recognize py-package and ensure it is importable
-            if exists(pydir) and exists(join(pydir, '__init__.py')):
-                #for p in sys.path:
-                #    if p == current:
-                #        return True
-                if current != sys.path[0]:  # if we are already first, then ok
-                    sys.stderr.write("inserting into sys.path: %s\n" % current)
-                    sys.path.insert(0, current)
-                return True
-        current = opd(current)
-        if last == current:
-            return False
-
-if not searchpy(abspath(os.curdir)):
-    if not searchpy(opd(abspath(sys.argv[0]))):
-        if not searchpy(opd(__file__)):
-            pass # let's hope it is just on sys.path
-
-import py
-import pytest
-
-if __name__ == '__main__':
-    print ("py lib is at %s" % py.__file__)
diff --git a/py/bin/py.test b/py/bin/py.test
deleted file mode 100755
--- a/py/bin/py.test
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env python
-from _findpy import pytest
-raise SystemExit(pytest.main())
diff --git a/pypy/module/pyexpat/test/__init__.py b/pypy/module/pyexpat/test/__init__.py
new file mode 100644
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old -rf
+addopts = --assert=plain -rf
diff --git a/pypy/test_all.py b/pypy/test_all.py
old mode 100755
new mode 100644
--- a/pypy/test_all.py
+++ b/pypy/test_all.py
@@ -11,11 +11,12 @@
 """
 import sys, os
 
-if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
-    print >> sys.stderr, __doc__
-    sys.exit(2)
 
 if __name__ == '__main__':
+    if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
+        print >> sys.stderr, __doc__
+        sys.exit(2)
+
     import tool.autopath
     import pytest
     import pytest_cov
diff --git a/pypy/tool/jitlogparser/test/__init__.py b/pypy/tool/jitlogparser/test/__init__.py
new file mode 100644
diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -165,7 +165,10 @@
     def test_one(self): exec 'blow'
     """)
 
-    ev, = sorter.getreports("pytest_runtest_logreport")
+    reports = sorter.getreports("pytest_runtest_logreport")
+    setup, ev, teardown = reports
     assert ev.failed
+    assert setup.passed
+    assert teardown.passed
     assert 'NameError' in ev.longrepr.reprcrash.message
     assert 'blow' in ev.longrepr.reprcrash.message
diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -110,7 +110,10 @@
                  do_dry_run=False, timeout=None,
                  _win32=(sys.platform=='win32')):
     args = interp + test_driver
-    args += ['-p', 'resultlog', '--resultlog=%s' % logfname, test]
+    args += ['-p', 'resultlog',
+             '--resultlog=%s' % logfname,
+             '--junitxml=%s.junit' % logfname,
+             test]
 
     args = map(str, args)
     interp0 = args[0]
diff --git a/testrunner/scratchbox_runner.py b/testrunner/scratchbox_runner.py
--- a/testrunner/scratchbox_runner.py
+++ b/testrunner/scratchbox_runner.py
@@ -14,14 +14,14 @@
 def dry_run_scratchbox(args, cwd, out, timeout=None):
     return dry_run(args_for_scratchbox(cwd, args), cwd, out, timeout)
 
-import runner
-# XXX hack hack hack
-dry_run = runner.dry_run
-run = runner.run
+if __name__ == '__main__':
+    import runner
+    # XXX hack hack hack
+    dry_run = runner.dry_run
+    run = runner.run
 
-runner.dry_run = dry_run_scratchbox
-runner.run = run_scratchbox
+    runner.dry_run = dry_run_scratchbox
+    runner.run = run_scratchbox
 
-if __name__ == '__main__':
     import sys
     runner.main(sys.argv)
diff --git a/testrunner/test/conftest.py b/testrunner/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/testrunner/test/conftest.py
@@ -0,0 +1,6 @@
+
+def pytest_runtest_makereport(__multicall__, item):
+    report = __multicall__.execute()
+    if 'out' in item.funcargs:
+        report.sections.append(('out', item.funcargs['out'].read()))
+    return report
diff --git a/testrunner/test/test_runner.py b/testrunner/test/test_runner.py
--- a/testrunner/test/test_runner.py
+++ b/testrunner/test/test_runner.py
@@ -53,49 +53,44 @@
     assert not should_report_failure("F Def\n. Ghi\n. Jkl\n")
 
 
+
 class TestRunHelper(object):
+    def pytest_funcarg__out(self, request):
+        tmpdir = request.getfuncargvalue('tmpdir')
+        return tmpdir.ensure('out')
 
-    def setup_method(self, meth):
-        h, self.fn = tempfile.mkstemp()
-        os.close(h)
+    def test_run(self, out):
+        res = runner.run([sys.executable, "-c", "print 42"], '.', out)
+        assert res == 0
+        assert out.read() == "42\n"
 
-    def teardown_method(self, meth):
-        os.unlink(self.fn)
-
-    def test_run(self):
-        res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn))
-        assert res == 0
-        out = py.path.local(self.fn).read('r')
-        assert out == "42\n"
-
-    def test_error(self):
-        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', py.path.local(self.fn))
+    def test_error(self, out):
+        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', out)
         assert res == 3
 
-    def test_signal(self):
+    def test_signal(self, out):
         if sys.platform == 'win32':
             py.test.skip("no death by signal on windows")
-        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', py.path.local(self.fn))
+        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', out)
         assert res == -9
 
-    def test_timeout(self):
-        res = runner.run([sys.executable, "-c", "while True: pass"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout(self, out):
+        res = runner.run([sys.executable, "-c", "while True: pass"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_lock(self):
-        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_lock(self, out):
+        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_syscall(self):
-        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_syscall(self, out):
+        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', out, timeout=3)
         assert res == -999        
 
-    def test_timeout_success(self):
+    def test_timeout_success(self, out):
         res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn), timeout=2)
+                         out, timeout=2)
         assert res == 0
-        out = py.path.local(self.fn).read('r')
+        out = out.read()
         assert out == "42\n"        
 
 
@@ -122,7 +117,10 @@
 
         expected = ['INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
+
                     'test_one']
 
         assert self.called == (expected, '/wd', 'out', 'secs')        
@@ -138,9 +136,11 @@
 
         expected = ['/wd' + os.sep + './INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
                     'test_one']
-
+        assert self.called[0] == expected
         assert self.called == (expected, '/wd', 'out', 'secs')        
         assert res == 0
 
@@ -251,7 +251,7 @@
         assert '\n' in log        
         log_lines = log.splitlines()
 
-        assert log_lines[0] == ". test_normal/test_example.py:test_one"
+        assert ". test_normal/test_example.py::test_one" in log_lines
         nfailures = 0
         noutcomes = 0
         for line in log_lines:

From noreply at buildbot.pypy.org  Mon Apr  9 18:22:28 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon,  9 Apr 2012 18:22:28 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: Simplify BUILD_LIST_FROM_ARG
 again, and hope that benchmarks pass
Message-ID: <20120409162228.DA2BD46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r179:d9436ef872ff
Date: 2012-04-09 18:21 +0200
http://bitbucket.org/pypy/benchmarks/changeset/d9436ef872ff/

Log:	Simplify BUILD_LIST_FROM_ARG again, and hope that benchmarks pass

diff --git a/lib/pypy/pypy/interpreter/pyopcode.py b/lib/pypy/pypy/interpreter/pyopcode.py
--- a/lib/pypy/pypy/interpreter/pyopcode.py
+++ b/lib/pypy/pypy/interpreter/pyopcode.py
@@ -717,10 +717,6 @@
         # this is a little dance, because list has to be before the
         # value
         last_val = self.popvalue()
-        try:
-            lgt = self.space.int_w(self.space.len(last_val))
-        except OperationError:
-            lgt = 0 # oh well
         self.pushvalue(self.space.newlist([]))
         self.pushvalue(last_val)
 

From noreply at buildbot.pypy.org  Mon Apr  9 18:41:28 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon,  9 Apr 2012 18:41:28 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge from default
Message-ID: <20120409164128.78C3B46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54267:94df3eb8e759
Date: 2012-04-08 20:08 +0300
http://bitbucket.org/pypy/pypy/changeset/94df3eb8e759/

Log:	merge from default

diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
new file mode 100644
--- /dev/null
+++ b/pypy/doc/tool/makecontributor.py
@@ -0,0 +1,133 @@
+import py
+import sys
+from collections import defaultdict
+import operator
+import re
+import mercurial.localrepo
+import mercurial.ui
+
+ROOT = py.path.local(__file__).join('..', '..', '..', '..')
+author_re = re.compile('(.*) <.*>')
+pair_programming_re = re.compile(r'^\((.*?)\)')
+excluded = set(["pypy", "convert-repo"])
+
+alias = {
+    'Anders Chrigstrom': ['arre'],
+    'Antonio Cuni': ['antocuni', 'anto'],
+    'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
+    'Maciej Fijalkowski': ['fijal'],
+    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Michael Hudson': ['mwh'],
+    'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
+    "Amaury Forgeot d'Arc": ['afa'],
+    'Alex Gaynor': ['alex', 'agaynor'],
+    'David Schneider': ['bivab', 'david'],
+    'Christian Tismer': ['chris', 'christian', 'tismer',
+                         'tismer at christia-wjtqxl.localdomain'],
+    'Benjamin Peterson': ['benjamin'],
+    'Hakan Ardo': ['hakan', 'hakanardo'],
+    'Niklaus Haldimann': ['nik'],
+    'Alexander Schremmer': ['xoraxax'],
+    'Anders Hammarquist': ['iko'],
+    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'Niko Matsakis': ['niko'],
+    'Jakub Gustak': ['jlg'],
+    'Guido Wesdorp': ['guido'],
+    'Michael Foord': ['mfoord'],
+    'Mark Pearse': ['mwp'],
+    'Toon Verwaest': ['tverwaes'],
+    'Eric van Riet Paap': ['ericvrp'],
+    'Jacob Hallen': ['jacob', 'jakob'],
+    'Anders Lehmann': ['ale', 'anders'],
+    'Bert Freudenberg': ['bert'],
+    'Boris Feigin': ['boris', 'boria'],
+    'Valentino Volonghi': ['valentino', 'dialtone'],
+    'Aurelien Campeas': ['aurelien', 'aureliene'],
+    'Adrien Di Mascio': ['adim'],
+    'Jacek Generowicz': ['Jacek', 'jacek'],
+    'Jim Hunziker': ['landtuna at gmail.com'],
+    'Kristjan Valur Jonsson': ['kristjan at kristjan-lp.ccp.ad.local'],
+    'Laura Creighton': ['lac'],
+    'Aaron Iles': ['aliles'],
+    'Ludovic Aubry': ['ludal', 'ludovic'],
+    'Lukas Diekmann': ['l.diekmann', 'ldiekmann'],
+    'Matti Picus': ['Matti Picus matti.picus at gmail.com',
+                    'matthp', 'mattip', 'mattip>'],
+    'Michael Cheng': ['mikefc'],
+    'Richard Emslie': ['rxe'],
+    'Roberto De Ioris': ['roberto at goyle'],
+    'Roberto De Ioris': ['roberto at mrspurr'],
+    'Sven Hager': ['hager'],
+    'Tomo Cocoa': ['cocoatomo'],
+    }
+
+alias_map = {}
+for name, nicks in alias.iteritems():
+    for nick in nicks:
+        alias_map[nick] = name
+
+def get_canonical_author(name):
+    match = author_re.match(name)
+    if match:
+        name = match.group(1)
+    return alias_map.get(name, name)
+
+ignored_nicknames = defaultdict(int)
+
+def get_more_authors(log):
+    match = pair_programming_re.match(log)
+    if not match:
+        return set()
+    ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
+                    'in-progress', 'bits of', 'even a little', 'floating',]
+    sep_words = ['and', ';', '+', '/', 'with special  by']
+    nicknames = match.group(1)
+    for word in ignore_words:
+        nicknames = nicknames.replace(word, '')
+    for word in sep_words:
+        nicknames = nicknames.replace(word, ',')
+    nicknames = [nick.strip().lower() for nick in nicknames.split(',')]
+    authors = set()
+    for nickname in nicknames:
+        author = alias_map.get(nickname)
+        if not author:
+            ignored_nicknames[nickname] += 1
+        else:
+            authors.add(author)
+    return authors
+
+def main(show_numbers):
+    ui = mercurial.ui.ui()
+    repo = mercurial.localrepo.localrepository(ui, str(ROOT))
+    authors_count = defaultdict(int)
+    for i in repo:
+        ctx = repo[i]
+        authors = set()
+        authors.add(get_canonical_author(ctx.user()))
+        authors.update(get_more_authors(ctx.description()))
+        for author in authors:
+            if author not in excluded:
+                authors_count[author] += 1
+
+    # uncomment the next lines to get the list of nicknamed which could not be
+    # parsed from commit logs
+    ## items = ignored_nicknames.items()
+    ## items.sort(key=operator.itemgetter(1), reverse=True)
+    ## for name, n in items:
+    ##     if show_numbers:
+    ##         print '%5d %s' % (n, name)
+    ##     else:
+    ##         print name
+                
+    items = authors_count.items()
+    items.sort(key=operator.itemgetter(1), reverse=True)
+    for name, n in items:
+        if show_numbers:
+            print '%5d %s' % (n, name)
+        else:
+            print name
+
+if __name__ == '__main__':
+    show_numbers = '-n' in sys.argv
+    main(show_numbers)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -597,9 +597,8 @@
 
     def test_shift(self):
         from _numpypy import left_shift, right_shift
-        import sys
 
-        assert (left_shift([5, 1], [2, 31]) == [20, 2**31]).all()
+        assert (left_shift([5, 1], [2, 13]) == [20, 2**13]).all()
         assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
 
     def test_comparisons(self):
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 2, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 9, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,8 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, classofinstance=None,
-                                   strdict=False):
+                                   instance=False, strdict=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -563,10 +562,7 @@
     def listview_int(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
 
-    def w_keys(self, w_dict):
-        # XXX there is no space.newlist_int yet
-        space = self.space
-        return space.call_function(space.w_list, w_dict)
+    # XXX there is no space.newlist_int yet to implement w_keys more efficiently
 
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -1,5 +1,5 @@
 ## ----------------------------------------------------------------------------
-## dict strategy (see dict_multiobject.py)
+## dict strategy (see dictmultiobject.py)
 
 from pypy.rlib import rerased
 from pypy.rlib.debug import mark_dict_non_null
@@ -80,8 +80,8 @@
     def iter(self, w_dict):
         return IdentityDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class IdentityDictIteratorImplementation(_UnwrappedIteratorMixin, IteratorImplementation):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,10 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                classofinstance=classofinstance,
                 strdict=strdict)
 
     def newset(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -885,10 +885,9 @@
     def newtuple(self, l):
         return tuple(l)
 
-    def newdict(self, module=False, instance=False, classofinstance=None):
+    def newdict(self, module=False, instance=False):
         return W_DictMultiObject.allocate_and_init_instance(
-                self, module=module, instance=instance,
-                classofinstance=classofinstance)
+                self, module=module, instance=instance)
 
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict
@@ -968,6 +967,20 @@
         assert type(self.impl.strategy) is self.StrategyClass
         #assert self.impl.r_dict_content is None
 
+    def test_popitem(self):
+        self.fill_impl()
+        assert self.impl.length() == 2
+        a, b = self.impl.popitem()
+        assert self.impl.length() == 1
+        if a == self.string:
+            assert b == 1000
+            assert self.impl.getitem(self.string2) == 2000
+        else:
+            assert a == self.string2
+            assert b == 2000
+            assert self.impl.getitem_str(self.string) == 1000
+        self.check_not_devolved()
+
     def test_setitem(self):
         self.impl.setitem(self.string, 1000)
         assert self.impl.length() == 1
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -847,6 +847,10 @@
                 if sources:
                     target, = sources
 
+        if target.endswith('@PLT'):
+            # In -fPIC mode, all functions calls have this suffix
+            target = target[:-4]
+
         if target in self.FUNCTIONS_NOT_RETURNING:
             return [InsnStop(target)]
         if self.format == 'mingw32' and target == '__alloca':
@@ -1137,7 +1141,7 @@
     r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$")
 
     r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$")
-    r_jmptable_item = re.compile(r"\t.quad\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$")
+    r_jmptable_item = re.compile(r"\t.(?:quad|long)\t"+LABEL+"(-\"[A-Za-z0-9$]+\"|-"+LABEL+")?\s*$")
     r_jmptable_end  = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL)
 
     r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/")

From noreply at buildbot.pypy.org  Mon Apr  9 18:41:29 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon,  9 Apr 2012 18:41:29 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: remove poll() from select for
 cpython compatability
Message-ID: <20120409164129.C2A8F46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54268:774652376acf
Date: 2012-04-09 19:33 +0300
http://bitbucket.org/pypy/pypy/changeset/774652376acf/

Log:	remove poll() from select for cpython compatability

diff --git a/pypy/module/select/__init__.py b/pypy/module/select/__init__.py
--- a/pypy/module/select/__init__.py
+++ b/pypy/module/select/__init__.py
@@ -7,12 +7,17 @@
 class Module(MixedModule):
     appleveldefs = {
     }
-
-    interpleveldefs = {
-        'poll'  : 'interp_select.poll',
-        'select': 'interp_select.select',
-        'error' : 'space.fromcache(interp_select.Cache).w_error'
-    }
+    if sys.platform.startswith('win'):
+        interpleveldefs = {
+            'select': 'interp_select.select',
+            'error' : 'space.fromcache(interp_select.Cache).w_error'
+        }
+    else:    
+        interpleveldefs = {
+            'poll'  : 'interp_select.poll',
+            'select': 'interp_select.select',
+            'error' : 'space.fromcache(interp_select.Cache).w_error'
+        }
 
     if sys.platform.startswith('linux'):
         interpleveldefs['epoll'] = 'interp_epoll.W_Epoll'
diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py
--- a/pypy/module/select/interp_select.py
+++ b/pypy/module/select/interp_select.py
@@ -53,7 +53,6 @@
             except (OverflowError, ValueError):
                 raise OperationError(space.w_ValueError,
                                      space.wrap("math range error"))
-
         try:
             retval = rpoll.poll(self.fddict, timeout)
         except rpoll.PollError, e:
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -214,6 +214,8 @@
 
     def test_poll(self):
         import select
+        if 'poll' not in dir(select):
+            skip('select.poll not available')
         readend, writeend = self.getpair()
         try:
             class A(object):

From noreply at buildbot.pypy.org  Mon Apr  9 18:41:31 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon,  9 Apr 2012 18:41:31 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge default into branch
Message-ID: <20120409164131.ECF8E46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54269:f6da3cc07d42
Date: 2012-04-09 19:34 +0300
http://bitbucket.org/pypy/pypy/changeset/f6da3cc07d42/

Log:	merge default into branch

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return pyc
+def pytest_unconfigure(config):
+    hook = config._assertstate.hook
+    if hook is not None:
+        sys.meta_path.remove(hook)
 
-def before_module_import(mod):
-    if mod.config._assertstate.mode != "on":
-        return
-    # Some deep magic: load the source, rewrite the asserts, and write a
-    # fake pyc, so that it'll be loaded when the module is imported.
-    source = mod.fspath.read()
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
-        return
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, str(mod.fspath), "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the assertion
-        # rewriting, but I don't know of a fast way to tell.
-        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
-        return
-    mod._pyc = _write_pyc(co, mod.fspath)
-    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+def pytest_collection(session):
+    # this hook is only called when test modules are collected
+    # so for example not in the master process of pytest-xdist
+    # (which does not collect test modules)
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.set_session(session)
 
-def after_module_import(mod):
-    if not hasattr(mod, "_pyc"):
-        return
-    state = mod.config._assertstate
-    try:
-        mod._pyc.remove()
-    except py.error.ENOENT:
-        state.trace("couldn't find pyc: %r" % (mod._pyc,))
-    else:
-        state.trace("removed pyc: %r" % (mod._pyc,))
+def pytest_runtest_setup(item):
+    def callbinrepr(op, left, right):
+        hook_result = item.ihook.pytest_assertrepr_compare(
+            config=item.config, op=op, left=left, right=right)
+        for new_expl in hook_result:
+            if new_expl:
+                res = '\n~'.join(new_expl)
+                if item.config.getvalue("assertmode") == "rewrite":
+                    # The result will be fed back a python % formatting
+                    # operation, which will fail if there are extraneous
+                    # '%'s in the string. Escape them here.
+                    res = res.replace("%", "%%")
+                return res
+    util._reprcompare = callbinrepr
 
-def warn_about_missing_assertion():
+def pytest_runtest_teardown(item):
+    util._reprcompare = None
+
+def pytest_sessionfinish(session):
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.session = None
+
+def _load_modules(mode):
+    """Lazily import assertion related code."""
+    global rewrite, reinterpret
+    from _pytest.assertion import reinterpret
+    if mode == "rewrite":
+        from _pytest.assertion import rewrite
+
+def warn_about_missing_assertion(mode):
     try:
         assert False
     except AssertionError:
         pass
     else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
+        if mode == "rewrite":
+            specifically = ("assertions which are not in test modules "
+                            "will be ignored")
+        else:
+            specifically = "failing tests may report as passing"
+
+        sys.stderr.write("WARNING: " + specifically +
+                        " because assert statements are not executed "
+                        "by the underlying Python interpreter "
+                        "(are you using python -O?)\n")
 
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -53,7 +53,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
+                "compute assert expression before the assert or use --assert=plain)")
 
 def run(offending_line, frame=None):
     if frame is None:
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -482,7 +482,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --nomagic)")
+                "compute assert expression before the assert or use --assert=plain)")
     else:
         return None
 
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,14 +1,258 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
+import errno
 import itertools
+import imp
+import marshal
+import os
+import struct
 import sys
+import types
 
 import py
 from _pytest.assertion import util
 
 
+# Windows gives ENOENT in places *nix gives ENOTDIR.
+if sys.platform.startswith("win"):
+    PATH_COMPONENT_NOT_DIR = errno.ENOENT
+else:
+    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
+
+# py.test caches rewritten pycs in __pycache__.
+if hasattr(imp, "get_tag"):
+    PYTEST_TAG = imp.get_tag() + "-PYTEST"
+else:
+    if hasattr(sys, "pypy_version_info"):
+        impl = "pypy"
+    elif sys.platform == "java":
+        impl = "jython"
+    else:
+        impl = "cpython"
+    ver = sys.version_info
+    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
+    del ver, impl
+
+PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
+
+REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+
+class AssertionRewritingHook(object):
+    """Import hook which rewrites asserts."""
+
+    def __init__(self):
+        self.session = None
+        self.modules = {}
+
+    def set_session(self, session):
+        self.fnpats = session.config.getini("python_files")
+        self.session = session
+
+    def find_module(self, name, path=None):
+        if self.session is None:
+            return None
+        sess = self.session
+        state = sess.config._assertstate
+        state.trace("find_module called for: %s" % name)
+        names = name.rsplit(".", 1)
+        lastname = names[-1]
+        pth = None
+        if path is not None and len(path) == 1:
+            pth = path[0]
+        if pth is None:
+            try:
+                fd, fn, desc = imp.find_module(lastname, path)
+            except ImportError:
+                return None
+            if fd is not None:
+                fd.close()
+            tp = desc[2]
+            if tp == imp.PY_COMPILED:
+                if hasattr(imp, "source_from_cache"):
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
+            elif tp != imp.PY_SOURCE:
+                # Don't know what this is.
+                return None
+        else:
+            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
+        fn_pypath = py.path.local(fn)
+        # Is this a test file?
+        if not sess.isinitpath(fn):
+            # We have to be very careful here because imports in this code can
+            # trigger a cycle.
+            self.session = None
+            try:
+                for pat in self.fnpats:
+                    if fn_pypath.fnmatch(pat):
+                        state.trace("matched test file %r" % (fn,))
+                        break
+                else:
+                    return None
+            finally:
+                self.session = sess
+        else:
+            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+        # The requested module looks like a test file, so rewrite it. This is
+        # the most magical part of the process: load the source, rewrite the
+        # asserts, and load the rewritten source. We also cache the rewritten
+        # module code in a special pyc. We must be aware of the possibility of
+        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # tricky race conditions, we maintain the following invariant: The
+        # cached pyc is always a complete, valid pyc. Operations on it must be
+        # atomic. POSIX's atomic rename comes in handy.
+        write = not sys.dont_write_bytecode
+        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
+        if write:
+            try:
+                os.mkdir(cache_dir)
+            except OSError:
+                e = sys.exc_info()[1].errno
+                if e == errno.EEXIST:
+                    # Either the __pycache__ directory already exists (the
+                    # common case) or it's blocked by a non-dir node. In the
+                    # latter case, we'll ignore it in _write_pyc.
+                    pass
+                elif e == PATH_COMPONENT_NOT_DIR:
+                    # One of the path components was not a directory, likely
+                    # because we're in a zip file.
+                    write = False
+                elif e == errno.EACCES:
+                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    write = False
+                else:
+                    raise
+        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
+        pyc = os.path.join(cache_dir, cache_name)
+        # Notice that even if we're in a read-only directory, I'm going to check
+        # for a cached pyc. This may not be optimal...
+        co = _read_pyc(fn_pypath, pyc)
+        if co is None:
+            state.trace("rewriting %r" % (fn,))
+            co = _rewrite_test(state, fn_pypath)
+            if co is None:
+                # Probably a SyntaxError in the test.
+                return None
+            if write:
+                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+        else:
+            state.trace("found cached rewritten pyc for %r" % (fn,))
+        self.modules[name] = co, pyc
+        return self
+
+    def load_module(self, name):
+        co, pyc = self.modules.pop(name)
+        # I wish I could just call imp.load_compiled here, but __file__ has to
+        # be set properly. In Python 3.2+, this all would be handled correctly
+        # by load_compiled.
+        mod = sys.modules[name] = imp.new_module(name)
+        try:
+            mod.__file__ = co.co_filename
+            # Normally, this attribute is 3.2+.
+            mod.__cached__ = pyc
+            py.builtin.exec_(co, mod.__dict__)
+        except:
+            del sys.modules[name]
+            raise
+        return sys.modules[name]
+
+def _write_pyc(co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as (C)Python, since
+    # these "pycs" should never be seen by builtin import. However, there's
+    # little reason deviate, and I hope sometime to be able to use
+    # imp.load_compiled to load them. (See the comment in load_module above.)
+    mtime = int(source_path.mtime())
+    try:
+        fp = open(pyc, "wb")
+    except IOError:
+        err = sys.exc_info()[1].errno
+        if err == PATH_COMPONENT_NOT_DIR:
+            # This happens when we get a EEXIST in find_module creating the
+            # __pycache__ directory and __pycache__ is by some non-dir node.
+            return False
+        raise
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return True
+
+RN = "\r\n".encode("utf-8")
+N = "\n".encode("utf-8")
+
+def _rewrite_test(state, fn):
+    """Try to read and rewrite *fn* and return the code object."""
+    try:
+        source = fn.read("rb")
+    except EnvironmentError:
+        return None
+    # On Python versions which are not 2.7 and less than or equal to 3.1, the
+    # parser expects *nix newlines.
+    if REWRITE_NEWLINES:
+        source = source.replace(RN, N) + N
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        state.trace("failed to parse: %r" % (fn,))
+        return None
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, fn.strpath, "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the
+        # assertion rewriting, but I don't know of a fast way to tell.
+        state.trace("failed to compile: %r" % (fn,))
+        return None
+    return co
+
+def _make_rewritten_pyc(state, fn, pyc, co):
+    """Try to dump rewritten code to *pyc*."""
+    if sys.platform.startswith("win"):
+        # Windows grants exclusive access to open files and doesn't have atomic
+        # rename, so just write into the final file.
+        _write_pyc(co, fn, pyc)
+    else:
+        # When not on windows, assume rename is atomic. Dump the code object
+        # into a file specific to this process and atomically replace it.
+        proc_pyc = pyc + "." + str(os.getpid())
+        if _write_pyc(co, fn, proc_pyc):
+            os.rename(proc_pyc, pyc)
+
+def _read_pyc(source, pyc):
+    """Possibly read a py.test pyc containing rewritten code.
+
+    Return rewritten code if successful or None if not.
+    """
+    try:
+        fp = open(pyc, "rb")
+    except IOError:
+        return None
+    try:
+        try:
+            mtime = int(source.mtime())
+            data = fp.read(8)
+        except EnvironmentError:
+            return None
+        # Check for invalid or out of date pyc file.
+        if (len(data) != 8 or
+            data[:4] != imp.get_magic() or
+            struct.unpack("<l", data[4:])[0] != mtime):
+            return None
+        co = marshal.load(fp)
+        if not isinstance(co, types.CodeType):
+            # That's interesting....
+            return None
+        return co
+    finally:
+        fp.close()
+
+
 def rewrite_asserts(mod):
     """Rewrite the assert statements in mod."""
     AssertionRewriter().run(mod)
@@ -17,13 +261,8 @@
 _saferepr = py.io.saferepr
 from _pytest.assertion.util import format_explanation as _format_explanation
 
-def _format_boolop(operands, explanations, is_or):
-    show_explanations = []
-    for operand, expl in zip(operands, explanations):
-        show_explanations.append(expl)
-        if operand == is_or:
-            break
-    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+def _format_boolop(explanations, is_or):
+    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -109,8 +348,8 @@
                     return
                 lineno += len(doc) - 1
                 expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +357,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +382,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +437,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +460,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +480,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +545,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,21 +25,39 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +86,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("</testcase>")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="xfail-marked test passes unexpectedly"/>')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('<failure message="test failure">%s</failure>',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<failure message="collection failure">%s</failure>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<skipped message="collection skipped">%s</skipped>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('<error message="test setup failure">%s</error>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="expected test failure">%s</skipped>',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('<skipped type="pytest.skip" '
-                           'message="%s">%s</skipped>',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +185,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n<testcase classname="pytest" name="internal">'
-            '    <error message="internal error">'
-            '%s</error></testcase>' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +203,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write('<testsuite ')
-        logfile.write('name="" ')
-        logfile.write('errors="%i" ' % self.errors)
-        logfile.write('failures="%i" ' % self.failed)
-        logfile.write('skips="%i" ' % self.skipped)
-        logfile.write('tests="%i" ' % numtests)
-        logfile.write('time="%.3f"' % suite_time_delta)
-        logfile.write(' >')
-        logfile.writelines(self.test_logs)
-        logfile.write('</testsuite>')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -399,6 +410,7 @@
         self._notfound = []
         self._initialpaths = set()
         self._initialparts = []
+        self.items = items = []
         for arg in args:
             parts = self._parsearg(arg)
             self._initialparts.append(parts)
@@ -414,7 +426,6 @@
         if not genitems:
             return rep.result
         else:
-            self.items = items = []
             if rep.passed:
                 for node in rep.result:
                     self.items.extend(self.genitems(node))
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "<CallSpec id=%r param=%r funcargs=%r>" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,71 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        if not indirect:
+            #XXX should we also check for the opposite case?
+            for arg in argnames:
+                if arg not in self.funcargnames:
+                    raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +678,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +769,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +781,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +830,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +956,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         <ExceptionInfo ...>
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         <ExceptionInfo ...>
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,60 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
-
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
+def pytest_sessionfinish(session):
+    session._setupstate.teardown_all()
 
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +76,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +93,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +114,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +143,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +166,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +188,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +208,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "<TestReport %r when=%r outcome=%r>" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +234,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +259,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +319,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -117,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
@@ -169,21 +192,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -289,10 +282,18 @@
         # we take care to leave out Instances aka ()
         # because later versions are going to get rid of them anyway
         if self.config.option.verbose < 0:
-            for item in items:
-                nodeid = item.nodeid
-                nodeid = nodeid.replace("::()::", "::")
-                self._tw.line(nodeid)
+            if self.config.option.verbose < -1:
+                counts = {}
+                for item in items:
+                    name = item.nodeid.split('::', 1)[0]
+                    counts[name] = counts.get(name, 0) + 1
+                for name, count in sorted(counts.items()):
+                    self._tw.line("%s: %d" % (name, count))
+            else:
+                for item in items:
+                    nodeid = item.nodeid
+                    nodeid = nodeid.replace("::()::", "::")
+                    self._tw.line(nodeid)
             return
         stack = []
         indent = ""
@@ -318,12 +319,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +394,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +412,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +431,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +445,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -83,28 +91,37 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +137,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.4.dev1'
+__version__ = '1.4.7'
 
 from py import _apipkg
 
@@ -70,6 +70,11 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
+        '_AssertionError'   : '._code.assertion:AssertionError',
+        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
+        '_reinterpret'      : '._code.assertion:reinterpret',
+        '_reprcompare'      : '._code.assertion:_reprcompare',
+        '_format_explanation' : '._code.assertion:_format_explanation',
     },
 
     # backports and additions of builtins
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -113,9 +113,12 @@
 
     # some backward compatibility helpers
     _basestring = str
-    def _totext(obj, encoding=None):
+    def _totext(obj, encoding=None, errors=None):
         if isinstance(obj, bytes):
-            obj = obj.decode(encoding)
+            if errors is None:
+                obj = obj.decode(encoding)
+            else:
+                obj = obj.decode(encoding, errors)
         elif not isinstance(obj, str):
             obj = str(obj)
         return obj
@@ -142,7 +145,7 @@
             del back
         elif locs is None:
             locs = globs
-        fp = open(fn, "rb")
+        fp = open(fn, "r")
         try:
             source = fp.read()
         finally:
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionnew.py
@@ -0,0 +1,339 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace _assertionold.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from py._code.assertion import _format_explanation, BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(failure):
+    explanation = _format_explanation(failure.explanation)
+    value = failure.cause[1]
+    if str(value):
+        lines = explanation.splitlines()
+        if not lines:
+            lines.append("")
+        lines[0] += " << %s" % (value,)
+        explanation = "\n".join(lines)
+    text = "%s: %s" % (failure.cause[0].__name__, explanation)
+    if text.startswith("AssertionError: assert "):
+        text = text[16:]
+    return text
+
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = False
+        if not local:
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not result:
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        rcomp = py.code._reprcompare
+        if rcomp:
+            res = rcomp(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = True
+        if from_instance:
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        if test_explanation.startswith("False\n{False =") and \
+                test_explanation.endswith("\n"):
+            test_explanation = test_explanation[15:-2]
+        explanation = "assert %s" % (test_explanation,)
+        if not test_result:
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionold.py
@@ -0,0 +1,555 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from py._code.assertion import BuiltinAssertionError, _format_explanation
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = '<eval>'
+            self.__obj__.filename = '<eval>'
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = '<run>'
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return _format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # simplify 'assert False where False = ...'
+        if (test.explanation.startswith('False\n{False = ') and
+            test.explanation.endswith('\n}')):
+            test.explanation = test.explanation[15:-2]
+        # print the result as  'assert <explanation>'
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = '<run>'
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
new file mode 100644
--- /dev/null
+++ b/py/_code/assertion.py
@@ -0,0 +1,94 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+_reprcompare = None # if set, will be called by assert reinterp for comparison ops
+
+def _format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "<run>".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = "<could not determine information>"
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from py._code._assertionold import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from py._code._assertionnew import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,6 +145,17 @@
         return self.frame.f_locals
     locals = property(getlocals, None, None, "locals of underlaying frame")
 
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        if self.exprinfo is None:
+            source = str(self.statement).strip()
+            x = py.code._reinterpret(source, self.frame, should_fail=True)
+            if not isinstance(x, str):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
     def getfirstlinesource(self):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
@@ -158,13 +169,12 @@
         end = self.lineno
         try:
             _, end = source.getstatementrange(end)
-        except IndexError:
+        except (IndexError, ValueError):
             end = self.lineno + 1
         # heuristic to stop displaying source on e.g.
         #   if something:  # assume this causes a NameError
         #      # _this_ lines and the one
                #        below we don't want from entry.getsource()
-        end = min(end, len(source))
         for i in range(self.lineno, end):
             if source[i].rstrip().endswith(':'):
                 end = i + 1
@@ -273,7 +283,11 @@
         """
         cache = {}
         for i, entry in enumerate(self):
-            key = entry.frame.code.path, entry.lineno
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
             #print "checking for recursion at", key
             l = cache.setdefault(key, [])
             if l:
@@ -308,7 +322,7 @@
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
         self.type, self.value, tb = self._excinfo
-        self.typename = getattr(self.type, "__name__", "???")
+        self.typename = self.type.__name__
         self.traceback = py.code.Traceback(tb)
 
     def __repr__(self):
@@ -347,14 +361,16 @@
             showlocals: show locals per traceback entry
             style: long|short|no|native traceback style
             tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
         """
         if style == 'native':
-            import traceback
-            return ''.join(traceback.format_exception(
-                self.type,
-                self.value,
-                self.traceback[0]._rawentry,
-                ))
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
 
         fmt = FormattedExcinfo(showlocals=showlocals, style=style,
             abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
@@ -452,7 +468,7 @@
     def repr_locals(self, locals):
         if self.showlocals:
             lines = []
-            keys = list(locals)
+            keys = [loc for loc in locals if loc[0] != "@"]
             keys.sort()
             for name in keys:
                 value = locals[name]
@@ -506,7 +522,10 @@
 
     def _makepath(self, path):
         if not self.abspath:
-            np = py.path.local().bestrelpath(path)
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
             if len(np) < len(str(path)):
                 path = np
         return path
@@ -595,6 +614,19 @@
         if self.extraline:
             tw.line(self.extraline)
 
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
 class ReprEntry(TerminalRepr):
     localssep = "_ "
 
@@ -680,19 +712,26 @@
 
 oldbuiltins = {}
 
-def patch_builtins(compile=True):
-    """ put compile builtins to Python's builtins. """
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from py._code import assertion
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = assertion.AssertionError
     if compile:
         l = oldbuiltins.setdefault('compile', [])
         l.append(py.builtin.builtins.compile)
         py.builtin.builtins.compile = py.code.compile
 
-def unpatch_builtins(compile=True):
+def unpatch_builtins(assertion=True, compile=True):
     """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
     if compile:
         py.builtin.builtins.compile = oldbuiltins['compile'].pop()
 
-def getrawcode(obj):
+def getrawcode(obj, trycall=True):
     """ return code object for given function. """
     try:
         return obj.__code__
@@ -701,5 +740,10 @@
         obj = getattr(obj, 'func_code', obj)
         obj = getattr(obj, 'f_code', obj)
         obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
         return obj
 
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,6 +108,7 @@
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
+            raise an IndexError if no such statementrange can be found.
         """
         # XXX there must be a better than these heuristic ways ...
         # XXX there may even be better heuristics :-)
@@ -116,6 +117,7 @@
 
         # 1. find the start of the statement
         from codeop import compile_command
+        end = None
         for start in range(lineno, -1, -1):
             if assertion:
                 line = self.lines[start]
@@ -139,7 +141,9 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, len(self)
+        if end is None:
+            raise IndexError("no valid source range around line %d " % (lineno,))
+        return start, end
 
     def getblockend(self, lineno):
         # XXX
@@ -257,23 +261,29 @@
 
 
 def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
     try:
         code = py.code.Code(obj)
     except TypeError:
-        # fallback to
-        fn = (py.std.inspect.getsourcefile(obj) or
-              py.std.inspect.getfile(obj))
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
         fspath = fn and py.path.local(fn) or None
+        lineno = -1
         if fspath:
             try:
                 _, lineno = findsource(obj)
             except IOError:
-                lineno = None
-        else:
-            lineno = None
+                pass
     else:
         fspath = code.path
         lineno = code.firstlineno
+    assert isinstance(lineno, int)
     return fspath, lineno
 
 #
@@ -286,7 +296,7 @@
     except py.builtin._sysex:
         raise
     except:
-        return None, None
+        return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
diff --git a/py/_error.py b/py/_error.py
--- a/py/_error.py
+++ b/py/_error.py
@@ -23,6 +23,7 @@
     2: errno.ENOENT,
     3: errno.ENOENT,
     17: errno.EEXIST,
+    13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -103,6 +103,7 @@
     def _parseline(self, line, lineno):
         # comments
         line = line.split('#')[0].rstrip()
+        line = line.split(';')[0].rstrip()
         # blank lines
         if not line:
             return None, None
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -12,7 +12,7 @@
     class TextIO(StringIO):
         def write(self, data):
             if not isinstance(data, unicode):
-                data = unicode(data, getattr(self, '_encoding', 'UTF-8'))
+                data = unicode(data, getattr(self, '_encoding', 'UTF-8'), 'replace')
             StringIO.write(self, data)
 else:
     TextIO = StringIO
@@ -258,6 +258,9 @@
                 f = getattr(self, name).tmpfile
                 f.seek(0)
                 res = f.read()
+                enc = getattr(f, 'encoding', None)
+                if enc:
+                    res = py.builtin._totext(res, enc, 'replace')
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -105,6 +105,8 @@
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
+    _newline = None   # the last line printed
+
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -112,11 +114,9 @@
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-                if hasattr(file, 'encoding'):
-                    encoding = file.encoding
         elif hasattr(file, '__call__'):
             file = WriteFile(file, encoding=encoding)
-        self.encoding = encoding
+        self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
@@ -182,8 +182,31 @@
         return s
 
     def line(self, s='', **kw):
+        if self._newline == False:
+            self.write("\n")
         self.write(s, **kw)
         self.write('\n')
+        self._newline = True
+
+    def reline(self, line, **opts):
+        if not self.hasmarkup:
+            raise ValueError("cannot use rewrite-line without terminal")
+        if not self._newline:
+            self.write("\r")
+        self.write(line, **opts)
+        # see if we need to fill up some spaces at the end
+        # xxx have a more exact lastlinelen working from self.write?
+        lenline = len(line)
+        try:
+            lastlen = self._lastlinelen
+        except AttributeError:
+            pass
+        else:
+            if lenline < lastlen:
+                self.write(" " * (lastlen - lenline + 1))
+        self._lastlinelen = lenline
+        self._newline = False
+
 
 class Win32ConsoleWriter(TerminalWriter):
     def write(self, s, **kw):
@@ -280,10 +303,10 @@
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
     SetConsoleTextAttribute.argtypes = [wintypes.HANDLE, wintypes.WORD]
     SetConsoleTextAttribute.restype = wintypes.BOOL
-        
+
     _GetConsoleScreenBufferInfo = \
         ctypes.windll.kernel32.GetConsoleScreenBufferInfo
-    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, 
+    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE,
                                 ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
     _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
     def GetConsoleInfo(handle):
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -64,7 +64,10 @@
                 else:
                     if bool(value) ^ bool(meth()) ^ invert:
                         return False
-            except (py.error.ENOENT, py.error.ENOTDIR):
+            except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
+                # EBUSY feels not entirely correct,
+                # but its kind of necessary since ENOMEDIUM
+                # is not accessible in python     
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
@@ -368,6 +371,5 @@
         else:
             name = str(path) # path.strpath # XXX svn?
             pattern = '*' + path.sep + pattern
-        from fnmatch import fnmatch
-        return fnmatch(name, pattern)
+        return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -157,14 +157,16 @@
         return str(self) < str(other)
 
     def samefile(self, other):
-        """ return True if 'other' references the same file as 'self'. """
-        if not iswin32:
-            return py.error.checked_call(
-                    os.path.samefile, str(self), str(other))
+        """ return True if 'other' references the same file as 'self'.
+        """
+        if not isinstance(other, py.path.local):
+            other = os.path.abspath(str(other))
         if self == other:
             return True
-        other = os.path.abspath(str(other))
-        return self == other
+        if iswin32:
+            return False # ther is no samefile
+        return py.error.checked_call(
+                os.path.samefile, str(self), str(other))
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -539,7 +541,11 @@
                 if self.basename != "__init__.py":
                     modfile = modfile[:-12]
 
-            if not self.samefile(modfile):
+            try:
+                issame = self.samefile(modfile)
+            except py.error.ENOENT:
+                issame = False
+            if not issame:
                 raise self.ImportMismatchError(modname, modfile, self)
             return mod
         else:
diff --git a/py/_path/svnurl.py b/py/_path/svnurl.py
--- a/py/_path/svnurl.py
+++ b/py/_path/svnurl.py
@@ -233,6 +233,8 @@
                 e = sys.exc_info()[1]
                 if e.err.find('non-existent in that revision') != -1:
                     raise py.error.ENOENT(self, e.err)
+                elif e.err.find("E200009:") != -1:
+                    raise py.error.ENOENT(self, e.err)
                 elif e.err.find('File not found') != -1:
                     raise py.error.ENOENT(self, e.err)
                 elif e.err.find('not part of a repository')!=-1:
diff --git a/py/_path/svnwc.py b/py/_path/svnwc.py
--- a/py/_path/svnwc.py
+++ b/py/_path/svnwc.py
@@ -482,10 +482,13 @@
         except py.process.cmdexec.Error:
             e = sys.exc_info()[1]
             strerr = e.err.lower()
-            if strerr.find('file not found') != -1:
+            if strerr.find('not found') != -1:
+                raise py.error.ENOENT(self)
+            elif strerr.find("E200009:") != -1:
                 raise py.error.ENOENT(self)
             if (strerr.find('file exists') != -1 or
                 strerr.find('file already exists') != -1 or
+                strerr.find('w150002:') != -1 or
                 strerr.find("can't create directory") != -1):
                 raise py.error.EEXIST(self)
             raise
@@ -593,7 +596,7 @@
         out = self._authsvn('lock').strip()
         if not out:
             # warning or error, raise exception
-            raise Exception(out[4:])
+            raise ValueError("unknown error in svn lock command")
 
     def unlock(self):
         """ unset a previously set lock """
@@ -1066,6 +1069,8 @@
                 modrev = '?'
                 author = '?'
                 date = ''
+            elif itemstatus == "replaced":
+                pass
             else:
                 #print entryel.toxml()
                 commitel = entryel.getElementsByTagName('commit')[0]
@@ -1148,7 +1153,11 @@
             raise  ValueError("Not a versioned resource")
             #raise ValueError, "Not a versioned resource %r" % path
         self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind']
-        self.rev = int(d['revision'])
+        try:
+            self.rev = int(d['revision'])
+        except KeyError:
+            self.rev = None
+
         self.path = py.path.local(d['path'])
         self.size = self.path.size()
         if 'lastchangedrev' in d:
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -52,7 +52,7 @@
     def unicode(self, indent=2):
         l = []
         SimpleUnicodeVisitor(l.append, indent).visit(self)
-        return "".join(l)
+        return u("").join(l)
 
     def __repr__(self):
         name = self.__class__.__name__
@@ -122,11 +122,13 @@
                 if visitmethod is not None:
                     break
             else:
-                visitmethod = self.object
+                visitmethod = self.__object
             self.cache[cls] = visitmethod
         visitmethod(node)
 
-    def object(self, obj):
+    # the default fallback handler is marked private
+    # to avoid clashes with the tag name object
+    def __object(self, obj):
         #self.write(obj)
         self.write(escape(unicode(obj)))
 
@@ -136,7 +138,8 @@
     def list(self, obj):
         assert id(obj) not in self.visited
         self.visited[id(obj)] = 1
-        map(self.visit, obj)
+        for elem in obj:
+            self.visit(elem)
 
     def Tag(self, tag):
         assert id(tag) not in self.visited
@@ -181,7 +184,11 @@
             value = getattr(attrs, name)
             if name.endswith('_'):
                 name = name[:-1]
-            return ' %s="%s"' % (name, escape(unicode(value)))
+            if isinstance(value, raw):
+                insert = value.uniobj
+            else:
+                insert = escape(unicode(value))
+            return ' %s="%s"' % (name, insert)
 
     def getstyle(self, tag):
         """ return attribute list suitable for styling. """
diff --git a/py/bin/_findpy.py b/py/bin/_findpy.py
deleted file mode 100644
--- a/py/bin/_findpy.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-#
-# find and import a version of 'py'
-#
-import sys
-import os
-from os.path import dirname as opd, exists, join, basename, abspath
-
-def searchpy(current):
-    while 1:
-        last = current
-        initpy = join(current, '__init__.py')
-        if not exists(initpy):
-            pydir = join(current, 'py')
-            # recognize py-package and ensure it is importable
-            if exists(pydir) and exists(join(pydir, '__init__.py')):
-                #for p in sys.path:
-                #    if p == current:
-                #        return True
-                if current != sys.path[0]:  # if we are already first, then ok
-                    sys.stderr.write("inserting into sys.path: %s\n" % current)
-                    sys.path.insert(0, current)
-                return True
-        current = opd(current)
-        if last == current:
-            return False
-
-if not searchpy(abspath(os.curdir)):
-    if not searchpy(opd(abspath(sys.argv[0]))):
-        if not searchpy(opd(__file__)):
-            pass # let's hope it is just on sys.path
-
-import py
-import pytest
-
-if __name__ == '__main__':
-    print ("py lib is at %s" % py.__file__)
diff --git a/py/bin/py.test b/py/bin/py.test
deleted file mode 100755
--- a/py/bin/py.test
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env python
-from _findpy import pytest
-raise SystemExit(pytest.main())
diff --git a/pypy/module/pyexpat/test/__init__.py b/pypy/module/pyexpat/test/__init__.py
new file mode 100644
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old -rf
+addopts = --assert=plain -rf
diff --git a/pypy/test_all.py b/pypy/test_all.py
old mode 100755
new mode 100644
--- a/pypy/test_all.py
+++ b/pypy/test_all.py
@@ -11,11 +11,12 @@
 """
 import sys, os
 
-if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
-    print >> sys.stderr, __doc__
-    sys.exit(2)
 
 if __name__ == '__main__':
+    if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
+        print >> sys.stderr, __doc__
+        sys.exit(2)
+
     import tool.autopath
     import pytest
     import pytest_cov
diff --git a/pypy/tool/jitlogparser/test/__init__.py b/pypy/tool/jitlogparser/test/__init__.py
new file mode 100644
diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -165,7 +165,10 @@
     def test_one(self): exec 'blow'
     """)
 
-    ev, = sorter.getreports("pytest_runtest_logreport")
+    reports = sorter.getreports("pytest_runtest_logreport")
+    setup, ev, teardown = reports
     assert ev.failed
+    assert setup.passed
+    assert teardown.passed
     assert 'NameError' in ev.longrepr.reprcrash.message
     assert 'blow' in ev.longrepr.reprcrash.message
diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -110,7 +110,10 @@
                  do_dry_run=False, timeout=None,
                  _win32=(sys.platform=='win32')):
     args = interp + test_driver
-    args += ['-p', 'resultlog', '--resultlog=%s' % logfname, test]
+    args += ['-p', 'resultlog',
+             '--resultlog=%s' % logfname,
+             '--junitxml=%s.junit' % logfname,
+             test]
 
     args = map(str, args)
     interp0 = args[0]
diff --git a/testrunner/scratchbox_runner.py b/testrunner/scratchbox_runner.py
--- a/testrunner/scratchbox_runner.py
+++ b/testrunner/scratchbox_runner.py
@@ -14,14 +14,14 @@
 def dry_run_scratchbox(args, cwd, out, timeout=None):
     return dry_run(args_for_scratchbox(cwd, args), cwd, out, timeout)
 
-import runner
-# XXX hack hack hack
-dry_run = runner.dry_run
-run = runner.run
+if __name__ == '__main__':
+    import runner
+    # XXX hack hack hack
+    dry_run = runner.dry_run
+    run = runner.run
 
-runner.dry_run = dry_run_scratchbox
-runner.run = run_scratchbox
+    runner.dry_run = dry_run_scratchbox
+    runner.run = run_scratchbox
 
-if __name__ == '__main__':
     import sys
     runner.main(sys.argv)
diff --git a/testrunner/test/conftest.py b/testrunner/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/testrunner/test/conftest.py
@@ -0,0 +1,6 @@
+
+def pytest_runtest_makereport(__multicall__, item):
+    report = __multicall__.execute()
+    if 'out' in item.funcargs:
+        report.sections.append(('out', item.funcargs['out'].read()))
+    return report
diff --git a/testrunner/test/test_runner.py b/testrunner/test/test_runner.py
--- a/testrunner/test/test_runner.py
+++ b/testrunner/test/test_runner.py
@@ -53,49 +53,44 @@
     assert not should_report_failure("F Def\n. Ghi\n. Jkl\n")
 
 
+
 class TestRunHelper(object):
+    def pytest_funcarg__out(self, request):
+        tmpdir = request.getfuncargvalue('tmpdir')
+        return tmpdir.ensure('out')
 
-    def setup_method(self, meth):
-        h, self.fn = tempfile.mkstemp()
-        os.close(h)
+    def test_run(self, out):
+        res = runner.run([sys.executable, "-c", "print 42"], '.', out)
+        assert res == 0
+        assert out.read() == "42\n"
 
-    def teardown_method(self, meth):
-        os.unlink(self.fn)
-
-    def test_run(self):
-        res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn))
-        assert res == 0
-        out = py.path.local(self.fn).read('r')
-        assert out == "42\n"
-
-    def test_error(self):
-        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', py.path.local(self.fn))
+    def test_error(self, out):
+        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', out)
         assert res == 3
 
-    def test_signal(self):
+    def test_signal(self, out):
         if sys.platform == 'win32':
             py.test.skip("no death by signal on windows")
-        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', py.path.local(self.fn))
+        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', out)
         assert res == -9
 
-    def test_timeout(self):
-        res = runner.run([sys.executable, "-c", "while True: pass"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout(self, out):
+        res = runner.run([sys.executable, "-c", "while True: pass"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_lock(self):
-        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_lock(self, out):
+        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_syscall(self):
-        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_syscall(self, out):
+        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', out, timeout=3)
         assert res == -999        
 
-    def test_timeout_success(self):
+    def test_timeout_success(self, out):
         res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn), timeout=2)
+                         out, timeout=2)
         assert res == 0
-        out = py.path.local(self.fn).read('r')
+        out = out.read()
         assert out == "42\n"        
 
 
@@ -122,7 +117,10 @@
 
         expected = ['INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
+
                     'test_one']
 
         assert self.called == (expected, '/wd', 'out', 'secs')        
@@ -138,9 +136,11 @@
 
         expected = ['/wd' + os.sep + './INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
                     'test_one']
-
+        assert self.called[0] == expected
         assert self.called == (expected, '/wd', 'out', 'secs')        
         assert res == 0
 
@@ -251,7 +251,7 @@
         assert '\n' in log        
         log_lines = log.splitlines()
 
-        assert log_lines[0] == ". test_normal/test_example.py:test_one"
+        assert ". test_normal/test_example.py::test_one" in log_lines
         nfailures = 0
         noutcomes = 0
         for line in log_lines:

From noreply at buildbot.pypy.org  Mon Apr  9 20:34:19 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon,  9 Apr 2012 20:34:19 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: yet another necessary fix (this
 is getting obscure)
Message-ID: <20120409183419.A1F6F46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r180:9946cdcf97a2
Date: 2012-04-09 20:34 +0200
http://bitbucket.org/pypy/benchmarks/changeset/9946cdcf97a2/

Log:	yet another necessary fix (this is getting obscure)

diff --git a/lib/pypy/pypy/interpreter/astcompiler/assemble.py b/lib/pypy/pypy/interpreter/astcompiler/assemble.py
--- a/lib/pypy/pypy/interpreter/astcompiler/assemble.py
+++ b/lib/pypy/pypy/interpreter/astcompiler/assemble.py
@@ -610,6 +610,8 @@
     ops.JUMP_IF_FALSE_OR_POP : 0,
     ops.POP_JUMP_IF_TRUE : -1,
     ops.POP_JUMP_IF_FALSE : -1,
+
+    ops.BUILD_LIST_FROM_ARG: 1,
 }
 
 

From noreply at buildbot.pypy.org  Mon Apr  9 20:54:37 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon,  9 Apr 2012 20:54:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix self translation
Message-ID: <20120409185437.118DE46E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54270:41c799d11717
Date: 2012-04-09 20:53 +0200
http://bitbucket.org/pypy/pypy/changeset/41c799d11717/

Log:	Fix self translation

diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -434,6 +434,13 @@
         self.lastblock = block
         self.pushvalue(w_result)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # This opcode was added with pypy-1.8.  Here is a simpler
+        # version, enough for annotation.
+        last_val = self.popvalue()
+        self.pushvalue(self.space.newlist([]))
+        self.pushvalue(last_val)
+
     # XXX Unimplemented 2.7 opcodes ----------------
 
     # Set literals, set comprehensions

From noreply at buildbot.pypy.org  Tue Apr 10 00:02:15 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 10 Apr 2012 00:02:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: a failing test for using
 specialize.memo and specialize.arg together
Message-ID: <20120409220215.07FA146E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r54271:09e76fd5aa32
Date: 2012-04-09 18:01 -0400
http://bitbucket.org/pypy/pypy/changeset/09e76fd5aa32/

Log:	a failing test for using specialize.memo and specialize.arg together

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -483,7 +483,7 @@
                 return a_str.strip(' ')
             elif n == 1:
                 return a_str.rstrip(' ')
-            else: 
+            else:
                 return a_str.lstrip(' ')
         s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
         assert s.no_nul
@@ -3737,6 +3737,25 @@
         s = a.build_types(f, [int])
         assert s.listdef.listitem.range_step == 0
 
+    def test_specialize_arg_memo(self):
+        @objectmodel.specialize.memo()
+        def g(n):
+            return n
+        @objectmodel.specialize.arg(0)
+        def f(i):
+            return g(i)
+        def main(i):
+            if i == 2:
+                return f(i)
+            elif i == 3:
+                return f(i)
+            else:
+                raise NotImplementedError
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(f, [int])
+        assert isinstance(s, SomeInteger)
+
 
 def g(n):
     return [0,1,2,n]

From noreply at buildbot.pypy.org  Tue Apr 10 00:36:51 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 10 Apr 2012 00:36:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: correc the test, still fails (amaury)
Message-ID: <20120409223651.A8F7346E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r54272:533a81aa5113
Date: 2012-04-09 18:36 -0400
http://bitbucket.org/pypy/pypy/changeset/533a81aa5113/

Log:	correc the test, still fails (amaury)

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3753,7 +3753,7 @@
                 raise NotImplementedError
 
         a = self.RPythonAnnotator()
-        s = a.build_types(f, [int])
+        s = a.build_types(main, [int])
         assert isinstance(s, SomeInteger)
 
 

From noreply at buildbot.pypy.org  Tue Apr 10 01:22:21 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 10 Apr 2012 01:22:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix nameerror
Message-ID: <20120409232221.16B6346E0EF@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r54273:124c48df1aab
Date: 2012-04-09 19:22 -0400
http://bitbucket.org/pypy/pypy/changeset/124c48df1aab/

Log:	fix nameerror

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3754,7 +3754,7 @@
 
         a = self.RPythonAnnotator()
         s = a.build_types(main, [int])
-        assert isinstance(s, SomeInteger)
+        assert isinstance(s, annmodel.SomeInteger)
 
 
 def g(n):

From notifications-noreply at bitbucket.org  Tue Apr 10 16:22:51 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 10 Apr 2012 14:22:51 -0000
Subject: [pypy-commit] Notification: benchmarks-pypy-phd
Message-ID: <20120410142251.8915.48112@bitbucket12.managed.contegix.com>

You have received a notification from Carl Friedrich Bolz.

Hi, I forked benchmarks. My fork is at https://bitbucket.org/cfbolz/benchmarks-pypy-phd.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Tue Apr 10 17:57:15 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 10 Apr 2012 17:57:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: re-add (again) these files,
 they are needed by buildbot (I think). See also e12baeb2c897
Message-ID: <20120410155715.6FF9582111@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54274:321cc342e9f6
Date: 2012-04-10 17:56 +0200
http://bitbucket.org/pypy/pypy/changeset/321cc342e9f6/

Log:	re-add (again) these files, they are needed by buildbot (I think).
	See also e12baeb2c897

diff --git a/py/bin/_findpy.py b/py/bin/_findpy.py
new file mode 100644
--- /dev/null
+++ b/py/bin/_findpy.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+#
+# find and import a version of 'py'
+#
+import sys
+import os
+from os.path import dirname as opd, exists, join, basename, abspath
+
+def searchpy(current):
+    while 1:
+        last = current
+        initpy = join(current, '__init__.py')
+        if not exists(initpy):
+            pydir = join(current, 'py')
+            # recognize py-package and ensure it is importable
+            if exists(pydir) and exists(join(pydir, '__init__.py')):
+                #for p in sys.path:
+                #    if p == current:
+                #        return True
+                if current != sys.path[0]:  # if we are already first, then ok
+                    sys.stderr.write("inserting into sys.path: %s\n" % current)
+                    sys.path.insert(0, current)
+                return True
+        current = opd(current)
+        if last == current:
+            return False
+
+if not searchpy(abspath(os.curdir)):
+    if not searchpy(opd(abspath(sys.argv[0]))):
+        if not searchpy(opd(__file__)):
+            pass # let's hope it is just on sys.path
+
+import py
+import pytest
+
+if __name__ == '__main__':
+    print ("py lib is at %s" % py.__file__)
diff --git a/py/bin/py.test b/py/bin/py.test
new file mode 100755
--- /dev/null
+++ b/py/bin/py.test
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from _findpy import pytest
+raise SystemExit(pytest.main())

From noreply at buildbot.pypy.org  Tue Apr 10 18:16:38 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 10 Apr 2012 18:16:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: add a point to how-to-release
Message-ID: <20120410161638.5371082111@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54275:658930dda580
Date: 2012-04-10 18:16 +0200
http://bitbucket.org/pypy/pypy/changeset/658930dda580/

Log:	add a point to how-to-release

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -37,6 +37,9 @@
     no JIT: windows, linux, os/x
     sandbox: linux, os/x
 
+* create source tarball/zip. Make sure that the directory inside them is
+  pypy-1.x and not pypy-pypy-revision
+
 * write release announcement pypy/doc/release-x.y(.z).txt
   the release announcement should contain a direct link to the download page
 * update pypy.org (under extradoc/pypy.org), rebuild and commit

From noreply at buildbot.pypy.org  Tue Apr 10 20:22:57 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Apr 2012 20:22:57 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: regen
Message-ID: <20120410182257.7146682F4F@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r353:64eeffac819c
Date: 2012-04-10 20:22 +0200
http://bitbucket.org/pypy/pypy.org/changeset/64eeffac819c/

Log:	regen

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -206,12 +206,14 @@
 1af8ee722721e9f5fd06b61af530ecb3  pypy-1.8-win32.zip
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
+661be06978fdc907d84f0ee1f1228c8b  release-1.8.tar.bz2 (source)
 a6bb7b277d5186385fd09b71ec4e35c9e93b380d  pypy-1.8-linux64.tar.bz2
 089f4269a6079da2eabdeabd614f668f56c4121a  pypy-1.8-linux.tar.bz2
 15b99f780b9714e3ebd82b2e41577afab232d148  pypy-1.8-osx64.tar.bz2
 77a565b1cfa4874a0079c17edd1b458b20e67bfd  pypy-1.8-win32.zip
 895aaf7bba5787dd30adda5cc0e0e7fc297c0ca7  pypy-1.8-sandbox-linux64.tar.bz2
 be94460bed8b2682880495435c309b6611ae2c31  pypy-1.8-sandbox-linux.tar.bz2
+4ff684619671d6076879eff88343184d7656c699  release-1.8.tar.bz2 (source)
 </pre>
 </div>
 </div>

From noreply at buildbot.pypy.org  Tue Apr 10 20:22:56 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 10 Apr 2012 20:22:56 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: As usual, I don't get it,
 but it seems to make people more happy if I
Message-ID: <20120410182256.4202B82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r352:5f5ac20464be
Date: 2012-04-10 20:22 +0200
http://bitbucket.org/pypy/pypy.org/changeset/5f5ac20464be/

Log:	As usual, I don't get it, but it seems to make people more happy if
	I download myself this file from the URL indicated, compute the
	checksums, and post it here.

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -214,6 +214,7 @@
    1af8ee722721e9f5fd06b61af530ecb3  pypy-1.8-win32.zip
    2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
    009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
+   661be06978fdc907d84f0ee1f1228c8b  release-1.8.tar.bz2 (source)
 
    a6bb7b277d5186385fd09b71ec4e35c9e93b380d  pypy-1.8-linux64.tar.bz2
    089f4269a6079da2eabdeabd614f668f56c4121a  pypy-1.8-linux.tar.bz2
@@ -221,3 +222,4 @@
    77a565b1cfa4874a0079c17edd1b458b20e67bfd  pypy-1.8-win32.zip
    895aaf7bba5787dd30adda5cc0e0e7fc297c0ca7  pypy-1.8-sandbox-linux64.tar.bz2
    be94460bed8b2682880495435c309b6611ae2c31  pypy-1.8-sandbox-linux.tar.bz2
+   4ff684619671d6076879eff88343184d7656c699  release-1.8.tar.bz2 (source)

From noreply at buildbot.pypy.org  Tue Apr 10 21:59:20 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Apr 2012 21:59:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: cpyext: Add PyUnicode_Count
Message-ID: <20120410195920.2552382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54276:b1e988d5a3f5
Date: 2012-04-10 21:58 +0200
http://bitbucket.org/pypy/pypy/changeset/b1e988d5a3f5/

Log:	cpyext: Add PyUnicode_Count

diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2300,16 +2300,6 @@
     systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
-def PyUnicode_Count(space, str, substr, start, end):
-    """Return the number of non-overlapping occurrences of substr in
-    str[start:end].  Return -1 if an error occurred.
-
-    This function returned an int type and used an int
-    type for start and end. This might require changes in your code for
-    properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
 def PyUnicode_RichCompare(space, left, right, op):
     """Rich compare two unicode strings and return one of the following:
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -457,3 +457,9 @@
         assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+
+    def test_count(self, space, api):
+        w_str = space.wrap(u"abcabdab")
+        assert api.PyUnicode_Count(w_str, space.wrap("ab"), 0, -1) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap("ab"), 0, 2) == 1
+        assert api.PyUnicode_Count(w_str, space.wrap("ab"), -5, 30) == 2
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -598,3 +598,10 @@
     else:
         return stringtype.stringendswith(str, substr, start, end)
 
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
+def PyUnicode_Count(space, w_str, w_substr, start, end):
+    """Return the number of non-overlapping occurrences of substr in
+    str[start:end].  Return -1 if an error occurred."""
+    w_count = space.call_method(w_str, "count", w_substr,
+                                space.wrap(start), space.wrap(end))
+    return space.int_w(w_count)

From noreply at buildbot.pypy.org  Tue Apr 10 23:24:53 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 10 Apr 2012 23:24:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: We don't do that. Those are created
 automatically from tags. I'm unsure how
Message-ID: <20120410212453.C24F782F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54277:88c488265ad9
Date: 2012-04-10 23:24 +0200
http://bitbucket.org/pypy/pypy/changeset/88c488265ad9/

Log:	We don't do that. Those are created automatically from tags. I'm
	unsure how to make sure they're in a well-named directories.

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -37,9 +37,6 @@
     no JIT: windows, linux, os/x
     sandbox: linux, os/x
 
-* create source tarball/zip. Make sure that the directory inside them is
-  pypy-1.x and not pypy-pypy-revision
-
 * write release announcement pypy/doc/release-x.y(.z).txt
   the release announcement should contain a direct link to the download page
 * update pypy.org (under extradoc/pypy.org), rebuild and commit

From noreply at buildbot.pypy.org  Tue Apr 10 23:55:14 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Apr 2012 23:55:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: cpyext: Add PyUnicode_Find
Message-ID: <20120410215514.C247B82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54278:10ee763df5ba
Date: 2012-04-10 22:42 +0200
http://bitbucket.org/pypy/pypy/changeset/10ee763df5ba/

Log:	cpyext: Add PyUnicode_Find

diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2287,19 +2287,6 @@
     use the default error handling."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t, error=-2)
-def PyUnicode_Find(space, str, substr, start, end, direction):
-    """Return the first position of substr in str*[*start:end] using the given
-    direction (direction == 1 means to do a forward search, direction == -1 a
-    backward search).  The return value is the index of the first match; a value of
-    -1 indicates that no match was found, and -2 indicates that an error
-    occurred and an exception has been set.
-
-    This function used an int type for start and end. This
-    might require changes in your code for properly supporting 64-bit
-    systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
 def PyUnicode_RichCompare(space, left, right, op):
     """Rich compare two unicode strings and return one of the following:
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -460,6 +460,15 @@
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")
-        assert api.PyUnicode_Count(w_str, space.wrap("ab"), 0, -1) == 2
-        assert api.PyUnicode_Count(w_str, space.wrap("ab"), 0, 2) == 1
-        assert api.PyUnicode_Count(w_str, space.wrap("ab"), -5, 30) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, -1) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, 2) == 1
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), -5, 30) == 2
+
+    def test_find(self, space, api):
+        w_str = space.wrap(u"abcabcd")
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, 1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, 1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -605,3 +605,20 @@
     w_count = space.call_method(w_str, "count", w_substr,
                                 space.wrap(start), space.wrap(end))
     return space.int_w(w_count)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
+             Py_ssize_t, error=-2)
+def PyUnicode_Find(space, w_str, w_substr, start, end, direction):
+    """Return the first position of substr in str*[*start:end] using
+    the given direction (direction == 1 means to do a forward search,
+    direction == -1 a backward search).  The return value is the index
+    of the first match; a value of -1 indicates that no match was
+    found, and -2 indicates that an error occurred and an exception
+    has been set."""
+    if rffi.cast(lltype.Signed, direction) > 0:
+        w_pos = space.call_method(w_str, "find", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    else:
+        w_pos = space.call_method(w_str, "rfind", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    return space.int_w(w_pos)

From noreply at buildbot.pypy.org  Tue Apr 10 23:55:16 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 10 Apr 2012 23:55:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: cpyext: implement PyUnicode_Split and
 PyUnicode_Splitlines
Message-ID: <20120410215516.1213782F4E@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54279:859f1579f2bd
Date: 2012-04-10 23:19 +0200
http://bitbucket.org/pypy/pypy/changeset/859f1579f2bd/

Log:	cpyext: implement PyUnicode_Split and PyUnicode_Splitlines

diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2253,24 +2253,6 @@
     """Concat two strings giving a new Unicode string."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
-def PyUnicode_Split(space, s, sep, maxsplit):
-    """Split a string giving a list of Unicode strings.  If sep is NULL, splitting
-    will be done at all whitespace substrings.  Otherwise, splits occur at the given
-    separator.  At most maxsplit splits will be done.  If negative, no limit is
-    set.  Separators are not included in the resulting list.
-
-    This function used an int type for maxsplit. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyUnicode_Splitlines(space, s, keepend):
-    """Split a Unicode string at line breaks, returning a list of Unicode strings.
-    CRLF is considered to be one line break.  If keepend is 0, the Line break
-    characters are not included in the resulting strings."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.CCHARP], PyObject)
 def PyUnicode_Translate(space, str, table, errors):
     """Translate a string by applying a character mapping table to it and return the
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -472,3 +472,14 @@
         assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
         assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
         assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
+
+    def test_split(self, space, api):
+        w_str = space.wrap(u"a\nb\nc\nd")
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
+        assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 0)))
+        assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 1)))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -622,3 +622,20 @@
         w_pos = space.call_method(w_str, "rfind", w_substr,
                                   space.wrap(start), space.wrap(end))
     return space.int_w(w_pos)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
+def PyUnicode_Split(space, w_str, w_sep, maxsplit):
+    """Split a string giving a list of Unicode strings.  If sep is
+    NULL, splitting will be done at all whitespace substrings.
+    Otherwise, splits occur at the given separator.  At most maxsplit
+    splits will be done.  If negative, no limit is set.  Separators
+    are not included in the resulting list."""
+    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyUnicode_Splitlines(space, w_str, keepend):
+    """Split a Unicode string at line breaks, returning a list of
+    Unicode strings.  CRLF is considered to be one line break.  If
+    keepend is 0, the Line break characters are not included in the
+    resulting strings."""
+    return space.call_method(w_str, "splitlines", space.wrap(keepend))

From noreply at buildbot.pypy.org  Wed Apr 11 11:19:15 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 11:19:15 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix include order
Message-ID: <20120411091915.6637582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54280:2921217e4b74
Date: 2012-04-09 20:05 +0300
http://bitbucket.org/pypy/pypy/changeset/2921217e4b74/

Log:	fix include order

diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ["windows.h"],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(

From noreply at buildbot.pypy.org  Wed Apr 11 11:19:16 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 11:19:16 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: remove/skip tests that check the
 sign of float('nan')
Message-ID: <20120411091916.B160F82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54281:1b052e5c79b5
Date: 2012-04-10 14:25 +0300
http://bitbucket.org/pypy/pypy/changeset/1b052e5c79b5/

Log:	remove/skip tests that check the sign of float('nan')

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -233,12 +233,15 @@
         assert a[1] == 0
 
     def test_signbit(self):
-        from _numpypy import signbit, copysign
+        from _numpypy import signbit
 
-        assert (signbit([0, 0.0, 1, 1.0, float('inf'), float('nan')]) ==
-            [False, False, False, False, False, False]).all()
-        assert (signbit([-0, -0.0, -1, -1.0, float('-inf'), -float('nan'), float('-nan')]) ==
-            [False,  True,  True,  True,  True,  True, True]).all()
+        assert (signbit([0, 0.0, 1, 1.0, float('inf')]) ==
+            [False, False, False, False, False]).all()
+        assert (signbit([-0, -0.0, -1, -1.0, float('-inf')]) ==
+            [False,  True,  True,  True,  True]).all()
+        skip('sign of nan is non-determinant')
+        assert (signbit([float('nan'), float('-nan'), -float('nan')]) ==
+            [False, True, True]).all()    
 
     def test_reciporocal(self):
         from _numpypy import array, reciprocal

From noreply at buildbot.pypy.org  Wed Apr 11 11:19:18 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 11:19:18 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: close file opened in test
Message-ID: <20120411091918.0DD7C82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54282:ac977b768615
Date: 2012-04-11 12:17 +0300
http://bitbucket.org/pypy/pypy/changeset/ac977b768615/

Log:	close file opened in test

diff --git a/lib-python/2.7/test/test_os.py b/lib-python/modified-2.7/test/test_os.py
rename from lib-python/2.7/test/test_os.py
rename to lib-python/modified-2.7/test/test_os.py
--- a/lib-python/2.7/test/test_os.py
+++ b/lib-python/modified-2.7/test/test_os.py
@@ -74,7 +74,8 @@
         self.assertFalse(os.path.exists(name),
                     "file already exists for temporary file")
         # make sure we can create the file
-        open(name, "w")
+        fid = open(name, "w")
+        fid.close()
         self.files.append(name)
 
     def test_tempnam(self):

From noreply at buildbot.pypy.org  Wed Apr 11 13:24:28 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 13:24:28 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: fix close() and tests for file
	closing
Message-ID: <20120411112428.31DCC82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54283:2a8e4f56269f
Date: 2012-04-11 13:26 +0300
http://bitbucket.org/pypy/pypy/changeset/2a8e4f56269f/

Log:	fix close() and tests for file closing

diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py
deleted file mode 100644
--- a/lib-python/2.7/mailbox.py
+++ /dev/null
@@ -1,2171 +0,0 @@
-#! /usr/bin/env python
-
-"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
-
-# Notes for authors of new mailbox subclasses:
-#
-# Remember to fsync() changes to disk before closing a modified file
-# or returning from a flush() method.  See functions _sync_flush() and
-# _sync_close().
-
-import sys
-import os
-import time
-import calendar
-import socket
-import errno
-import copy
-import email
-import email.message
-import email.generator
-import StringIO
-try:
-    if sys.platform == 'os2emx':
-        # OS/2 EMX fcntl() not adequate
-        raise ImportError
-    import fcntl
-except ImportError:
-    fcntl = None
-
-import warnings
-with warnings.catch_warnings():
-    if sys.py3kwarning:
-        warnings.filterwarnings("ignore", ".*rfc822 has been removed",
-                                DeprecationWarning)
-    import rfc822
-
-__all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
-            'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
-            'BabylMessage', 'MMDFMessage', 'UnixMailbox',
-            'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
-
-class Mailbox:
-    """A group of messages in a particular place."""
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize a Mailbox instance."""
-        self._path = os.path.abspath(os.path.expanduser(path))
-        self._factory = factory
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def __delitem__(self, key):
-        self.remove(key)
-
-    def discard(self, key):
-        """If the keyed message exists, remove it."""
-        try:
-            self.remove(key)
-        except KeyError:
-            pass
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def get(self, key, default=None):
-        """Return the keyed message, or default if it doesn't exist."""
-        try:
-            return self.__getitem__(key)
-        except KeyError:
-            return default
-
-    def __getitem__(self, key):
-        """Return the keyed message; raise KeyError if it doesn't exist."""
-        if not self._factory:
-            return self.get_message(key)
-        else:
-            return self._factory(self.get_file(key))
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def get_string(self, key):
-        """Return a string representation or raise a KeyError."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def get_file(self, key):
-        """Return a file-like representation or raise a KeyError."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def iterkeys(self):
-        """Return an iterator over keys."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def keys(self):
-        """Return a list of keys."""
-        return list(self.iterkeys())
-
-    def itervalues(self):
-        """Return an iterator over all messages."""
-        for key in self.iterkeys():
-            try:
-                value = self[key]
-            except KeyError:
-                continue
-            yield value
-
-    def __iter__(self):
-        return self.itervalues()
-
-    def values(self):
-        """Return a list of messages. Memory intensive."""
-        return list(self.itervalues())
-
-    def iteritems(self):
-        """Return an iterator over (key, message) tuples."""
-        for key in self.iterkeys():
-            try:
-                value = self[key]
-            except KeyError:
-                continue
-            yield (key, value)
-
-    def items(self):
-        """Return a list of (key, message) tuples. Memory intensive."""
-        return list(self.iteritems())
-
-    def has_key(self, key):
-        """Return True if the keyed message exists, False otherwise."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def __contains__(self, key):
-        return self.has_key(key)
-
-    def __len__(self):
-        """Return a count of messages in the mailbox."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def clear(self):
-        """Delete all messages."""
-        for key in self.iterkeys():
-            self.discard(key)
-
-    def pop(self, key, default=None):
-        """Delete the keyed message and return it, or default."""
-        try:
-            result = self[key]
-        except KeyError:
-            return default
-        self.discard(key)
-        return result
-
-    def popitem(self):
-        """Delete an arbitrary (key, message) pair and return it."""
-        for key in self.iterkeys():
-            return (key, self.pop(key))     # This is only run once.
-        else:
-            raise KeyError('No messages in mailbox')
-
-    def update(self, arg=None):
-        """Change the messages that correspond to certain keys."""
-        if hasattr(arg, 'iteritems'):
-            source = arg.iteritems()
-        elif hasattr(arg, 'items'):
-            source = arg.items()
-        else:
-            source = arg
-        bad_key = False
-        for key, message in source:
-            try:
-                self[key] = message
-            except KeyError:
-                bad_key = True
-        if bad_key:
-            raise KeyError('No message with key(s)')
-
-    def flush(self):
-        """Write any pending changes to the disk."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def lock(self):
-        """Lock the mailbox."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def unlock(self):
-        """Unlock the mailbox if it is locked."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def close(self):
-        """Flush and close the mailbox."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def _dump_message(self, message, target, mangle_from_=False):
-        # Most files are opened in binary mode to allow predictable seeking.
-        # To get native line endings on disk, the user-friendly \n line endings
-        # used in strings and by email.Message are translated here.
-        """Dump message contents to target file."""
-        if isinstance(message, email.message.Message):
-            buffer = StringIO.StringIO()
-            gen = email.generator.Generator(buffer, mangle_from_, 0)
-            gen.flatten(message)
-            buffer.seek(0)
-            target.write(buffer.read().replace('\n', os.linesep))
-        elif isinstance(message, str):
-            if mangle_from_:
-                message = message.replace('\nFrom ', '\n>From ')
-            message = message.replace('\n', os.linesep)
-            target.write(message)
-        elif hasattr(message, 'read'):
-            while True:
-                line = message.readline()
-                if line == '':
-                    break
-                if mangle_from_ and line.startswith('From '):
-                    line = '>From ' + line[5:]
-                line = line.replace('\n', os.linesep)
-                target.write(line)
-        else:
-            raise TypeError('Invalid message type: %s' % type(message))
-
-
-class Maildir(Mailbox):
-    """A qmail-style Maildir mailbox."""
-
-    colon = ':'
-
-    def __init__(self, dirname, factory=rfc822.Message, create=True):
-        """Initialize a Maildir instance."""
-        Mailbox.__init__(self, dirname, factory, create)
-        self._paths = {
-            'tmp': os.path.join(self._path, 'tmp'),
-            'new': os.path.join(self._path, 'new'),
-            'cur': os.path.join(self._path, 'cur'),
-            }
-        if not os.path.exists(self._path):
-            if create:
-                os.mkdir(self._path, 0700)
-                for path in self._paths.values():
-                    os.mkdir(path, 0o700)
-            else:
-                raise NoSuchMailboxError(self._path)
-        self._toc = {}
-        self._toc_mtimes = {}
-        for subdir in ('cur', 'new'):
-            self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir])
-        self._last_read = time.time()  # Records last time we read cur/new
-        self._skewfactor = 0.1         # Adjust if os/fs clocks are skewing
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        tmp_file = self._create_tmp()
-        try:
-            self._dump_message(message, tmp_file)
-        except BaseException:
-            tmp_file.close()
-            os.remove(tmp_file.name)
-            raise
-        _sync_close(tmp_file)
-        if isinstance(message, MaildirMessage):
-            subdir = message.get_subdir()
-            suffix = self.colon + message.get_info()
-            if suffix == self.colon:
-                suffix = ''
-        else:
-            subdir = 'new'
-            suffix = ''
-        uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
-        dest = os.path.join(self._path, subdir, uniq + suffix)
-        try:
-            if hasattr(os, 'link'):
-                os.link(tmp_file.name, dest)
-                os.remove(tmp_file.name)
-            else:
-                os.rename(tmp_file.name, dest)
-        except OSError, e:
-            os.remove(tmp_file.name)
-            if e.errno == errno.EEXIST:
-                raise ExternalClashError('Name clash with existing message: %s'
-                                         % dest)
-            else:
-                raise
-        if isinstance(message, MaildirMessage):
-            os.utime(dest, (os.path.getatime(dest), message.get_date()))
-        return uniq
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        os.remove(os.path.join(self._path, self._lookup(key)))
-
-    def discard(self, key):
-        """If the keyed message exists, remove it."""
-        # This overrides an inapplicable implementation in the superclass.
-        try:
-            self.remove(key)
-        except KeyError:
-            pass
-        except OSError, e:
-            if e.errno != errno.ENOENT:
-                raise
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        old_subpath = self._lookup(key)
-        temp_key = self.add(message)
-        temp_subpath = self._lookup(temp_key)
-        if isinstance(message, MaildirMessage):
-            # temp's subdir and suffix were specified by message.
-            dominant_subpath = temp_subpath
-        else:
-            # temp's subdir and suffix were defaults from add().
-            dominant_subpath = old_subpath
-        subdir = os.path.dirname(dominant_subpath)
-        if self.colon in dominant_subpath:
-            suffix = self.colon + dominant_subpath.split(self.colon)[-1]
-        else:
-            suffix = ''
-        self.discard(key)
-        new_path = os.path.join(self._path, subdir, key + suffix)
-        os.rename(os.path.join(self._path, temp_subpath), new_path)
-        if isinstance(message, MaildirMessage):
-            os.utime(new_path, (os.path.getatime(new_path),
-                                message.get_date()))
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        subpath = self._lookup(key)
-        f = open(os.path.join(self._path, subpath), 'r')
-        try:
-            if self._factory:
-                msg = self._factory(f)
-            else:
-                msg = MaildirMessage(f)
-        finally:
-            f.close()
-        subdir, name = os.path.split(subpath)
-        msg.set_subdir(subdir)
-        if self.colon in name:
-            msg.set_info(name.split(self.colon)[-1])
-        msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
-        return msg
-
-    def get_string(self, key):
-        """Return a string representation or raise a KeyError."""
-        f = open(os.path.join(self._path, self._lookup(key)), 'r')
-        try:
-            return f.read()
-        finally:
-            f.close()
-
-    def get_file(self, key):
-        """Return a file-like representation or raise a KeyError."""
-        f = open(os.path.join(self._path, self._lookup(key)), 'rb')
-        return _ProxyFile(f)
-
-    def iterkeys(self):
-        """Return an iterator over keys."""
-        self._refresh()
-        for key in self._toc:
-            try:
-                self._lookup(key)
-            except KeyError:
-                continue
-            yield key
-
-    def has_key(self, key):
-        """Return True if the keyed message exists, False otherwise."""
-        self._refresh()
-        return key in self._toc
-
-    def __len__(self):
-        """Return a count of messages in the mailbox."""
-        self._refresh()
-        return len(self._toc)
-
-    def flush(self):
-        """Write any pending changes to disk."""
-        # Maildir changes are always written immediately, so there's nothing
-        # to do.
-        pass
-
-    def lock(self):
-        """Lock the mailbox."""
-        return
-
-    def unlock(self):
-        """Unlock the mailbox if it is locked."""
-        return
-
-    def close(self):
-        """Flush and close the mailbox."""
-        return
-
-    def list_folders(self):
-        """Return a list of folder names."""
-        result = []
-        for entry in os.listdir(self._path):
-            if len(entry) > 1 and entry[0] == '.' and \
-               os.path.isdir(os.path.join(self._path, entry)):
-                result.append(entry[1:])
-        return result
-
-    def get_folder(self, folder):
-        """Return a Maildir instance for the named folder."""
-        return Maildir(os.path.join(self._path, '.' + folder),
-                       factory=self._factory,
-                       create=False)
-
-    def add_folder(self, folder):
-        """Create a folder and return a Maildir instance representing it."""
-        path = os.path.join(self._path, '.' + folder)
-        result = Maildir(path, factory=self._factory)
-        maildirfolder_path = os.path.join(path, 'maildirfolder')
-        if not os.path.exists(maildirfolder_path):
-            os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
-                0666))
-        return result
-
-    def remove_folder(self, folder):
-        """Delete the named folder, which must be empty."""
-        path = os.path.join(self._path, '.' + folder)
-        for entry in os.listdir(os.path.join(path, 'new')) + \
-                     os.listdir(os.path.join(path, 'cur')):
-            if len(entry) < 1 or entry[0] != '.':
-                raise NotEmptyError('Folder contains message(s): %s' % folder)
-        for entry in os.listdir(path):
-            if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
-               os.path.isdir(os.path.join(path, entry)):
-                raise NotEmptyError("Folder contains subdirectory '%s': %s" %
-                                    (folder, entry))
-        for root, dirs, files in os.walk(path, topdown=False):
-            for entry in files:
-                os.remove(os.path.join(root, entry))
-            for entry in dirs:
-                os.rmdir(os.path.join(root, entry))
-        os.rmdir(path)
-
-    def clean(self):
-        """Delete old files in "tmp"."""
-        now = time.time()
-        for entry in os.listdir(os.path.join(self._path, 'tmp')):
-            path = os.path.join(self._path, 'tmp', entry)
-            if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
-                os.remove(path)
-
-    _count = 1  # This is used to generate unique file names.
-
-    def _create_tmp(self):
-        """Create a file in the tmp subdirectory and open and return it."""
-        now = time.time()
-        hostname = socket.gethostname()
-        if '/' in hostname:
-            hostname = hostname.replace('/', r'\057')
-        if ':' in hostname:
-            hostname = hostname.replace(':', r'\072')
-        uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
-                                    Maildir._count, hostname)
-        path = os.path.join(self._path, 'tmp', uniq)
-        try:
-            os.stat(path)
-        except OSError, e:
-            if e.errno == errno.ENOENT:
-                Maildir._count += 1
-                try:
-                    return _create_carefully(path)
-                except OSError, e:
-                    if e.errno != errno.EEXIST:
-                        raise
-            else:
-                raise
-
-        # Fall through to here if stat succeeded or open raised EEXIST.
-        raise ExternalClashError('Name clash prevented file creation: %s' %
-                                 path)
-
-    def _refresh(self):
-        """Update table of contents mapping."""
-        # If it has been less than two seconds since the last _refresh() call,
-        # we have to unconditionally re-read the mailbox just in case it has
-        # been modified, because os.path.mtime() has a 2 sec resolution in the
-        # most common worst case (FAT) and a 1 sec resolution typically.  This
-        # results in a few unnecessary re-reads when _refresh() is called
-        # multiple times in that interval, but once the clock ticks over, we
-        # will only re-read as needed.  Because the filesystem might be being
-        # served by an independent system with its own clock, we record and
-        # compare with the mtimes from the filesystem.  Because the other
-        # system's clock might be skewing relative to our clock, we add an
-        # extra delta to our wait.  The default is one tenth second, but is an
-        # instance variable and so can be adjusted if dealing with a
-        # particularly skewed or irregular system.
-        if time.time() - self._last_read > 2 + self._skewfactor:
-            refresh = False
-            for subdir in self._toc_mtimes:
-                mtime = os.path.getmtime(self._paths[subdir])
-                if mtime > self._toc_mtimes[subdir]:
-                    refresh = True
-                self._toc_mtimes[subdir] = mtime
-            if not refresh:
-                return
-        # Refresh toc
-        self._toc = {}
-        for subdir in self._toc_mtimes:
-            path = self._paths[subdir]
-            for entry in os.listdir(path):
-                p = os.path.join(path, entry)
-                if os.path.isdir(p):
-                    continue
-                uniq = entry.split(self.colon)[0]
-                self._toc[uniq] = os.path.join(subdir, entry)
-        self._last_read = time.time()
-
-    def _lookup(self, key):
-        """Use TOC to return subpath for given key, or raise a KeyError."""
-        try:
-            if os.path.exists(os.path.join(self._path, self._toc[key])):
-                return self._toc[key]
-        except KeyError:
-            pass
-        self._refresh()
-        try:
-            return self._toc[key]
-        except KeyError:
-            raise KeyError('No message with key: %s' % key)
-
-    # This method is for backward compatibility only.
-    def next(self):
-        """Return the next message in a one-time iteration."""
-        if not hasattr(self, '_onetime_keys'):
-            self._onetime_keys = self.iterkeys()
-        while True:
-            try:
-                return self[self._onetime_keys.next()]
-            except StopIteration:
-                return None
-            except KeyError:
-                continue
-
-
-class _singlefileMailbox(Mailbox):
-    """A single-file mailbox."""
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize a single-file mailbox."""
-        Mailbox.__init__(self, path, factory, create)
-        try:
-            f = open(self._path, 'rb+')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                if create:
-                    f = open(self._path, 'wb+')
-                else:
-                    raise NoSuchMailboxError(self._path)
-            elif e.errno in (errno.EACCES, errno.EROFS):
-                f = open(self._path, 'rb')
-            else:
-                raise
-        self._file = f
-        self._toc = None
-        self._next_key = 0
-        self._pending = False   # No changes require rewriting the file.
-        self._locked = False
-        self._file_length = None        # Used to record mailbox size
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        self._lookup()
-        self._toc[self._next_key] = self._append_message(message)
-        self._next_key += 1
-        self._pending = True
-        return self._next_key - 1
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        self._lookup(key)
-        del self._toc[key]
-        self._pending = True
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        self._lookup(key)
-        self._toc[key] = self._append_message(message)
-        self._pending = True
-
-    def iterkeys(self):
-        """Return an iterator over keys."""
-        self._lookup()
-        for key in self._toc.keys():
-            yield key
-
-    def has_key(self, key):
-        """Return True if the keyed message exists, False otherwise."""
-        self._lookup()
-        return key in self._toc
-
-    def __len__(self):
-        """Return a count of messages in the mailbox."""
-        self._lookup()
-        return len(self._toc)
-
-    def lock(self):
-        """Lock the mailbox."""
-        if not self._locked:
-            _lock_file(self._file)
-            self._locked = True
-
-    def unlock(self):
-        """Unlock the mailbox if it is locked."""
-        if self._locked:
-            _unlock_file(self._file)
-            self._locked = False
-
-    def flush(self):
-        """Write any pending changes to disk."""
-        if not self._pending:
-            return
-
-        # In order to be writing anything out at all, self._toc must
-        # already have been generated (and presumably has been modified
-        # by adding or deleting an item).
-        assert self._toc is not None
-
-        # Check length of self._file; if it's changed, some other process
-        # has modified the mailbox since we scanned it.
-        self._file.seek(0, 2)
-        cur_len = self._file.tell()
-        if cur_len != self._file_length:
-            raise ExternalClashError('Size of mailbox file changed '
-                                     '(expected %i, found %i)' %
-                                     (self._file_length, cur_len))
-
-        new_file = _create_temporary(self._path)
-        try:
-            new_toc = {}
-            self._pre_mailbox_hook(new_file)
-            for key in sorted(self._toc.keys()):
-                start, stop = self._toc[key]
-                self._file.seek(start)
-                self._pre_message_hook(new_file)
-                new_start = new_file.tell()
-                while True:
-                    buffer = self._file.read(min(4096,
-                                                 stop - self._file.tell()))
-                    if buffer == '':
-                        break
-                    new_file.write(buffer)
-                new_toc[key] = (new_start, new_file.tell())
-                self._post_message_hook(new_file)
-        except:
-            new_file.close()
-            os.remove(new_file.name)
-            raise
-        _sync_close(new_file)
-        # self._file is about to get replaced, so no need to sync.
-        self._file.close()
-        try:
-            os.rename(new_file.name, self._path)
-        except OSError, e:
-            if e.errno == errno.EEXIST or \
-              (os.name == 'os2' and e.errno == errno.EACCES):
-                os.remove(self._path)
-                os.rename(new_file.name, self._path)
-            else:
-                raise
-        self._file = open(self._path, 'rb+')
-        self._toc = new_toc
-        self._pending = False
-        if self._locked:
-            _lock_file(self._file, dotlock=False)
-
-    def _pre_mailbox_hook(self, f):
-        """Called before writing the mailbox to file f."""
-        return
-
-    def _pre_message_hook(self, f):
-        """Called before writing each message to file f."""
-        return
-
-    def _post_message_hook(self, f):
-        """Called after writing each message to file f."""
-        return
-
-    def close(self):
-        """Flush and close the mailbox."""
-        self.flush()
-        if self._locked:
-            self.unlock()
-        self._file.close()  # Sync has been done by self.flush() above.
-
-    def _lookup(self, key=None):
-        """Return (start, stop) or raise KeyError."""
-        if self._toc is None:
-            self._generate_toc()
-        if key is not None:
-            try:
-                return self._toc[key]
-            except KeyError:
-                raise KeyError('No message with key: %s' % key)
-
-    def _append_message(self, message):
-        """Append message to mailbox and return (start, stop) offsets."""
-        self._file.seek(0, 2)
-        before = self._file.tell()
-        try:
-            self._pre_message_hook(self._file)
-            offsets = self._install_message(message)
-            self._post_message_hook(self._file)
-        except BaseException:
-            self._file.truncate(before)
-            raise
-        self._file.flush()
-        self._file_length = self._file.tell()  # Record current length of mailbox
-        return offsets
-
-
-
-class _mboxMMDF(_singlefileMailbox):
-    """An mbox or MMDF mailbox."""
-
-    _mangle_from_ = True
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        from_line = self._file.readline().replace(os.linesep, '')
-        string = self._file.read(stop - self._file.tell())
-        msg = self._message_factory(string.replace(os.linesep, '\n'))
-        msg.set_from(from_line[5:])
-        return msg
-
-    def get_string(self, key, from_=False):
-        """Return a string representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        if not from_:
-            self._file.readline()
-        string = self._file.read(stop - self._file.tell())
-        return string.replace(os.linesep, '\n')
-
-    def get_file(self, key, from_=False):
-        """Return a file-like representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        if not from_:
-            self._file.readline()
-        return _PartialFile(self._file, self._file.tell(), stop)
-
-    def _install_message(self, message):
-        """Format a message and blindly write to self._file."""
-        from_line = None
-        if isinstance(message, str) and message.startswith('From '):
-            newline = message.find('\n')
-            if newline != -1:
-                from_line = message[:newline]
-                message = message[newline + 1:]
-            else:
-                from_line = message
-                message = ''
-        elif isinstance(message, _mboxMMDFMessage):
-            from_line = 'From ' + message.get_from()
-        elif isinstance(message, email.message.Message):
-            from_line = message.get_unixfrom()  # May be None.
-        if from_line is None:
-            from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
-        start = self._file.tell()
-        self._file.write(from_line + os.linesep)
-        self._dump_message(message, self._file, self._mangle_from_)
-        stop = self._file.tell()
-        return (start, stop)
-
-
-class mbox(_mboxMMDF):
-    """A classic mbox mailbox."""
-
-    _mangle_from_ = True
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize an mbox mailbox."""
-        self._message_factory = mboxMessage
-        _mboxMMDF.__init__(self, path, factory, create)
-
-    def _pre_message_hook(self, f):
-        """Called before writing each message to file f."""
-        if f.tell() != 0:
-            f.write(os.linesep)
-
-    def _generate_toc(self):
-        """Generate key-to-(start, stop) table of contents."""
-        starts, stops = [], []
-        self._file.seek(0)
-        while True:
-            line_pos = self._file.tell()
-            line = self._file.readline()
-            if line.startswith('From '):
-                if len(stops) < len(starts):
-                    stops.append(line_pos - len(os.linesep))
-                starts.append(line_pos)
-            elif line == '':
-                stops.append(line_pos)
-                break
-        self._toc = dict(enumerate(zip(starts, stops)))
-        self._next_key = len(self._toc)
-        self._file_length = self._file.tell()
-
-
-class MMDF(_mboxMMDF):
-    """An MMDF mailbox."""
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize an MMDF mailbox."""
-        self._message_factory = MMDFMessage
-        _mboxMMDF.__init__(self, path, factory, create)
-
-    def _pre_message_hook(self, f):
-        """Called before writing each message to file f."""
-        f.write('\001\001\001\001' + os.linesep)
-
-    def _post_message_hook(self, f):
-        """Called after writing each message to file f."""
-        f.write(os.linesep + '\001\001\001\001' + os.linesep)
-
-    def _generate_toc(self):
-        """Generate key-to-(start, stop) table of contents."""
-        starts, stops = [], []
-        self._file.seek(0)
-        next_pos = 0
-        while True:
-            line_pos = next_pos
-            line = self._file.readline()
-            next_pos = self._file.tell()
-            if line.startswith('\001\001\001\001' + os.linesep):
-                starts.append(next_pos)
-                while True:
-                    line_pos = next_pos
-                    line = self._file.readline()
-                    next_pos = self._file.tell()
-                    if line == '\001\001\001\001' + os.linesep:
-                        stops.append(line_pos - len(os.linesep))
-                        break
-                    elif line == '':
-                        stops.append(line_pos)
-                        break
-            elif line == '':
-                break
-        self._toc = dict(enumerate(zip(starts, stops)))
-        self._next_key = len(self._toc)
-        self._file.seek(0, 2)
-        self._file_length = self._file.tell()
-
-
-class MH(Mailbox):
-    """An MH mailbox."""
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize an MH instance."""
-        Mailbox.__init__(self, path, factory, create)
-        if not os.path.exists(self._path):
-            if create:
-                os.mkdir(self._path, 0700)
-                os.close(os.open(os.path.join(self._path, '.mh_sequences'),
-                                 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
-            else:
-                raise NoSuchMailboxError(self._path)
-        self._locked = False
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        keys = self.keys()
-        if len(keys) == 0:
-            new_key = 1
-        else:
-            new_key = max(keys) + 1
-        new_path = os.path.join(self._path, str(new_key))
-        f = _create_carefully(new_path)
-        closed = False
-        try:
-            if self._locked:
-                _lock_file(f)
-            try:
-                try:
-                    self._dump_message(message, f)
-                except BaseException:
-                    # Unlock and close so it can be deleted on Windows
-                    if self._locked:
-                        _unlock_file(f)
-                    _sync_close(f)
-                    closed = True
-                    os.remove(new_path)
-                    raise
-                if isinstance(message, MHMessage):
-                    self._dump_sequences(message, new_key)
-            finally:
-                if self._locked:
-                    _unlock_file(f)
-        finally:
-            if not closed:
-                _sync_close(f)
-        return new_key
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        path = os.path.join(self._path, str(key))
-        try:
-            f = open(path, 'rb+')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        else:
-            f.close()
-            os.remove(path)
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        path = os.path.join(self._path, str(key))
-        try:
-            f = open(path, 'rb+')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        try:
-            if self._locked:
-                _lock_file(f)
-            try:
-                os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
-                self._dump_message(message, f)
-                if isinstance(message, MHMessage):
-                    self._dump_sequences(message, key)
-            finally:
-                if self._locked:
-                    _unlock_file(f)
-        finally:
-            _sync_close(f)
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        try:
-            if self._locked:
-                f = open(os.path.join(self._path, str(key)), 'r+')
-            else:
-                f = open(os.path.join(self._path, str(key)), 'r')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        try:
-            if self._locked:
-                _lock_file(f)
-            try:
-                msg = MHMessage(f)
-            finally:
-                if self._locked:
-                    _unlock_file(f)
-        finally:
-            f.close()
-        for name, key_list in self.get_sequences().iteritems():
-            if key in key_list:
-                msg.add_sequence(name)
-        return msg
-
-    def get_string(self, key):
-        """Return a string representation or raise a KeyError."""
-        try:
-            if self._locked:
-                f = open(os.path.join(self._path, str(key)), 'r+')
-            else:
-                f = open(os.path.join(self._path, str(key)), 'r')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        try:
-            if self._locked:
-                _lock_file(f)
-            try:
-                return f.read()
-            finally:
-                if self._locked:
-                    _unlock_file(f)
-        finally:
-            f.close()
-
-    def get_file(self, key):
-        """Return a file-like representation or raise a KeyError."""
-        try:
-            f = open(os.path.join(self._path, str(key)), 'rb')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        return _ProxyFile(f)
-
-    def iterkeys(self):
-        """Return an iterator over keys."""
-        return iter(sorted(int(entry) for entry in os.listdir(self._path)
-                                      if entry.isdigit()))
-
-    def has_key(self, key):
-        """Return True if the keyed message exists, False otherwise."""
-        return os.path.exists(os.path.join(self._path, str(key)))
-
-    def __len__(self):
-        """Return a count of messages in the mailbox."""
-        return len(list(self.iterkeys()))
-
-    def lock(self):
-        """Lock the mailbox."""
-        if not self._locked:
-            self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
-            _lock_file(self._file)
-            self._locked = True
-
-    def unlock(self):
-        """Unlock the mailbox if it is locked."""
-        if self._locked:
-            _unlock_file(self._file)
-            _sync_close(self._file)
-            del self._file
-            self._locked = False
-
-    def flush(self):
-        """Write any pending changes to the disk."""
-        return
-
-    def close(self):
-        """Flush and close the mailbox."""
-        if self._locked:
-            self.unlock()
-
-    def list_folders(self):
-        """Return a list of folder names."""
-        result = []
-        for entry in os.listdir(self._path):
-            if os.path.isdir(os.path.join(self._path, entry)):
-                result.append(entry)
-        return result
-
-    def get_folder(self, folder):
-        """Return an MH instance for the named folder."""
-        return MH(os.path.join(self._path, folder),
-                  factory=self._factory, create=False)
-
-    def add_folder(self, folder):
-        """Create a folder and return an MH instance representing it."""
-        return MH(os.path.join(self._path, folder),
-                  factory=self._factory)
-
-    def remove_folder(self, folder):
-        """Delete the named folder, which must be empty."""
-        path = os.path.join(self._path, folder)
-        entries = os.listdir(path)
-        if entries == ['.mh_sequences']:
-            os.remove(os.path.join(path, '.mh_sequences'))
-        elif entries == []:
-            pass
-        else:
-            raise NotEmptyError('Folder not empty: %s' % self._path)
-        os.rmdir(path)
-
-    def get_sequences(self):
-        """Return a name-to-key-list dictionary to define each sequence."""
-        results = {}
-        f = open(os.path.join(self._path, '.mh_sequences'), 'r')
-        try:
-            all_keys = set(self.keys())
-            for line in f:
-                try:
-                    name, contents = line.split(':')
-                    keys = set()
-                    for spec in contents.split():
-                        if spec.isdigit():
-                            keys.add(int(spec))
-                        else:
-                            start, stop = (int(x) for x in spec.split('-'))
-                            keys.update(range(start, stop + 1))
-                    results[name] = [key for key in sorted(keys) \
-                                         if key in all_keys]
-                    if len(results[name]) == 0:
-                        del results[name]
-                except ValueError:
-                    raise FormatError('Invalid sequence specification: %s' %
-                                      line.rstrip())
-        finally:
-            f.close()
-        return results
-
-    def set_sequences(self, sequences):
-        """Set sequences using the given name-to-key-list dictionary."""
-        f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
-        try:
-            os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
-            for name, keys in sequences.iteritems():
-                if len(keys) == 0:
-                    continue
-                f.write('%s:' % name)
-                prev = None
-                completing = False
-                for key in sorted(set(keys)):
-                    if key - 1 == prev:
-                        if not completing:
-                            completing = True
-                            f.write('-')
-                    elif completing:
-                        completing = False
-                        f.write('%s %s' % (prev, key))
-                    else:
-                        f.write(' %s' % key)
-                    prev = key
-                if completing:
-                    f.write(str(prev) + '\n')
-                else:
-                    f.write('\n')
-        finally:
-            _sync_close(f)
-
-    def pack(self):
-        """Re-name messages to eliminate numbering gaps. Invalidates keys."""
-        sequences = self.get_sequences()
-        prev = 0
-        changes = []
-        for key in self.iterkeys():
-            if key - 1 != prev:
-                changes.append((key, prev + 1))
-                if hasattr(os, 'link'):
-                    os.link(os.path.join(self._path, str(key)),
-                            os.path.join(self._path, str(prev + 1)))
-                    os.unlink(os.path.join(self._path, str(key)))
-                else:
-                    os.rename(os.path.join(self._path, str(key)),
-                              os.path.join(self._path, str(prev + 1)))
-            prev += 1
-        self._next_key = prev + 1
-        if len(changes) == 0:
-            return
-        for name, key_list in sequences.items():
-            for old, new in changes:
-                if old in key_list:
-                    key_list[key_list.index(old)] = new
-        self.set_sequences(sequences)
-
-    def _dump_sequences(self, message, key):
-        """Inspect a new MHMessage and update sequences appropriately."""
-        pending_sequences = message.get_sequences()
-        all_sequences = self.get_sequences()
-        for name, key_list in all_sequences.iteritems():
-            if name in pending_sequences:
-                key_list.append(key)
-            elif key in key_list:
-                del key_list[key_list.index(key)]
-        for sequence in pending_sequences:
-            if sequence not in all_sequences:
-                all_sequences[sequence] = [key]
-        self.set_sequences(all_sequences)
-
-
-class Babyl(_singlefileMailbox):
-    """An Rmail-style Babyl mailbox."""
-
-    _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
-                                 'forwarded', 'edited', 'resent'))
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize a Babyl mailbox."""
-        _singlefileMailbox.__init__(self, path, factory, create)
-        self._labels = {}
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        key = _singlefileMailbox.add(self, message)
-        if isinstance(message, BabylMessage):
-            self._labels[key] = message.get_labels()
-        return key
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        _singlefileMailbox.remove(self, key)
-        if key in self._labels:
-            del self._labels[key]
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        _singlefileMailbox.__setitem__(self, key, message)
-        if isinstance(message, BabylMessage):
-            self._labels[key] = message.get_labels()
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        self._file.readline()   # Skip '1,' line specifying labels.
-        original_headers = StringIO.StringIO()
-        while True:
-            line = self._file.readline()
-            if line == '*** EOOH ***' + os.linesep or line == '':
-                break
-            original_headers.write(line.replace(os.linesep, '\n'))
-        visible_headers = StringIO.StringIO()
-        while True:
-            line = self._file.readline()
-            if line == os.linesep or line == '':
-                break
-            visible_headers.write(line.replace(os.linesep, '\n'))
-        body = self._file.read(stop - self._file.tell()).replace(os.linesep,
-                                                                 '\n')
-        msg = BabylMessage(original_headers.getvalue() + body)
-        msg.set_visible(visible_headers.getvalue())
-        if key in self._labels:
-            msg.set_labels(self._labels[key])
-        return msg
-
-    def get_string(self, key):
-        """Return a string representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        self._file.readline()   # Skip '1,' line specifying labels.
-        original_headers = StringIO.StringIO()
-        while True:
-            line = self._file.readline()
-            if line == '*** EOOH ***' + os.linesep or line == '':
-                break
-            original_headers.write(line.replace(os.linesep, '\n'))
-        while True:
-            line = self._file.readline()
-            if line == os.linesep or line == '':
-                break
-        return original_headers.getvalue() + \
-               self._file.read(stop - self._file.tell()).replace(os.linesep,
-                                                                 '\n')
-
-    def get_file(self, key):
-        """Return a file-like representation or raise a KeyError."""
-        return StringIO.StringIO(self.get_string(key).replace('\n',
-                                                              os.linesep))
-
-    def get_labels(self):
-        """Return a list of user-defined labels in the mailbox."""
-        self._lookup()
-        labels = set()
-        for label_list in self._labels.values():
-            labels.update(label_list)
-        labels.difference_update(self._special_labels)
-        return list(labels)
-
-    def _generate_toc(self):
-        """Generate key-to-(start, stop) table of contents."""
-        starts, stops = [], []
-        self._file.seek(0)
-        next_pos = 0
-        label_lists = []
-        while True:
-            line_pos = next_pos
-            line = self._file.readline()
-            next_pos = self._file.tell()
-            if line == '\037\014' + os.linesep:
-                if len(stops) < len(starts):
-                    stops.append(line_pos - len(os.linesep))
-                starts.append(next_pos)
-                labels = [label.strip() for label
-                                        in self._file.readline()[1:].split(',')
-                                        if label.strip() != '']
-                label_lists.append(labels)
-            elif line == '\037' or line == '\037' + os.linesep:
-                if len(stops) < len(starts):
-                    stops.append(line_pos - len(os.linesep))
-            elif line == '':
-                stops.append(line_pos - len(os.linesep))
-                break
-        self._toc = dict(enumerate(zip(starts, stops)))
-        self._labels = dict(enumerate(label_lists))
-        self._next_key = len(self._toc)
-        self._file.seek(0, 2)
-        self._file_length = self._file.tell()
-
-    def _pre_mailbox_hook(self, f):
-        """Called before writing the mailbox to file f."""
-        f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
-                (os.linesep, os.linesep, ','.join(self.get_labels()),
-                 os.linesep))
-
-    def _pre_message_hook(self, f):
-        """Called before writing each message to file f."""
-        f.write('\014' + os.linesep)
-
-    def _post_message_hook(self, f):
-        """Called after writing each message to file f."""
-        f.write(os.linesep + '\037')
-
-    def _install_message(self, message):
-        """Write message contents and return (start, stop)."""
-        start = self._file.tell()
-        if isinstance(message, BabylMessage):
-            special_labels = []
-            labels = []
-            for label in message.get_labels():
-                if label in self._special_labels:
-                    special_labels.append(label)
-                else:
-                    labels.append(label)
-            self._file.write('1')
-            for label in special_labels:
-                self._file.write(', ' + label)
-            self._file.write(',,')
-            for label in labels:
-                self._file.write(' ' + label + ',')
-            self._file.write(os.linesep)
-        else:
-            self._file.write('1,,' + os.linesep)
-        if isinstance(message, email.message.Message):
-            orig_buffer = StringIO.StringIO()
-            orig_generator = email.generator.Generator(orig_buffer, False, 0)
-            orig_generator.flatten(message)
-            orig_buffer.seek(0)
-            while True:
-                line = orig_buffer.readline()
-                self._file.write(line.replace('\n', os.linesep))
-                if line == '\n' or line == '':
-                    break
-            self._file.write('*** EOOH ***' + os.linesep)
-            if isinstance(message, BabylMessage):
-                vis_buffer = StringIO.StringIO()
-                vis_generator = email.generator.Generator(vis_buffer, False, 0)
-                vis_generator.flatten(message.get_visible())
-                while True:
-                    line = vis_buffer.readline()
-                    self._file.write(line.replace('\n', os.linesep))
-                    if line == '\n' or line == '':
-                        break
-            else:
-                orig_buffer.seek(0)
-                while True:
-                    line = orig_buffer.readline()
-                    self._file.write(line.replace('\n', os.linesep))
-                    if line == '\n' or line == '':
-                        break
-            while True:
-                buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
-                if buffer == '':
-                    break
-                self._file.write(buffer.replace('\n', os.linesep))
-        elif isinstance(message, str):
-            body_start = message.find('\n\n') + 2
-            if body_start - 2 != -1:
-                self._file.write(message[:body_start].replace('\n',
-                                                              os.linesep))
-                self._file.write('*** EOOH ***' + os.linesep)
-                self._file.write(message[:body_start].replace('\n',
-                                                              os.linesep))
-                self._file.write(message[body_start:].replace('\n',
-                                                              os.linesep))
-            else:
-                self._file.write('*** EOOH ***' + os.linesep + os.linesep)
-                self._file.write(message.replace('\n', os.linesep))
-        elif hasattr(message, 'readline'):
-            original_pos = message.tell()
-            first_pass = True
-            while True:
-                line = message.readline()
-                self._file.write(line.replace('\n', os.linesep))
-                if line == '\n' or line == '':
-                    self._file.write('*** EOOH ***' + os.linesep)
-                    if first_pass:
-                        first_pass = False
-                        message.seek(original_pos)
-                    else:
-                        break
-            while True:
-                buffer = message.read(4096)     # Buffer size is arbitrary.
-                if buffer == '':
-                    break
-                self._file.write(buffer.replace('\n', os.linesep))
-        else:
-            raise TypeError('Invalid message type: %s' % type(message))
-        stop = self._file.tell()
-        return (start, stop)
-
-
-class Message(email.message.Message):
-    """Message with mailbox-format-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize a Message instance."""
-        if isinstance(message, email.message.Message):
-            self._become_message(copy.deepcopy(message))
-            if isinstance(message, Message):
-                message._explain_to(self)
-        elif isinstance(message, str):
-            self._become_message(email.message_from_string(message))
-        elif hasattr(message, "read"):
-            self._become_message(email.message_from_file(message))
-        elif message is None:
-            email.message.Message.__init__(self)
-        else:
-            raise TypeError('Invalid message type: %s' % type(message))
-
-    def _become_message(self, message):
-        """Assume the non-format-specific state of message."""
-        for name in ('_headers', '_unixfrom', '_payload', '_charset',
-                     'preamble', 'epilogue', 'defects', '_default_type'):
-            self.__dict__[name] = message.__dict__[name]
-
-    def _explain_to(self, message):
-        """Copy format-specific state to message insofar as possible."""
-        if isinstance(message, Message):
-            return  # There's nothing format-specific to explain.
-        else:
-            raise TypeError('Cannot convert to specified type')
-
-
-class MaildirMessage(Message):
-    """Message with Maildir-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize a MaildirMessage instance."""
-        self._subdir = 'new'
-        self._info = ''
-        self._date = time.time()
-        Message.__init__(self, message)
-
-    def get_subdir(self):
-        """Return 'new' or 'cur'."""
-        return self._subdir
-
-    def set_subdir(self, subdir):
-        """Set subdir to 'new' or 'cur'."""
-        if subdir == 'new' or subdir == 'cur':
-            self._subdir = subdir
-        else:
-            raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
-
-    def get_flags(self):
-        """Return as a string the flags that are set."""
-        if self._info.startswith('2,'):
-            return self._info[2:]
-        else:
-            return ''
-
-    def set_flags(self, flags):
-        """Set the given flags and unset all others."""
-        self._info = '2,' + ''.join(sorted(flags))
-
-    def add_flag(self, flag):
-        """Set the given flag(s) without changing others."""
-        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
-
-    def remove_flag(self, flag):
-        """Unset the given string flag(s) without changing others."""
-        if self.get_flags() != '':
-            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
-
-    def get_date(self):
-        """Return delivery date of message, in seconds since the epoch."""
-        return self._date
-
-    def set_date(self, date):
-        """Set delivery date of message, in seconds since the epoch."""
-        try:
-            self._date = float(date)
-        except ValueError:
-            raise TypeError("can't convert to float: %s" % date)
-
-    def get_info(self):
-        """Get the message's "info" as a string."""
-        return self._info
-
-    def set_info(self, info):
-        """Set the message's "info" string."""
-        if isinstance(info, str):
-            self._info = info
-        else:
-            raise TypeError('info must be a string: %s' % type(info))
-
-    def _explain_to(self, message):
-        """Copy Maildir-specific state to message insofar as possible."""
-        if isinstance(message, MaildirMessage):
-            message.set_flags(self.get_flags())
-            message.set_subdir(self.get_subdir())
-            message.set_date(self.get_date())
-        elif isinstance(message, _mboxMMDFMessage):
-            flags = set(self.get_flags())
-            if 'S' in flags:
-                message.add_flag('R')
-            if self.get_subdir() == 'cur':
-                message.add_flag('O')
-            if 'T' in flags:
-                message.add_flag('D')
-            if 'F' in flags:
-                message.add_flag('F')
-            if 'R' in flags:
-                message.add_flag('A')
-            message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
-        elif isinstance(message, MHMessage):
-            flags = set(self.get_flags())
-            if 'S' not in flags:
-                message.add_sequence('unseen')
-            if 'R' in flags:
-                message.add_sequence('replied')
-            if 'F' in flags:
-                message.add_sequence('flagged')
-        elif isinstance(message, BabylMessage):
-            flags = set(self.get_flags())
-            if 'S' not in flags:
-                message.add_label('unseen')
-            if 'T' in flags:
-                message.add_label('deleted')
-            if 'R' in flags:
-                message.add_label('answered')
-            if 'P' in flags:
-                message.add_label('forwarded')
-        elif isinstance(message, Message):
-            pass
-        else:
-            raise TypeError('Cannot convert to specified type: %s' %
-                            type(message))
-
-
-class _mboxMMDFMessage(Message):
-    """Message with mbox- or MMDF-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize an mboxMMDFMessage instance."""
-        self.set_from('MAILER-DAEMON', True)
-        if isinstance(message, email.message.Message):
-            unixfrom = message.get_unixfrom()
-            if unixfrom is not None and unixfrom.startswith('From '):
-                self.set_from(unixfrom[5:])
-        Message.__init__(self, message)
-
-    def get_from(self):
-        """Return contents of "From " line."""
-        return self._from
-
-    def set_from(self, from_, time_=None):
-        """Set "From " line, formatting and appending time_ if specified."""
-        if time_ is not None:
-            if time_ is True:
-                time_ = time.gmtime()
-            from_ += ' ' + time.asctime(time_)
-        self._from = from_
-
-    def get_flags(self):
-        """Return as a string the flags that are set."""
-        return self.get('Status', '') + self.get('X-Status', '')
-
-    def set_flags(self, flags):
-        """Set the given flags and unset all others."""
-        flags = set(flags)
-        status_flags, xstatus_flags = '', ''
-        for flag in ('R', 'O'):
-            if flag in flags:
-                status_flags += flag
-                flags.remove(flag)
-        for flag in ('D', 'F', 'A'):
-            if flag in flags:
-                xstatus_flags += flag
-                flags.remove(flag)
-        xstatus_flags += ''.join(sorted(flags))
-        try:
-            self.replace_header('Status', status_flags)
-        except KeyError:
-            self.add_header('Status', status_flags)
-        try:
-            self.replace_header('X-Status', xstatus_flags)
-        except KeyError:
-            self.add_header('X-Status', xstatus_flags)
-
-    def add_flag(self, flag):
-        """Set the given flag(s) without changing others."""
-        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
-
-    def remove_flag(self, flag):
-        """Unset the given string flag(s) without changing others."""
-        if 'Status' in self or 'X-Status' in self:
-            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
-
-    def _explain_to(self, message):
-        """Copy mbox- or MMDF-specific state to message insofar as possible."""
-        if isinstance(message, MaildirMessage):
-            flags = set(self.get_flags())
-            if 'O' in flags:
-                message.set_subdir('cur')
-            if 'F' in flags:
-                message.add_flag('F')
-            if 'A' in flags:
-                message.add_flag('R')
-            if 'R' in flags:
-                message.add_flag('S')
-            if 'D' in flags:
-                message.add_flag('T')
-            del message['status']
-            del message['x-status']
-            maybe_date = ' '.join(self.get_from().split()[-5:])
-            try:
-                message.set_date(calendar.timegm(time.strptime(maybe_date,
-                                                      '%a %b %d %H:%M:%S %Y')))
-            except (ValueError, OverflowError):
-                pass
-        elif isinstance(message, _mboxMMDFMessage):
-            message.set_flags(self.get_flags())
-            message.set_from(self.get_from())
-        elif isinstance(message, MHMessage):
-            flags = set(self.get_flags())
-            if 'R' not in flags:
-                message.add_sequence('unseen')
-            if 'A' in flags:
-                message.add_sequence('replied')
-            if 'F' in flags:
-                message.add_sequence('flagged')
-            del message['status']
-            del message['x-status']
-        elif isinstance(message, BabylMessage):
-            flags = set(self.get_flags())
-            if 'R' not in flags:
-                message.add_label('unseen')
-            if 'D' in flags:
-                message.add_label('deleted')
-            if 'A' in flags:
-                message.add_label('answered')
-            del message['status']
-            del message['x-status']
-        elif isinstance(message, Message):
-            pass
-        else:
-            raise TypeError('Cannot convert to specified type: %s' %
-                            type(message))
-
-
-class mboxMessage(_mboxMMDFMessage):
-    """Message with mbox-specific properties."""
-
-
-class MHMessage(Message):
-    """Message with MH-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize an MHMessage instance."""
-        self._sequences = []
-        Message.__init__(self, message)
-
-    def get_sequences(self):
-        """Return a list of sequences that include the message."""
-        return self._sequences[:]
-
-    def set_sequences(self, sequences):
-        """Set the list of sequences that include the message."""
-        self._sequences = list(sequences)
-
-    def add_sequence(self, sequence):
-        """Add sequence to list of sequences including the message."""
-        if isinstance(sequence, str):
-            if not sequence in self._sequences:
-                self._sequences.append(sequence)
-        else:
-            raise TypeError('sequence must be a string: %s' % type(sequence))
-
-    def remove_sequence(self, sequence):
-        """Remove sequence from the list of sequences including the message."""
-        try:
-            self._sequences.remove(sequence)
-        except ValueError:
-            pass
-
-    def _explain_to(self, message):
-        """Copy MH-specific state to message insofar as possible."""
-        if isinstance(message, MaildirMessage):
-            sequences = set(self.get_sequences())
-            if 'unseen' in sequences:
-                message.set_subdir('cur')
-            else:
-                message.set_subdir('cur')
-                message.add_flag('S')
-            if 'flagged' in sequences:
-                message.add_flag('F')
-            if 'replied' in sequences:
-                message.add_flag('R')
-        elif isinstance(message, _mboxMMDFMessage):
-            sequences = set(self.get_sequences())
-            if 'unseen' not in sequences:
-                message.add_flag('RO')
-            else:
-                message.add_flag('O')
-            if 'flagged' in sequences:
-                message.add_flag('F')
-            if 'replied' in sequences:
-                message.add_flag('A')
-        elif isinstance(message, MHMessage):
-            for sequence in self.get_sequences():
-                message.add_sequence(sequence)
-        elif isinstance(message, BabylMessage):
-            sequences = set(self.get_sequences())
-            if 'unseen' in sequences:
-                message.add_label('unseen')
-            if 'replied' in sequences:
-                message.add_label('answered')
-        elif isinstance(message, Message):
-            pass
-        else:
-            raise TypeError('Cannot convert to specified type: %s' %
-                            type(message))
-
-
-class BabylMessage(Message):
-    """Message with Babyl-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize an BabylMessage instance."""
-        self._labels = []
-        self._visible = Message()
-        Message.__init__(self, message)
-
-    def get_labels(self):
-        """Return a list of labels on the message."""
-        return self._labels[:]
-
-    def set_labels(self, labels):
-        """Set the list of labels on the message."""
-        self._labels = list(labels)
-
-    def add_label(self, label):
-        """Add label to list of labels on the message."""
-        if isinstance(label, str):
-            if label not in self._labels:
-                self._labels.append(label)
-        else:
-            raise TypeError('label must be a string: %s' % type(label))
-
-    def remove_label(self, label):
-        """Remove label from the list of labels on the message."""
-        try:
-            self._labels.remove(label)
-        except ValueError:
-            pass
-
-    def get_visible(self):
-        """Return a Message representation of visible headers."""
-        return Message(self._visible)
-
-    def set_visible(self, visible):
-        """Set the Message representation of visible headers."""
-        self._visible = Message(visible)
-
-    def update_visible(self):
-        """Update and/or sensibly generate a set of visible headers."""
-        for header in self._visible.keys():
-            if header in self:
-                self._visible.replace_header(header, self[header])
-            else:
-                del self._visible[header]
-        for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
-            if header in self and header not in self._visible:
-                self._visible[header] = self[header]
-
-    def _explain_to(self, message):
-        """Copy Babyl-specific state to message insofar as possible."""
-        if isinstance(message, MaildirMessage):
-            labels = set(self.get_labels())
-            if 'unseen' in labels:
-                message.set_subdir('cur')
-            else:
-                message.set_subdir('cur')
-                message.add_flag('S')
-            if 'forwarded' in labels or 'resent' in labels:
-                message.add_flag('P')
-            if 'answered' in labels:
-                message.add_flag('R')
-            if 'deleted' in labels:
-                message.add_flag('T')
-        elif isinstance(message, _mboxMMDFMessage):
-            labels = set(self.get_labels())
-            if 'unseen' not in labels:
-                message.add_flag('RO')
-            else:
-                message.add_flag('O')
-            if 'deleted' in labels:
-                message.add_flag('D')
-            if 'answered' in labels:
-                message.add_flag('A')
-        elif isinstance(message, MHMessage):
-            labels = set(self.get_labels())
-            if 'unseen' in labels:
-                message.add_sequence('unseen')
-            if 'answered' in labels:
-                message.add_sequence('replied')
-        elif isinstance(message, BabylMessage):
-            message.set_visible(self.get_visible())
-            for label in self.get_labels():
-                message.add_label(label)
-        elif isinstance(message, Message):
-            pass
-        else:
-            raise TypeError('Cannot convert to specified type: %s' %
-                            type(message))
-
-
-class MMDFMessage(_mboxMMDFMessage):
-    """Message with MMDF-specific properties."""
-
-
-class _ProxyFile:
-    """A read-only wrapper of a file."""
-
-    def __init__(self, f, pos=None):
-        """Initialize a _ProxyFile."""
-        self._file = f
-        if pos is None:
-            self._pos = f.tell()
-        else:
-            self._pos = pos
-
-    def read(self, size=None):
-        """Read bytes."""
-        return self._read(size, self._file.read)
-
-    def readline(self, size=None):
-        """Read a line."""
-        return self._read(size, self._file.readline)
-
-    def readlines(self, sizehint=None):
-        """Read multiple lines."""
-        result = []
-        for line in self:
-            result.append(line)
-            if sizehint is not None:
-                sizehint -= len(line)
-                if sizehint <= 0:
-                    break
-        return result
-
-    def __iter__(self):
-        """Iterate over lines."""
-        return iter(self.readline, "")
-
-    def tell(self):
-        """Return the position."""
-        return self._pos
-
-    def seek(self, offset, whence=0):
-        """Change position."""
-        if whence == 1:
-            self._file.seek(self._pos)
-        self._file.seek(offset, whence)
-        self._pos = self._file.tell()
-
-    def close(self):
-        """Close the file."""
-        del self._file
-
-    def _read(self, size, read_method):
-        """Read size bytes using read_method."""
-        if size is None:
-            size = -1
-        self._file.seek(self._pos)
-        result = read_method(size)
-        self._pos = self._file.tell()
-        return result
-
-
-class _PartialFile(_ProxyFile):
-    """A read-only wrapper of part of a file."""
-
-    def __init__(self, f, start=None, stop=None):
-        """Initialize a _PartialFile."""
-        _ProxyFile.__init__(self, f, start)
-        self._start = start
-        self._stop = stop
-
-    def tell(self):
-        """Return the position with respect to start."""
-        return _ProxyFile.tell(self) - self._start
-
-    def seek(self, offset, whence=0):
-        """Change position, possibly with respect to start or stop."""
-        if whence == 0:
-            self._pos = self._start
-            whence = 1
-        elif whence == 2:
-            self._pos = self._stop
-            whence = 1
-        _ProxyFile.seek(self, offset, whence)
-
-    def _read(self, size, read_method):
-        """Read size bytes using read_method, honoring start and stop."""
-        remaining = self._stop - self._pos
-        if remaining <= 0:
-            return ''
-        if size is None or size < 0 or size > remaining:
-            size = remaining
-        return _ProxyFile._read(self, size, read_method)
-
-
-def _lock_file(f, dotlock=True):
-    """Lock file f using lockf and dot locking."""
-    dotlock_done = False
-    try:
-        if fcntl:
-            try:
-                fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
-            except IOError, e:
-                if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
-                    raise ExternalClashError('lockf: lock unavailable: %s' %
-                                             f.name)
-                else:
-                    raise
-        if dotlock:
-            try:
-                pre_lock = _create_temporary(f.name + '.lock')
-                pre_lock.close()
-            except IOError, e:
-                if e.errno in (errno.EACCES, errno.EROFS):
-                    return  # Without write access, just skip dotlocking.
-                else:
-                    raise
-            try:
-                if hasattr(os, 'link'):
-                    os.link(pre_lock.name, f.name + '.lock')
-                    dotlock_done = True
-                    os.unlink(pre_lock.name)
-                else:
-                    os.rename(pre_lock.name, f.name + '.lock')
-                    dotlock_done = True
-            except OSError, e:
-                if e.errno == errno.EEXIST or \
-                  (os.name == 'os2' and e.errno == errno.EACCES):
-                    os.remove(pre_lock.name)
-                    raise ExternalClashError('dot lock unavailable: %s' %
-                                             f.name)
-                else:
-                    raise
-    except:
-        if fcntl:
-            fcntl.lockf(f, fcntl.LOCK_UN)
-        if dotlock_done:
-            os.remove(f.name + '.lock')
-        raise
-
-def _unlock_file(f):
-    """Unlock file f using lockf and dot locking."""
-    if fcntl:
-        fcntl.lockf(f, fcntl.LOCK_UN)
-    if os.path.exists(f.name + '.lock'):
-        os.remove(f.name + '.lock')
-
-def _create_carefully(path):
-    """Create a file if it doesn't exist and open for reading and writing."""
-    fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
-    try:
-        return open(path, 'rb+')
-    finally:
-        os.close(fd)
-
-def _create_temporary(path):
-    """Create a temp file based on path and open for reading and writing."""
-    return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
-                                              socket.gethostname(),
-                                              os.getpid()))
-
-def _sync_flush(f):
-    """Ensure changes to file f are physically on disk."""
-    f.flush()
-    if hasattr(os, 'fsync'):
-        os.fsync(f.fileno())
-
-def _sync_close(f):
-    """Close file f, ensuring all changes are physically on disk."""
-    _sync_flush(f)
-    f.close()
-
-## Start: classes from the original module (for backward compatibility).
-
-# Note that the Maildir class, whose name is unchanged, itself offers a next()
-# method for backward compatibility.
-
-class _Mailbox:
-
-    def __init__(self, fp, factory=rfc822.Message):
-        self.fp = fp
-        self.seekp = 0
-        self.factory = factory
-
-    def __iter__(self):
-        return iter(self.next, None)
-
-    def next(self):
-        while 1:
-            self.fp.seek(self.seekp)
-            try:
-                self._search_start()
-            except EOFError:
-                self.seekp = self.fp.tell()
-                return None
-            start = self.fp.tell()
-            self._search_end()
-            self.seekp = stop = self.fp.tell()
-            if start != stop:
-                break
-        return self.factory(_PartialFile(self.fp, start, stop))
-
-# Recommended to use PortableUnixMailbox instead!
-class UnixMailbox(_Mailbox):
-
-    def _search_start(self):
-        while 1:
-            pos = self.fp.tell()
-            line = self.fp.readline()
-            if not line:
-                raise EOFError
-            if line[:5] == 'From ' and self._isrealfromline(line):
-                self.fp.seek(pos)
-                return
-
-    def _search_end(self):
-        self.fp.readline()      # Throw away header line
-        while 1:
-            pos = self.fp.tell()
-            line = self.fp.readline()
-            if not line:
-                return
-            if line[:5] == 'From ' and self._isrealfromline(line):
-                self.fp.seek(pos)
-                return
-
-    # An overridable mechanism to test for From-line-ness.  You can either
-    # specify a different regular expression or define a whole new
-    # _isrealfromline() method.  Note that this only gets called for lines
-    # starting with the 5 characters "From ".
-    #
-    # BAW: According to
-    #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
-    # the only portable, reliable way to find message delimiters in a BSD (i.e
-    # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
-    # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
-    # like a good idea, in practice, there are too many variations for more
-    # strict parsing of the line to be completely accurate.
-    #
-    # _strict_isrealfromline() is the old version which tries to do stricter
-    # parsing of the From_ line.  _portable_isrealfromline() simply returns
-    # true, since it's never called if the line doesn't already start with
-    # "From ".
-    #
-    # This algorithm, and the way it interacts with _search_start() and
-    # _search_end() may not be completely correct, because it doesn't check
-    # that the two characters preceding "From " are \n\n or the beginning of
-    # the file.  Fixing this would require a more extensive rewrite than is
-    # necessary.  For convenience, we've added a PortableUnixMailbox class
-    # which does no checking of the format of the 'From' line.
-
-    _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
-                        r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
-                        r"[^\s]*\s*"
-                        "$")
-    _regexp = None
-
-    def _strict_isrealfromline(self, line):
-        if not self._regexp:
-            import re
-            self._regexp = re.compile(self._fromlinepattern)
-        return self._regexp.match(line)
-
-    def _portable_isrealfromline(self, line):
-        return True
-
-    _isrealfromline = _strict_isrealfromline
-
-
-class PortableUnixMailbox(UnixMailbox):
-    _isrealfromline = UnixMailbox._portable_isrealfromline
-
-
-class MmdfMailbox(_Mailbox):
-
-    def _search_start(self):
-        while 1:
-            line = self.fp.readline()
-            if not line:
-                raise EOFError
-            if line[:5] == '\001\001\001\001\n':
-                return
-
-    def _search_end(self):
-        while 1:
-            pos = self.fp.tell()
-            line = self.fp.readline()
-            if not line:
-                return
-            if line == '\001\001\001\001\n':
-                self.fp.seek(pos)
-                return
-
-
-class MHMailbox:
-
-    def __init__(self, dirname, factory=rfc822.Message):
-        import re
-        pat = re.compile('^[1-9][0-9]*$')
-        self.dirname = dirname
-        # the three following lines could be combined into:
-        # list = map(long, filter(pat.match, os.listdir(self.dirname)))
-        list = os.listdir(self.dirname)
-        list = filter(pat.match, list)
-        list = map(long, list)
-        list.sort()
-        # This only works in Python 1.6 or later;
-        # before that str() added 'L':
-        self.boxes = map(str, list)
-        self.boxes.reverse()
-        self.factory = factory
-
-    def __iter__(self):
-        return iter(self.next, None)
-
-    def next(self):
-        if not self.boxes:
-            return None
-        fn = self.boxes.pop()
-        fp = open(os.path.join(self.dirname, fn))
-        msg = self.factory(fp)
-        try:
-            msg._mh_msgno = fn
-        except (AttributeError, TypeError):
-            pass
-        return msg
-
-
-class BabylMailbox(_Mailbox):
-
-    def _search_start(self):
-        while 1:
-            line = self.fp.readline()
-            if not line:
-                raise EOFError
-            if line == '*** EOOH ***\n':
-                return
-
-    def _search_end(self):
-        while 1:
-            pos = self.fp.tell()
-            line = self.fp.readline()
-            if not line:
-                return
-            if line == '\037\014\n' or line == '\037':
-                self.fp.seek(pos)
-                return
-
-## End: classes from the original module (for backward compatibility).
-
-
-class Error(Exception):
-    """Raised for module-specific errors."""
-
-class NoSuchMailboxError(Error):
-    """The specified mailbox does not exist and won't be created."""
-
-class NotEmptyError(Error):
-    """The specified mailbox is not empty and deletion was requested."""
-
-class ExternalClashError(Error):
-    """Another process caused an action to fail."""
-
-class FormatError(Error):
-    """A file appears to have an invalid format."""
diff --git a/lib-python/modified-2.7/mailbox.py b/lib-python/modified-2.7/mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/mailbox.py
@@ -0,0 +1,2172 @@
+#! /usr/bin/env python
+
+"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
+
+# Notes for authors of new mailbox subclasses:
+#
+# Remember to fsync() changes to disk before closing a modified file
+# or returning from a flush() method.  See functions _sync_flush() and
+# _sync_close().
+
+import sys
+import os
+import time
+import calendar
+import socket
+import errno
+import copy
+import email
+import email.message
+import email.generator
+import StringIO
+try:
+    if sys.platform == 'os2emx':
+        # OS/2 EMX fcntl() not adequate
+        raise ImportError
+    import fcntl
+except ImportError:
+    fcntl = None
+
+import warnings
+with warnings.catch_warnings():
+    if sys.py3kwarning:
+        warnings.filterwarnings("ignore", ".*rfc822 has been removed",
+                                DeprecationWarning)
+    import rfc822
+
+__all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
+            'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
+            'BabylMessage', 'MMDFMessage', 'UnixMailbox',
+            'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
+
+class Mailbox:
+    """A group of messages in a particular place."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a Mailbox instance."""
+        self._path = os.path.abspath(os.path.expanduser(path))
+        self._factory = factory
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def __delitem__(self, key):
+        self.remove(key)
+
+    def discard(self, key):
+        """If the keyed message exists, remove it."""
+        try:
+            self.remove(key)
+        except KeyError:
+            pass
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get(self, key, default=None):
+        """Return the keyed message, or default if it doesn't exist."""
+        try:
+            return self.__getitem__(key)
+        except KeyError:
+            return default
+
+    def __getitem__(self, key):
+        """Return the keyed message; raise KeyError if it doesn't exist."""
+        if not self._factory:
+            return self.get_message(key)
+        else:
+            return self._factory(self.get_file(key))
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def keys(self):
+        """Return a list of keys."""
+        return list(self.iterkeys())
+
+    def itervalues(self):
+        """Return an iterator over all messages."""
+        for key in self.iterkeys():
+            try:
+                value = self[key]
+            except KeyError:
+                continue
+            yield value
+
+    def __iter__(self):
+        return self.itervalues()
+
+    def values(self):
+        """Return a list of messages. Memory intensive."""
+        return list(self.itervalues())
+
+    def iteritems(self):
+        """Return an iterator over (key, message) tuples."""
+        for key in self.iterkeys():
+            try:
+                value = self[key]
+            except KeyError:
+                continue
+            yield (key, value)
+
+    def items(self):
+        """Return a list of (key, message) tuples. Memory intensive."""
+        return list(self.iteritems())
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def __contains__(self, key):
+        return self.has_key(key)
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def clear(self):
+        """Delete all messages."""
+        for key in self.iterkeys():
+            self.discard(key)
+
+    def pop(self, key, default=None):
+        """Delete the keyed message and return it, or default."""
+        try:
+            result = self[key]
+        except KeyError:
+            return default
+        self.discard(key)
+        return result
+
+    def popitem(self):
+        """Delete an arbitrary (key, message) pair and return it."""
+        for key in self.iterkeys():
+            return (key, self.pop(key))     # This is only run once.
+        else:
+            raise KeyError('No messages in mailbox')
+
+    def update(self, arg=None):
+        """Change the messages that correspond to certain keys."""
+        if hasattr(arg, 'iteritems'):
+            source = arg.iteritems()
+        elif hasattr(arg, 'items'):
+            source = arg.items()
+        else:
+            source = arg
+        bad_key = False
+        for key, message in source:
+            try:
+                self[key] = message
+            except KeyError:
+                bad_key = True
+        if bad_key:
+            raise KeyError('No message with key(s)')
+
+    def flush(self):
+        """Write any pending changes to the disk."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def lock(self):
+        """Lock the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def close(self):
+        """Flush and close the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def _dump_message(self, message, target, mangle_from_=False):
+        # Most files are opened in binary mode to allow predictable seeking.
+        # To get native line endings on disk, the user-friendly \n line endings
+        # used in strings and by email.Message are translated here.
+        """Dump message contents to target file."""
+        if isinstance(message, email.message.Message):
+            buffer = StringIO.StringIO()
+            gen = email.generator.Generator(buffer, mangle_from_, 0)
+            gen.flatten(message)
+            buffer.seek(0)
+            target.write(buffer.read().replace('\n', os.linesep))
+        elif isinstance(message, str):
+            if mangle_from_:
+                message = message.replace('\nFrom ', '\n>From ')
+            message = message.replace('\n', os.linesep)
+            target.write(message)
+        elif hasattr(message, 'read'):
+            while True:
+                line = message.readline()
+                if line == '':
+                    break
+                if mangle_from_ and line.startswith('From '):
+                    line = '>From ' + line[5:]
+                line = line.replace('\n', os.linesep)
+                target.write(line)
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+
+
+class Maildir(Mailbox):
+    """A qmail-style Maildir mailbox."""
+
+    colon = ':'
+
+    def __init__(self, dirname, factory=rfc822.Message, create=True):
+        """Initialize a Maildir instance."""
+        Mailbox.__init__(self, dirname, factory, create)
+        self._paths = {
+            'tmp': os.path.join(self._path, 'tmp'),
+            'new': os.path.join(self._path, 'new'),
+            'cur': os.path.join(self._path, 'cur'),
+            }
+        if not os.path.exists(self._path):
+            if create:
+                os.mkdir(self._path, 0700)
+                for path in self._paths.values():
+                    os.mkdir(path, 0o700)
+            else:
+                raise NoSuchMailboxError(self._path)
+        self._toc = {}
+        self._toc_mtimes = {}
+        for subdir in ('cur', 'new'):
+            self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir])
+        self._last_read = time.time()  # Records last time we read cur/new
+        self._skewfactor = 0.1         # Adjust if os/fs clocks are skewing
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        tmp_file = self._create_tmp()
+        try:
+            self._dump_message(message, tmp_file)
+        except BaseException:
+            tmp_file.close()
+            os.remove(tmp_file.name)
+            raise
+        _sync_close(tmp_file)
+        if isinstance(message, MaildirMessage):
+            subdir = message.get_subdir()
+            suffix = self.colon + message.get_info()
+            if suffix == self.colon:
+                suffix = ''
+        else:
+            subdir = 'new'
+            suffix = ''
+        uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
+        dest = os.path.join(self._path, subdir, uniq + suffix)
+        try:
+            if hasattr(os, 'link'):
+                os.link(tmp_file.name, dest)
+                os.remove(tmp_file.name)
+            else:
+                os.rename(tmp_file.name, dest)
+        except OSError, e:
+            os.remove(tmp_file.name)
+            if e.errno == errno.EEXIST:
+                raise ExternalClashError('Name clash with existing message: %s'
+                                         % dest)
+            else:
+                raise
+        if isinstance(message, MaildirMessage):
+            os.utime(dest, (os.path.getatime(dest), message.get_date()))
+        return uniq
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        os.remove(os.path.join(self._path, self._lookup(key)))
+
+    def discard(self, key):
+        """If the keyed message exists, remove it."""
+        # This overrides an inapplicable implementation in the superclass.
+        try:
+            self.remove(key)
+        except KeyError:
+            pass
+        except OSError, e:
+            if e.errno != errno.ENOENT:
+                raise
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        old_subpath = self._lookup(key)
+        temp_key = self.add(message)
+        temp_subpath = self._lookup(temp_key)
+        if isinstance(message, MaildirMessage):
+            # temp's subdir and suffix were specified by message.
+            dominant_subpath = temp_subpath
+        else:
+            # temp's subdir and suffix were defaults from add().
+            dominant_subpath = old_subpath
+        subdir = os.path.dirname(dominant_subpath)
+        if self.colon in dominant_subpath:
+            suffix = self.colon + dominant_subpath.split(self.colon)[-1]
+        else:
+            suffix = ''
+        self.discard(key)
+        new_path = os.path.join(self._path, subdir, key + suffix)
+        os.rename(os.path.join(self._path, temp_subpath), new_path)
+        if isinstance(message, MaildirMessage):
+            os.utime(new_path, (os.path.getatime(new_path),
+                                message.get_date()))
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        subpath = self._lookup(key)
+        f = open(os.path.join(self._path, subpath), 'r')
+        try:
+            if self._factory:
+                msg = self._factory(f)
+            else:
+                msg = MaildirMessage(f)
+        finally:
+            f.close()
+        subdir, name = os.path.split(subpath)
+        msg.set_subdir(subdir)
+        if self.colon in name:
+            msg.set_info(name.split(self.colon)[-1])
+        msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        f = open(os.path.join(self._path, self._lookup(key)), 'r')
+        try:
+            return f.read()
+        finally:
+            f.close()
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        f = open(os.path.join(self._path, self._lookup(key)), 'rb')
+        return _ProxyFile(f)
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        self._refresh()
+        for key in self._toc:
+            try:
+                self._lookup(key)
+            except KeyError:
+                continue
+            yield key
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        self._refresh()
+        return key in self._toc
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        self._refresh()
+        return len(self._toc)
+
+    def flush(self):
+        """Write any pending changes to disk."""
+        # Maildir changes are always written immediately, so there's nothing
+        # to do.
+        pass
+
+    def lock(self):
+        """Lock the mailbox."""
+        return
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        return
+
+    def list_folders(self):
+        """Return a list of folder names."""
+        result = []
+        for entry in os.listdir(self._path):
+            if len(entry) > 1 and entry[0] == '.' and \
+               os.path.isdir(os.path.join(self._path, entry)):
+                result.append(entry[1:])
+        return result
+
+    def get_folder(self, folder):
+        """Return a Maildir instance for the named folder."""
+        return Maildir(os.path.join(self._path, '.' + folder),
+                       factory=self._factory,
+                       create=False)
+
+    def add_folder(self, folder):
+        """Create a folder and return a Maildir instance representing it."""
+        path = os.path.join(self._path, '.' + folder)
+        result = Maildir(path, factory=self._factory)
+        maildirfolder_path = os.path.join(path, 'maildirfolder')
+        if not os.path.exists(maildirfolder_path):
+            os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
+                0666))
+        return result
+
+    def remove_folder(self, folder):
+        """Delete the named folder, which must be empty."""
+        path = os.path.join(self._path, '.' + folder)
+        for entry in os.listdir(os.path.join(path, 'new')) + \
+                     os.listdir(os.path.join(path, 'cur')):
+            if len(entry) < 1 or entry[0] != '.':
+                raise NotEmptyError('Folder contains message(s): %s' % folder)
+        for entry in os.listdir(path):
+            if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
+               os.path.isdir(os.path.join(path, entry)):
+                raise NotEmptyError("Folder contains subdirectory '%s': %s" %
+                                    (folder, entry))
+        for root, dirs, files in os.walk(path, topdown=False):
+            for entry in files:
+                os.remove(os.path.join(root, entry))
+            for entry in dirs:
+                os.rmdir(os.path.join(root, entry))
+        os.rmdir(path)
+
+    def clean(self):
+        """Delete old files in "tmp"."""
+        now = time.time()
+        for entry in os.listdir(os.path.join(self._path, 'tmp')):
+            path = os.path.join(self._path, 'tmp', entry)
+            if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
+                os.remove(path)
+
+    _count = 1  # This is used to generate unique file names.
+
+    def _create_tmp(self):
+        """Create a file in the tmp subdirectory and open and return it."""
+        now = time.time()
+        hostname = socket.gethostname()
+        if '/' in hostname:
+            hostname = hostname.replace('/', r'\057')
+        if ':' in hostname:
+            hostname = hostname.replace(':', r'\072')
+        uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
+                                    Maildir._count, hostname)
+        path = os.path.join(self._path, 'tmp', uniq)
+        try:
+            os.stat(path)
+        except OSError, e:
+            if e.errno == errno.ENOENT:
+                Maildir._count += 1
+                try:
+                    return _create_carefully(path)
+                except OSError, e:
+                    if e.errno != errno.EEXIST:
+                        raise
+            else:
+                raise
+
+        # Fall through to here if stat succeeded or open raised EEXIST.
+        raise ExternalClashError('Name clash prevented file creation: %s' %
+                                 path)
+
+    def _refresh(self):
+        """Update table of contents mapping."""
+        # If it has been less than two seconds since the last _refresh() call,
+        # we have to unconditionally re-read the mailbox just in case it has
+        # been modified, because os.path.mtime() has a 2 sec resolution in the
+        # most common worst case (FAT) and a 1 sec resolution typically.  This
+        # results in a few unnecessary re-reads when _refresh() is called
+        # multiple times in that interval, but once the clock ticks over, we
+        # will only re-read as needed.  Because the filesystem might be being
+        # served by an independent system with its own clock, we record and
+        # compare with the mtimes from the filesystem.  Because the other
+        # system's clock might be skewing relative to our clock, we add an
+        # extra delta to our wait.  The default is one tenth second, but is an
+        # instance variable and so can be adjusted if dealing with a
+        # particularly skewed or irregular system.
+        if time.time() - self._last_read > 2 + self._skewfactor:
+            refresh = False
+            for subdir in self._toc_mtimes:
+                mtime = os.path.getmtime(self._paths[subdir])
+                if mtime > self._toc_mtimes[subdir]:
+                    refresh = True
+                self._toc_mtimes[subdir] = mtime
+            if not refresh:
+                return
+        # Refresh toc
+        self._toc = {}
+        for subdir in self._toc_mtimes:
+            path = self._paths[subdir]
+            for entry in os.listdir(path):
+                p = os.path.join(path, entry)
+                if os.path.isdir(p):
+                    continue
+                uniq = entry.split(self.colon)[0]
+                self._toc[uniq] = os.path.join(subdir, entry)
+        self._last_read = time.time()
+
+    def _lookup(self, key):
+        """Use TOC to return subpath for given key, or raise a KeyError."""
+        try:
+            if os.path.exists(os.path.join(self._path, self._toc[key])):
+                return self._toc[key]
+        except KeyError:
+            pass
+        self._refresh()
+        try:
+            return self._toc[key]
+        except KeyError:
+            raise KeyError('No message with key: %s' % key)
+
+    # This method is for backward compatibility only.
+    def next(self):
+        """Return the next message in a one-time iteration."""
+        if not hasattr(self, '_onetime_keys'):
+            self._onetime_keys = self.iterkeys()
+        while True:
+            try:
+                return self[self._onetime_keys.next()]
+            except StopIteration:
+                return None
+            except KeyError:
+                continue
+
+
+class _singlefileMailbox(Mailbox):
+    """A single-file mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a single-file mailbox."""
+        Mailbox.__init__(self, path, factory, create)
+        try:
+            f = open(self._path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                if create:
+                    f = open(self._path, 'wb+')
+                else:
+                    raise NoSuchMailboxError(self._path)
+            elif e.errno in (errno.EACCES, errno.EROFS):
+                f = open(self._path, 'rb')
+            else:
+                raise
+        self._file = f
+        self._toc = None
+        self._next_key = 0
+        self._pending = False   # No changes require rewriting the file.
+        self._locked = False
+        self._file_length = None        # Used to record mailbox size
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        self._lookup()
+        self._toc[self._next_key] = self._append_message(message)
+        self._next_key += 1
+        self._pending = True
+        return self._next_key - 1
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        self._lookup(key)
+        del self._toc[key]
+        self._pending = True
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        self._lookup(key)
+        self._toc[key] = self._append_message(message)
+        self._pending = True
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        self._lookup()
+        for key in self._toc.keys():
+            yield key
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        self._lookup()
+        return key in self._toc
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        self._lookup()
+        return len(self._toc)
+
+    def lock(self):
+        """Lock the mailbox."""
+        if not self._locked:
+            _lock_file(self._file)
+            self._locked = True
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        if self._locked:
+            _unlock_file(self._file)
+            self._locked = False
+
+    def flush(self):
+        """Write any pending changes to disk."""
+        if not self._pending:
+            return
+
+        # In order to be writing anything out at all, self._toc must
+        # already have been generated (and presumably has been modified
+        # by adding or deleting an item).
+        assert self._toc is not None
+
+        # Check length of self._file; if it's changed, some other process
+        # has modified the mailbox since we scanned it.
+        self._file.seek(0, 2)
+        cur_len = self._file.tell()
+        if cur_len != self._file_length:
+            raise ExternalClashError('Size of mailbox file changed '
+                                     '(expected %i, found %i)' %
+                                     (self._file_length, cur_len))
+
+        new_file = _create_temporary(self._path)
+        try:
+            new_toc = {}
+            self._pre_mailbox_hook(new_file)
+            for key in sorted(self._toc.keys()):
+                start, stop = self._toc[key]
+                self._file.seek(start)
+                self._pre_message_hook(new_file)
+                new_start = new_file.tell()
+                while True:
+                    buffer = self._file.read(min(4096,
+                                                 stop - self._file.tell()))
+                    if buffer == '':
+                        break
+                    new_file.write(buffer)
+                new_toc[key] = (new_start, new_file.tell())
+                self._post_message_hook(new_file)
+        except:
+            new_file.close()
+            os.remove(new_file.name)
+            raise
+        _sync_close(new_file)
+        # self._file is about to get replaced, so no need to sync.
+        self._file.close()
+        try:
+            os.rename(new_file.name, self._path)
+        except OSError, e:
+            if e.errno == errno.EEXIST or \
+              (os.name == 'os2' and e.errno == errno.EACCES):
+                os.remove(self._path)
+                os.rename(new_file.name, self._path)
+            else:
+                raise
+        self._file = open(self._path, 'rb+')
+        self._toc = new_toc
+        self._pending = False
+        if self._locked:
+            _lock_file(self._file, dotlock=False)
+
+    def _pre_mailbox_hook(self, f):
+        """Called before writing the mailbox to file f."""
+        return
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        return
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        self.flush()
+        if self._locked:
+            self.unlock()
+        self._file.close()  # Sync has been done by self.flush() above.
+
+    def _lookup(self, key=None):
+        """Return (start, stop) or raise KeyError."""
+        if self._toc is None:
+            self._generate_toc()
+        if key is not None:
+            try:
+                return self._toc[key]
+            except KeyError:
+                raise KeyError('No message with key: %s' % key)
+
+    def _append_message(self, message):
+        """Append message to mailbox and return (start, stop) offsets."""
+        self._file.seek(0, 2)
+        before = self._file.tell()
+        try:
+            self._pre_message_hook(self._file)
+            offsets = self._install_message(message)
+            self._post_message_hook(self._file)
+        except BaseException:
+            self._file.truncate(before)
+            raise
+        self._file.flush()
+        self._file_length = self._file.tell()  # Record current length of mailbox
+        return offsets
+
+
+
+class _mboxMMDF(_singlefileMailbox):
+    """An mbox or MMDF mailbox."""
+
+    _mangle_from_ = True
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        from_line = self._file.readline().replace(os.linesep, '')
+        string = self._file.read(stop - self._file.tell())
+        msg = self._message_factory(string.replace(os.linesep, '\n'))
+        msg.set_from(from_line[5:])
+        return msg
+
+    def get_string(self, key, from_=False):
+        """Return a string representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        if not from_:
+            self._file.readline()
+        string = self._file.read(stop - self._file.tell())
+        return string.replace(os.linesep, '\n')
+
+    def get_file(self, key, from_=False):
+        """Return a file-like representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        if not from_:
+            self._file.readline()
+        return _PartialFile(self._file, self._file.tell(), stop)
+
+    def _install_message(self, message):
+        """Format a message and blindly write to self._file."""
+        from_line = None
+        if isinstance(message, str) and message.startswith('From '):
+            newline = message.find('\n')
+            if newline != -1:
+                from_line = message[:newline]
+                message = message[newline + 1:]
+            else:
+                from_line = message
+                message = ''
+        elif isinstance(message, _mboxMMDFMessage):
+            from_line = 'From ' + message.get_from()
+        elif isinstance(message, email.message.Message):
+            from_line = message.get_unixfrom()  # May be None.
+        if from_line is None:
+            from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
+        start = self._file.tell()
+        self._file.write(from_line + os.linesep)
+        self._dump_message(message, self._file, self._mangle_from_)
+        stop = self._file.tell()
+        return (start, stop)
+
+
+class mbox(_mboxMMDF):
+    """A classic mbox mailbox."""
+
+    _mangle_from_ = True
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an mbox mailbox."""
+        self._message_factory = mboxMessage
+        _mboxMMDF.__init__(self, path, factory, create)
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        if f.tell() != 0:
+            f.write(os.linesep)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        while True:
+            line_pos = self._file.tell()
+            line = self._file.readline()
+            if line.startswith('From '):
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+                starts.append(line_pos)
+            elif line == '':
+                stops.append(line_pos)
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._next_key = len(self._toc)
+        self._file_length = self._file.tell()
+
+
+class MMDF(_mboxMMDF):
+    """An MMDF mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an MMDF mailbox."""
+        self._message_factory = MMDFMessage
+        _mboxMMDF.__init__(self, path, factory, create)
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        f.write('\001\001\001\001' + os.linesep)
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        f.write(os.linesep + '\001\001\001\001' + os.linesep)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        next_pos = 0
+        while True:
+            line_pos = next_pos
+            line = self._file.readline()
+            next_pos = self._file.tell()
+            if line.startswith('\001\001\001\001' + os.linesep):
+                starts.append(next_pos)
+                while True:
+                    line_pos = next_pos
+                    line = self._file.readline()
+                    next_pos = self._file.tell()
+                    if line == '\001\001\001\001' + os.linesep:
+                        stops.append(line_pos - len(os.linesep))
+                        break
+                    elif line == '':
+                        stops.append(line_pos)
+                        break
+            elif line == '':
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._next_key = len(self._toc)
+        self._file.seek(0, 2)
+        self._file_length = self._file.tell()
+
+
+class MH(Mailbox):
+    """An MH mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an MH instance."""
+        Mailbox.__init__(self, path, factory, create)
+        if not os.path.exists(self._path):
+            if create:
+                os.mkdir(self._path, 0700)
+                os.close(os.open(os.path.join(self._path, '.mh_sequences'),
+                                 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
+            else:
+                raise NoSuchMailboxError(self._path)
+        self._locked = False
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        keys = self.keys()
+        if len(keys) == 0:
+            new_key = 1
+        else:
+            new_key = max(keys) + 1
+        new_path = os.path.join(self._path, str(new_key))
+        f = _create_carefully(new_path)
+        closed = False
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                try:
+                    self._dump_message(message, f)
+                except BaseException:
+                    # Unlock and close so it can be deleted on Windows
+                    if self._locked:
+                        _unlock_file(f)
+                    _sync_close(f)
+                    closed = True
+                    os.remove(new_path)
+                    raise
+                if isinstance(message, MHMessage):
+                    self._dump_sequences(message, new_key)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            if not closed:
+                _sync_close(f)
+        return new_key
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        path = os.path.join(self._path, str(key))
+        try:
+            f = open(path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        else:
+            f.close()
+            os.remove(path)
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        path = os.path.join(self._path, str(key))
+        try:
+            f = open(path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
+                self._dump_message(message, f)
+                if isinstance(message, MHMessage):
+                    self._dump_sequences(message, key)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            _sync_close(f)
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        try:
+            if self._locked:
+                f = open(os.path.join(self._path, str(key)), 'r+')
+            else:
+                f = open(os.path.join(self._path, str(key)), 'r')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                msg = MHMessage(f)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            f.close()
+        for name, key_list in self.get_sequences().iteritems():
+            if key in key_list:
+                msg.add_sequence(name)
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        try:
+            if self._locked:
+                f = open(os.path.join(self._path, str(key)), 'r+')
+            else:
+                f = open(os.path.join(self._path, str(key)), 'r')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                return f.read()
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            f.close()
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        try:
+            f = open(os.path.join(self._path, str(key)), 'rb')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        return _ProxyFile(f)
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        return iter(sorted(int(entry) for entry in os.listdir(self._path)
+                                      if entry.isdigit()))
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        return os.path.exists(os.path.join(self._path, str(key)))
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        return len(list(self.iterkeys()))
+
+    def lock(self):
+        """Lock the mailbox."""
+        if not self._locked:
+            self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
+            _lock_file(self._file)
+            self._locked = True
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        if self._locked:
+            _unlock_file(self._file)
+            _sync_close(self._file)
+            del self._file
+            self._locked = False
+
+    def flush(self):
+        """Write any pending changes to the disk."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        if self._locked:
+            self.unlock()
+
+    def list_folders(self):
+        """Return a list of folder names."""
+        result = []
+        for entry in os.listdir(self._path):
+            if os.path.isdir(os.path.join(self._path, entry)):
+                result.append(entry)
+        return result
+
+    def get_folder(self, folder):
+        """Return an MH instance for the named folder."""
+        return MH(os.path.join(self._path, folder),
+                  factory=self._factory, create=False)
+
+    def add_folder(self, folder):
+        """Create a folder and return an MH instance representing it."""
+        return MH(os.path.join(self._path, folder),
+                  factory=self._factory)
+
+    def remove_folder(self, folder):
+        """Delete the named folder, which must be empty."""
+        path = os.path.join(self._path, folder)
+        entries = os.listdir(path)
+        if entries == ['.mh_sequences']:
+            os.remove(os.path.join(path, '.mh_sequences'))
+        elif entries == []:
+            pass
+        else:
+            raise NotEmptyError('Folder not empty: %s' % self._path)
+        os.rmdir(path)
+
+    def get_sequences(self):
+        """Return a name-to-key-list dictionary to define each sequence."""
+        results = {}
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r')
+        try:
+            all_keys = set(self.keys())
+            for line in f:
+                try:
+                    name, contents = line.split(':')
+                    keys = set()
+                    for spec in contents.split():
+                        if spec.isdigit():
+                            keys.add(int(spec))
+                        else:
+                            start, stop = (int(x) for x in spec.split('-'))
+                            keys.update(range(start, stop + 1))
+                    results[name] = [key for key in sorted(keys) \
+                                         if key in all_keys]
+                    if len(results[name]) == 0:
+                        del results[name]
+                except ValueError:
+                    raise FormatError('Invalid sequence specification: %s' %
+                                      line.rstrip())
+        finally:
+            f.close()
+        return results
+
+    def set_sequences(self, sequences):
+        """Set sequences using the given name-to-key-list dictionary."""
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
+        try:
+            os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
+            for name, keys in sequences.iteritems():
+                if len(keys) == 0:
+                    continue
+                f.write('%s:' % name)
+                prev = None
+                completing = False
+                for key in sorted(set(keys)):
+                    if key - 1 == prev:
+                        if not completing:
+                            completing = True
+                            f.write('-')
+                    elif completing:
+                        completing = False
+                        f.write('%s %s' % (prev, key))
+                    else:
+                        f.write(' %s' % key)
+                    prev = key
+                if completing:
+                    f.write(str(prev) + '\n')
+                else:
+                    f.write('\n')
+        finally:
+            _sync_close(f)
+
+    def pack(self):
+        """Re-name messages to eliminate numbering gaps. Invalidates keys."""
+        sequences = self.get_sequences()
+        prev = 0
+        changes = []
+        for key in self.iterkeys():
+            if key - 1 != prev:
+                changes.append((key, prev + 1))
+                if hasattr(os, 'link'):
+                    os.link(os.path.join(self._path, str(key)),
+                            os.path.join(self._path, str(prev + 1)))
+                    os.unlink(os.path.join(self._path, str(key)))
+                else:
+                    os.rename(os.path.join(self._path, str(key)),
+                              os.path.join(self._path, str(prev + 1)))
+            prev += 1
+        self._next_key = prev + 1
+        if len(changes) == 0:
+            return
+        for name, key_list in sequences.items():
+            for old, new in changes:
+                if old in key_list:
+                    key_list[key_list.index(old)] = new
+        self.set_sequences(sequences)
+
+    def _dump_sequences(self, message, key):
+        """Inspect a new MHMessage and update sequences appropriately."""
+        pending_sequences = message.get_sequences()
+        all_sequences = self.get_sequences()
+        for name, key_list in all_sequences.iteritems():
+            if name in pending_sequences:
+                key_list.append(key)
+            elif key in key_list:
+                del key_list[key_list.index(key)]
+        for sequence in pending_sequences:
+            if sequence not in all_sequences:
+                all_sequences[sequence] = [key]
+        self.set_sequences(all_sequences)
+
+
+class Babyl(_singlefileMailbox):
+    """An Rmail-style Babyl mailbox."""
+
+    _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
+                                 'forwarded', 'edited', 'resent'))
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a Babyl mailbox."""
+        _singlefileMailbox.__init__(self, path, factory, create)
+        self._labels = {}
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        key = _singlefileMailbox.add(self, message)
+        if isinstance(message, BabylMessage):
+            self._labels[key] = message.get_labels()
+        return key
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        _singlefileMailbox.remove(self, key)
+        if key in self._labels:
+            del self._labels[key]
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        _singlefileMailbox.__setitem__(self, key, message)
+        if isinstance(message, BabylMessage):
+            self._labels[key] = message.get_labels()
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        self._file.readline()   # Skip '1,' line specifying labels.
+        original_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == '*** EOOH ***' + os.linesep or line == '':
+                break
+            original_headers.write(line.replace(os.linesep, '\n'))
+        visible_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == os.linesep or line == '':
+                break
+            visible_headers.write(line.replace(os.linesep, '\n'))
+        body = self._file.read(stop - self._file.tell()).replace(os.linesep,
+                                                                 '\n')
+        msg = BabylMessage(original_headers.getvalue() + body)
+        msg.set_visible(visible_headers.getvalue())
+        if key in self._labels:
+            msg.set_labels(self._labels[key])
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        self._file.readline()   # Skip '1,' line specifying labels.
+        original_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == '*** EOOH ***' + os.linesep or line == '':
+                break
+            original_headers.write(line.replace(os.linesep, '\n'))
+        while True:
+            line = self._file.readline()
+            if line == os.linesep or line == '':
+                break
+        return original_headers.getvalue() + \
+               self._file.read(stop - self._file.tell()).replace(os.linesep,
+                                                                 '\n')
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        return StringIO.StringIO(self.get_string(key).replace('\n',
+                                                              os.linesep))
+
+    def get_labels(self):
+        """Return a list of user-defined labels in the mailbox."""
+        self._lookup()
+        labels = set()
+        for label_list in self._labels.values():
+            labels.update(label_list)
+        labels.difference_update(self._special_labels)
+        return list(labels)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        next_pos = 0
+        label_lists = []
+        while True:
+            line_pos = next_pos
+            line = self._file.readline()
+            next_pos = self._file.tell()
+            if line == '\037\014' + os.linesep:
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+                starts.append(next_pos)
+                labels = [label.strip() for label
+                                        in self._file.readline()[1:].split(',')
+                                        if label.strip() != '']
+                label_lists.append(labels)
+            elif line == '\037' or line == '\037' + os.linesep:
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+            elif line == '':
+                stops.append(line_pos - len(os.linesep))
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._labels = dict(enumerate(label_lists))
+        self._next_key = len(self._toc)
+        self._file.seek(0, 2)
+        self._file_length = self._file.tell()
+
+    def _pre_mailbox_hook(self, f):
+        """Called before writing the mailbox to file f."""
+        f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
+                (os.linesep, os.linesep, ','.join(self.get_labels()),
+                 os.linesep))
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        f.write('\014' + os.linesep)
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        f.write(os.linesep + '\037')
+
+    def _install_message(self, message):
+        """Write message contents and return (start, stop)."""
+        start = self._file.tell()
+        if isinstance(message, BabylMessage):
+            special_labels = []
+            labels = []
+            for label in message.get_labels():
+                if label in self._special_labels:
+                    special_labels.append(label)
+                else:
+                    labels.append(label)
+            self._file.write('1')
+            for label in special_labels:
+                self._file.write(', ' + label)
+            self._file.write(',,')
+            for label in labels:
+                self._file.write(' ' + label + ',')
+            self._file.write(os.linesep)
+        else:
+            self._file.write('1,,' + os.linesep)
+        if isinstance(message, email.message.Message):
+            orig_buffer = StringIO.StringIO()
+            orig_generator = email.generator.Generator(orig_buffer, False, 0)
+            orig_generator.flatten(message)
+            orig_buffer.seek(0)
+            while True:
+                line = orig_buffer.readline()
+                self._file.write(line.replace('\n', os.linesep))
+                if line == '\n' or line == '':
+                    break
+            self._file.write('*** EOOH ***' + os.linesep)
+            if isinstance(message, BabylMessage):
+                vis_buffer = StringIO.StringIO()
+                vis_generator = email.generator.Generator(vis_buffer, False, 0)
+                vis_generator.flatten(message.get_visible())
+                while True:
+                    line = vis_buffer.readline()
+                    self._file.write(line.replace('\n', os.linesep))
+                    if line == '\n' or line == '':
+                        break
+            else:
+                orig_buffer.seek(0)
+                while True:
+                    line = orig_buffer.readline()
+                    self._file.write(line.replace('\n', os.linesep))
+                    if line == '\n' or line == '':
+                        break
+            while True:
+                buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
+                if buffer == '':
+                    break
+                self._file.write(buffer.replace('\n', os.linesep))
+        elif isinstance(message, str):
+            body_start = message.find('\n\n') + 2
+            if body_start - 2 != -1:
+                self._file.write(message[:body_start].replace('\n',
+                                                              os.linesep))
+                self._file.write('*** EOOH ***' + os.linesep)
+                self._file.write(message[:body_start].replace('\n',
+                                                              os.linesep))
+                self._file.write(message[body_start:].replace('\n',
+                                                              os.linesep))
+            else:
+                self._file.write('*** EOOH ***' + os.linesep + os.linesep)
+                self._file.write(message.replace('\n', os.linesep))
+        elif hasattr(message, 'readline'):
+            original_pos = message.tell()
+            first_pass = True
+            while True:
+                line = message.readline()
+                self._file.write(line.replace('\n', os.linesep))
+                if line == '\n' or line == '':
+                    self._file.write('*** EOOH ***' + os.linesep)
+                    if first_pass:
+                        first_pass = False
+                        message.seek(original_pos)
+                    else:
+                        break
+            while True:
+                buffer = message.read(4096)     # Buffer size is arbitrary.
+                if buffer == '':
+                    break
+                self._file.write(buffer.replace('\n', os.linesep))
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+        stop = self._file.tell()
+        return (start, stop)
+
+
+class Message(email.message.Message):
+    """Message with mailbox-format-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize a Message instance."""
+        if isinstance(message, email.message.Message):
+            self._become_message(copy.deepcopy(message))
+            if isinstance(message, Message):
+                message._explain_to(self)
+        elif isinstance(message, str):
+            self._become_message(email.message_from_string(message))
+        elif hasattr(message, "read"):
+            self._become_message(email.message_from_file(message))
+        elif message is None:
+            email.message.Message.__init__(self)
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+
+    def _become_message(self, message):
+        """Assume the non-format-specific state of message."""
+        for name in ('_headers', '_unixfrom', '_payload', '_charset',
+                     'preamble', 'epilogue', 'defects', '_default_type'):
+            self.__dict__[name] = message.__dict__[name]
+
+    def _explain_to(self, message):
+        """Copy format-specific state to message insofar as possible."""
+        if isinstance(message, Message):
+            return  # There's nothing format-specific to explain.
+        else:
+            raise TypeError('Cannot convert to specified type')
+
+
+class MaildirMessage(Message):
+    """Message with Maildir-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize a MaildirMessage instance."""
+        self._subdir = 'new'
+        self._info = ''
+        self._date = time.time()
+        Message.__init__(self, message)
+
+    def get_subdir(self):
+        """Return 'new' or 'cur'."""
+        return self._subdir
+
+    def set_subdir(self, subdir):
+        """Set subdir to 'new' or 'cur'."""
+        if subdir == 'new' or subdir == 'cur':
+            self._subdir = subdir
+        else:
+            raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
+
+    def get_flags(self):
+        """Return as a string the flags that are set."""
+        if self._info.startswith('2,'):
+            return self._info[2:]
+        else:
+            return ''
+
+    def set_flags(self, flags):
+        """Set the given flags and unset all others."""
+        self._info = '2,' + ''.join(sorted(flags))
+
+    def add_flag(self, flag):
+        """Set the given flag(s) without changing others."""
+        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+    def remove_flag(self, flag):
+        """Unset the given string flag(s) without changing others."""
+        if self.get_flags() != '':
+            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+    def get_date(self):
+        """Return delivery date of message, in seconds since the epoch."""
+        return self._date
+
+    def set_date(self, date):
+        """Set delivery date of message, in seconds since the epoch."""
+        try:
+            self._date = float(date)
+        except ValueError:
+            raise TypeError("can't convert to float: %s" % date)
+
+    def get_info(self):
+        """Get the message's "info" as a string."""
+        return self._info
+
+    def set_info(self, info):
+        """Set the message's "info" string."""
+        if isinstance(info, str):
+            self._info = info
+        else:
+            raise TypeError('info must be a string: %s' % type(info))
+
+    def _explain_to(self, message):
+        """Copy Maildir-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            message.set_flags(self.get_flags())
+            message.set_subdir(self.get_subdir())
+            message.set_date(self.get_date())
+        elif isinstance(message, _mboxMMDFMessage):
+            flags = set(self.get_flags())
+            if 'S' in flags:
+                message.add_flag('R')
+            if self.get_subdir() == 'cur':
+                message.add_flag('O')
+            if 'T' in flags:
+                message.add_flag('D')
+            if 'F' in flags:
+                message.add_flag('F')
+            if 'R' in flags:
+                message.add_flag('A')
+            message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
+        elif isinstance(message, MHMessage):
+            flags = set(self.get_flags())
+            if 'S' not in flags:
+                message.add_sequence('unseen')
+            if 'R' in flags:
+                message.add_sequence('replied')
+            if 'F' in flags:
+                message.add_sequence('flagged')
+        elif isinstance(message, BabylMessage):
+            flags = set(self.get_flags())
+            if 'S' not in flags:
+                message.add_label('unseen')
+            if 'T' in flags:
+                message.add_label('deleted')
+            if 'R' in flags:
+                message.add_label('answered')
+            if 'P' in flags:
+                message.add_label('forwarded')
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class _mboxMMDFMessage(Message):
+    """Message with mbox- or MMDF-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an mboxMMDFMessage instance."""
+        self.set_from('MAILER-DAEMON', True)
+        if isinstance(message, email.message.Message):
+            unixfrom = message.get_unixfrom()
+            if unixfrom is not None and unixfrom.startswith('From '):
+                self.set_from(unixfrom[5:])
+        Message.__init__(self, message)
+
+    def get_from(self):
+        """Return contents of "From " line."""
+        return self._from
+
+    def set_from(self, from_, time_=None):
+        """Set "From " line, formatting and appending time_ if specified."""
+        if time_ is not None:
+            if time_ is True:
+                time_ = time.gmtime()
+            from_ += ' ' + time.asctime(time_)
+        self._from = from_
+
+    def get_flags(self):
+        """Return as a string the flags that are set."""
+        return self.get('Status', '') + self.get('X-Status', '')
+
+    def set_flags(self, flags):
+        """Set the given flags and unset all others."""
+        flags = set(flags)
+        status_flags, xstatus_flags = '', ''
+        for flag in ('R', 'O'):
+            if flag in flags:
+                status_flags += flag
+                flags.remove(flag)
+        for flag in ('D', 'F', 'A'):
+            if flag in flags:
+                xstatus_flags += flag
+                flags.remove(flag)
+        xstatus_flags += ''.join(sorted(flags))
+        try:
+            self.replace_header('Status', status_flags)
+        except KeyError:
+            self.add_header('Status', status_flags)
+        try:
+            self.replace_header('X-Status', xstatus_flags)
+        except KeyError:
+            self.add_header('X-Status', xstatus_flags)
+
+    def add_flag(self, flag):
+        """Set the given flag(s) without changing others."""
+        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+    def remove_flag(self, flag):
+        """Unset the given string flag(s) without changing others."""
+        if 'Status' in self or 'X-Status' in self:
+            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+    def _explain_to(self, message):
+        """Copy mbox- or MMDF-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            flags = set(self.get_flags())
+            if 'O' in flags:
+                message.set_subdir('cur')
+            if 'F' in flags:
+                message.add_flag('F')
+            if 'A' in flags:
+                message.add_flag('R')
+            if 'R' in flags:
+                message.add_flag('S')
+            if 'D' in flags:
+                message.add_flag('T')
+            del message['status']
+            del message['x-status']
+            maybe_date = ' '.join(self.get_from().split()[-5:])
+            try:
+                message.set_date(calendar.timegm(time.strptime(maybe_date,
+                                                      '%a %b %d %H:%M:%S %Y')))
+            except (ValueError, OverflowError):
+                pass
+        elif isinstance(message, _mboxMMDFMessage):
+            message.set_flags(self.get_flags())
+            message.set_from(self.get_from())
+        elif isinstance(message, MHMessage):
+            flags = set(self.get_flags())
+            if 'R' not in flags:
+                message.add_sequence('unseen')
+            if 'A' in flags:
+                message.add_sequence('replied')
+            if 'F' in flags:
+                message.add_sequence('flagged')
+            del message['status']
+            del message['x-status']
+        elif isinstance(message, BabylMessage):
+            flags = set(self.get_flags())
+            if 'R' not in flags:
+                message.add_label('unseen')
+            if 'D' in flags:
+                message.add_label('deleted')
+            if 'A' in flags:
+                message.add_label('answered')
+            del message['status']
+            del message['x-status']
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class mboxMessage(_mboxMMDFMessage):
+    """Message with mbox-specific properties."""
+
+
+class MHMessage(Message):
+    """Message with MH-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an MHMessage instance."""
+        self._sequences = []
+        Message.__init__(self, message)
+
+    def get_sequences(self):
+        """Return a list of sequences that include the message."""
+        return self._sequences[:]
+
+    def set_sequences(self, sequences):
+        """Set the list of sequences that include the message."""
+        self._sequences = list(sequences)
+
+    def add_sequence(self, sequence):
+        """Add sequence to list of sequences including the message."""
+        if isinstance(sequence, str):
+            if not sequence in self._sequences:
+                self._sequences.append(sequence)
+        else:
+            raise TypeError('sequence must be a string: %s' % type(sequence))
+
+    def remove_sequence(self, sequence):
+        """Remove sequence from the list of sequences including the message."""
+        try:
+            self._sequences.remove(sequence)
+        except ValueError:
+            pass
+
+    def _explain_to(self, message):
+        """Copy MH-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' in sequences:
+                message.set_subdir('cur')
+            else:
+                message.set_subdir('cur')
+                message.add_flag('S')
+            if 'flagged' in sequences:
+                message.add_flag('F')
+            if 'replied' in sequences:
+                message.add_flag('R')
+        elif isinstance(message, _mboxMMDFMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' not in sequences:
+                message.add_flag('RO')
+            else:
+                message.add_flag('O')
+            if 'flagged' in sequences:
+                message.add_flag('F')
+            if 'replied' in sequences:
+                message.add_flag('A')
+        elif isinstance(message, MHMessage):
+            for sequence in self.get_sequences():
+                message.add_sequence(sequence)
+        elif isinstance(message, BabylMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' in sequences:
+                message.add_label('unseen')
+            if 'replied' in sequences:
+                message.add_label('answered')
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class BabylMessage(Message):
+    """Message with Babyl-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an BabylMessage instance."""
+        self._labels = []
+        self._visible = Message()
+        Message.__init__(self, message)
+
+    def get_labels(self):
+        """Return a list of labels on the message."""
+        return self._labels[:]
+
+    def set_labels(self, labels):
+        """Set the list of labels on the message."""
+        self._labels = list(labels)
+
+    def add_label(self, label):
+        """Add label to list of labels on the message."""
+        if isinstance(label, str):
+            if label not in self._labels:
+                self._labels.append(label)
+        else:
+            raise TypeError('label must be a string: %s' % type(label))
+
+    def remove_label(self, label):
+        """Remove label from the list of labels on the message."""
+        try:
+            self._labels.remove(label)
+        except ValueError:
+            pass
+
+    def get_visible(self):
+        """Return a Message representation of visible headers."""
+        return Message(self._visible)
+
+    def set_visible(self, visible):
+        """Set the Message representation of visible headers."""
+        self._visible = Message(visible)
+
+    def update_visible(self):
+        """Update and/or sensibly generate a set of visible headers."""
+        for header in self._visible.keys():
+            if header in self:
+                self._visible.replace_header(header, self[header])
+            else:
+                del self._visible[header]
+        for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
+            if header in self and header not in self._visible:
+                self._visible[header] = self[header]
+
+    def _explain_to(self, message):
+        """Copy Babyl-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            labels = set(self.get_labels())
+            if 'unseen' in labels:
+                message.set_subdir('cur')
+            else:
+                message.set_subdir('cur')
+                message.add_flag('S')
+            if 'forwarded' in labels or 'resent' in labels:
+                message.add_flag('P')
+            if 'answered' in labels:
+                message.add_flag('R')
+            if 'deleted' in labels:
+                message.add_flag('T')
+        elif isinstance(message, _mboxMMDFMessage):
+            labels = set(self.get_labels())
+            if 'unseen' not in labels:
+                message.add_flag('RO')
+            else:
+                message.add_flag('O')
+            if 'deleted' in labels:
+                message.add_flag('D')
+            if 'answered' in labels:
+                message.add_flag('A')
+        elif isinstance(message, MHMessage):
+            labels = set(self.get_labels())
+            if 'unseen' in labels:
+                message.add_sequence('unseen')
+            if 'answered' in labels:
+                message.add_sequence('replied')
+        elif isinstance(message, BabylMessage):
+            message.set_visible(self.get_visible())
+            for label in self.get_labels():
+                message.add_label(label)
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class MMDFMessage(_mboxMMDFMessage):
+    """Message with MMDF-specific properties."""
+
+
+class _ProxyFile:
+    """A read-only wrapper of a file."""
+
+    def __init__(self, f, pos=None):
+        """Initialize a _ProxyFile."""
+        self._file = f
+        if pos is None:
+            self._pos = f.tell()
+        else:
+            self._pos = pos
+
+    def read(self, size=None):
+        """Read bytes."""
+        return self._read(size, self._file.read)
+
+    def readline(self, size=None):
+        """Read a line."""
+        return self._read(size, self._file.readline)
+
+    def readlines(self, sizehint=None):
+        """Read multiple lines."""
+        result = []
+        for line in self:
+            result.append(line)
+            if sizehint is not None:
+                sizehint -= len(line)
+                if sizehint <= 0:
+                    break
+        return result
+
+    def __iter__(self):
+        """Iterate over lines."""
+        return iter(self.readline, "")
+
+    def tell(self):
+        """Return the position."""
+        return self._pos
+
+    def seek(self, offset, whence=0):
+        """Change position."""
+        if whence == 1:
+            self._file.seek(self._pos)
+        self._file.seek(offset, whence)
+        self._pos = self._file.tell()
+
+    def close(self):
+        """Close the file."""
+        self._file.close()
+        del self._file
+
+    def _read(self, size, read_method):
+        """Read size bytes using read_method."""
+        if size is None:
+            size = -1
+        self._file.seek(self._pos)
+        result = read_method(size)
+        self._pos = self._file.tell()
+        return result
+
+
+class _PartialFile(_ProxyFile):
+    """A read-only wrapper of part of a file."""
+
+    def __init__(self, f, start=None, stop=None):
+        """Initialize a _PartialFile."""
+        _ProxyFile.__init__(self, f, start)
+        self._start = start
+        self._stop = stop
+
+    def tell(self):
+        """Return the position with respect to start."""
+        return _ProxyFile.tell(self) - self._start
+
+    def seek(self, offset, whence=0):
+        """Change position, possibly with respect to start or stop."""
+        if whence == 0:
+            self._pos = self._start
+            whence = 1
+        elif whence == 2:
+            self._pos = self._stop
+            whence = 1
+        _ProxyFile.seek(self, offset, whence)
+
+    def _read(self, size, read_method):
+        """Read size bytes using read_method, honoring start and stop."""
+        remaining = self._stop - self._pos
+        if remaining <= 0:
+            return ''
+        if size is None or size < 0 or size > remaining:
+            size = remaining
+        return _ProxyFile._read(self, size, read_method)
+
+
+def _lock_file(f, dotlock=True):
+    """Lock file f using lockf and dot locking."""
+    dotlock_done = False
+    try:
+        if fcntl:
+            try:
+                fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except IOError, e:
+                if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
+                    raise ExternalClashError('lockf: lock unavailable: %s' %
+                                             f.name)
+                else:
+                    raise
+        if dotlock:
+            try:
+                pre_lock = _create_temporary(f.name + '.lock')
+                pre_lock.close()
+            except IOError, e:
+                if e.errno in (errno.EACCES, errno.EROFS):
+                    return  # Without write access, just skip dotlocking.
+                else:
+                    raise
+            try:
+                if hasattr(os, 'link'):
+                    os.link(pre_lock.name, f.name + '.lock')
+                    dotlock_done = True
+                    os.unlink(pre_lock.name)
+                else:
+                    os.rename(pre_lock.name, f.name + '.lock')
+                    dotlock_done = True
+            except OSError, e:
+                if e.errno == errno.EEXIST or \
+                  (os.name == 'os2' and e.errno == errno.EACCES):
+                    os.remove(pre_lock.name)
+                    raise ExternalClashError('dot lock unavailable: %s' %
+                                             f.name)
+                else:
+                    raise
+    except:
+        if fcntl:
+            fcntl.lockf(f, fcntl.LOCK_UN)
+        if dotlock_done:
+            os.remove(f.name + '.lock')
+        raise
+
+def _unlock_file(f):
+    """Unlock file f using lockf and dot locking."""
+    if fcntl:
+        fcntl.lockf(f, fcntl.LOCK_UN)
+    if os.path.exists(f.name + '.lock'):
+        os.remove(f.name + '.lock')
+
+def _create_carefully(path):
+    """Create a file if it doesn't exist and open for reading and writing."""
+    fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
+    try:
+        return open(path, 'rb+')
+    finally:
+        os.close(fd)
+
+def _create_temporary(path):
+    """Create a temp file based on path and open for reading and writing."""
+    return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
+                                              socket.gethostname(),
+                                              os.getpid()))
+
+def _sync_flush(f):
+    """Ensure changes to file f are physically on disk."""
+    f.flush()
+    if hasattr(os, 'fsync'):
+        os.fsync(f.fileno())
+
+def _sync_close(f):
+    """Close file f, ensuring all changes are physically on disk."""
+    _sync_flush(f)
+    f.close()
+
+## Start: classes from the original module (for backward compatibility).
+
+# Note that the Maildir class, whose name is unchanged, itself offers a next()
+# method for backward compatibility.
+
+class _Mailbox:
+
+    def __init__(self, fp, factory=rfc822.Message):
+        self.fp = fp
+        self.seekp = 0
+        self.factory = factory
+
+    def __iter__(self):
+        return iter(self.next, None)
+
+    def next(self):
+        while 1:
+            self.fp.seek(self.seekp)
+            try:
+                self._search_start()
+            except EOFError:
+                self.seekp = self.fp.tell()
+                return None
+            start = self.fp.tell()
+            self._search_end()
+            self.seekp = stop = self.fp.tell()
+            if start != stop:
+                break
+        return self.factory(_PartialFile(self.fp, start, stop))
+
+# Recommended to use PortableUnixMailbox instead!
+class UnixMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line[:5] == 'From ' and self._isrealfromline(line):
+                self.fp.seek(pos)
+                return
+
+    def _search_end(self):
+        self.fp.readline()      # Throw away header line
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line[:5] == 'From ' and self._isrealfromline(line):
+                self.fp.seek(pos)
+                return
+
+    # An overridable mechanism to test for From-line-ness.  You can either
+    # specify a different regular expression or define a whole new
+    # _isrealfromline() method.  Note that this only gets called for lines
+    # starting with the 5 characters "From ".
+    #
+    # BAW: According to
+    #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
+    # the only portable, reliable way to find message delimiters in a BSD (i.e
+    # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
+    # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
+    # like a good idea, in practice, there are too many variations for more
+    # strict parsing of the line to be completely accurate.
+    #
+    # _strict_isrealfromline() is the old version which tries to do stricter
+    # parsing of the From_ line.  _portable_isrealfromline() simply returns
+    # true, since it's never called if the line doesn't already start with
+    # "From ".
+    #
+    # This algorithm, and the way it interacts with _search_start() and
+    # _search_end() may not be completely correct, because it doesn't check
+    # that the two characters preceding "From " are \n\n or the beginning of
+    # the file.  Fixing this would require a more extensive rewrite than is
+    # necessary.  For convenience, we've added a PortableUnixMailbox class
+    # which does no checking of the format of the 'From' line.
+
+    _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
+                        r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
+                        r"[^\s]*\s*"
+                        "$")
+    _regexp = None
+
+    def _strict_isrealfromline(self, line):
+        if not self._regexp:
+            import re
+            self._regexp = re.compile(self._fromlinepattern)
+        return self._regexp.match(line)
+
+    def _portable_isrealfromline(self, line):
+        return True
+
+    _isrealfromline = _strict_isrealfromline
+
+
+class PortableUnixMailbox(UnixMailbox):
+    _isrealfromline = UnixMailbox._portable_isrealfromline
+
+
+class MmdfMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line[:5] == '\001\001\001\001\n':
+                return
+
+    def _search_end(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line == '\001\001\001\001\n':
+                self.fp.seek(pos)
+                return
+
+
+class MHMailbox:
+
+    def __init__(self, dirname, factory=rfc822.Message):
+        import re
+        pat = re.compile('^[1-9][0-9]*$')
+        self.dirname = dirname
+        # the three following lines could be combined into:
+        # list = map(long, filter(pat.match, os.listdir(self.dirname)))
+        list = os.listdir(self.dirname)
+        list = filter(pat.match, list)
+        list = map(long, list)
+        list.sort()
+        # This only works in Python 1.6 or later;
+        # before that str() added 'L':
+        self.boxes = map(str, list)
+        self.boxes.reverse()
+        self.factory = factory
+
+    def __iter__(self):
+        return iter(self.next, None)
+
+    def next(self):
+        if not self.boxes:
+            return None
+        fn = self.boxes.pop()
+        fp = open(os.path.join(self.dirname, fn))
+        msg = self.factory(fp)
+        try:
+            msg._mh_msgno = fn
+        except (AttributeError, TypeError):
+            pass
+        return msg
+
+
+class BabylMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line == '*** EOOH ***\n':
+                return
+
+    def _search_end(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line == '\037\014\n' or line == '\037':
+                self.fp.seek(pos)
+                return
+
+## End: classes from the original module (for backward compatibility).
+
+
+class Error(Exception):
+    """Raised for module-specific errors."""
+
+class NoSuchMailboxError(Error):
+    """The specified mailbox does not exist and won't be created."""
+
+class NotEmptyError(Error):
+    """The specified mailbox is not empty and deletion was requested."""
+
+class ExternalClashError(Error):
+    """Another process caused an action to fail."""
+
+class FormatError(Error):
+    """A file appears to have an invalid format."""
diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/modified-2.7/test/test_old_mailbox.py
rename from lib-python/2.7/test/test_old_mailbox.py
rename to lib-python/modified-2.7/test/test_old_mailbox.py
--- a/lib-python/2.7/test/test_old_mailbox.py
+++ b/lib-python/modified-2.7/test/test_old_mailbox.py
@@ -7,6 +7,15 @@
 import unittest
 from test import test_support
 
+def myunlink(path):
+    try:
+        os._unlink(path)
+    except:
+        import pdb;pdb.set_trace()
+
+#os._unlink = os.unlink
+#os.unlink = myunlink
+
 # cleanup earlier tests
 try:
     os.unlink(test_support.TESTFN)
@@ -35,11 +44,14 @@
         self._msgfiles = []
 
     def tearDown(self):
-        map(os.unlink, self._msgfiles)
-        os.rmdir(os.path.join(self._dir, "cur"))
-        os.rmdir(os.path.join(self._dir, "tmp"))
-        os.rmdir(os.path.join(self._dir, "new"))
-        os.rmdir(self._dir)
+        try:  
+            map(os.unlink, self._msgfiles)
+            os.rmdir(os.path.join(self._dir, "cur"))
+            os.rmdir(os.path.join(self._dir, "tmp"))
+            os.rmdir(os.path.join(self._dir, "new"))
+            os.rmdir(self._dir)
+        except:
+            import pdb;pdb.set_trace()
 
     def createMessage(self, dir, mbox=False):
         t = int(time.time() % 1000000)
@@ -73,7 +85,9 @@
         self.createMessage("cur")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 1)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 
@@ -81,7 +95,9 @@
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 1)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 
@@ -90,8 +106,12 @@
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 2)
-        self.assertTrue(self.mbox.next() is not None)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 

From noreply at buildbot.pypy.org  Wed Apr 11 13:24:29 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 13:24:29 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: add __del__, is it ever called?
Message-ID: <20120411112429.CB74A82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54284:a1087214df4c
Date: 2012-04-11 13:42 +0300
http://bitbucket.org/pypy/pypy/changeset/a1087214df4c/

Log:	add __del__, is it ever called?

diff --git a/lib-python/modified-2.7/mailbox.py b/lib-python/modified-2.7/mailbox.py
--- a/lib-python/modified-2.7/mailbox.py
+++ b/lib-python/modified-2.7/mailbox.py
@@ -1818,6 +1818,10 @@
         else:
             self._pos = pos
 
+    def __del__(self):
+        if hasattr(self,'_file'):
+            self.close()
+
     def read(self, size=None):
         """Read bytes."""
         return self._read(size, self._file.read)

From noreply at buildbot.pypy.org  Wed Apr 11 13:24:31 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 13:24:31 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: clean up cruft,
	fix parts of test_mailbox
Message-ID: <20120411112431.4A74982F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54285:870298de090f
Date: 2012-04-11 14:23 +0300
http://bitbucket.org/pypy/pypy/changeset/870298de090f/

Log:	clean up cruft, fix parts of test_mailbox

diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/modified-2.7/test/test_mailbox.py
rename from lib-python/2.7/test/test_mailbox.py
rename to lib-python/modified-2.7/test/test_mailbox.py
--- a/lib-python/2.7/test/test_mailbox.py
+++ b/lib-python/modified-2.7/test/test_mailbox.py
@@ -44,7 +44,7 @@
                 for name in dirs:
                     os.rmdir(os.path.join(path, name))
             os.rmdir(target)
-        elif os.path.exists(target):
+        elif os.pathtexists(target):
             os.remove(target)
 
 
@@ -137,6 +137,7 @@
         msg = self._box.get(key1)
         self.assertEqual(msg['from'], 'foo')
         self.assertEqual(msg.fp.read(), '1')
+        msg.fp.close()
 
     def test_getitem(self):
         # Retrieve message using __getitem__()
@@ -169,10 +170,12 @@
         # Get file representations of messages
         key0 = self._box.add(self._template % 0)
         key1 = self._box.add(_sample_message)
-        self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'),
-                         self._template % 0)
-        self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'),
-                         _sample_message)
+        msg = self._box.get_file(key0)
+        self.assertEqual(msg.read().replace(os.linesep, '\n'), self._template % 0)
+        msg.close()
+        msg = self._box.get_file(key1)
+        self.assertEqual(msg.read().replace(os.linesep, '\n'), _sample_message)
+        msg.close()
 
     def test_iterkeys(self):
         # Get keys using iterkeys()
diff --git a/lib-python/modified-2.7/test/test_old_mailbox.py b/lib-python/modified-2.7/test/test_old_mailbox.py
--- a/lib-python/modified-2.7/test/test_old_mailbox.py
+++ b/lib-python/modified-2.7/test/test_old_mailbox.py
@@ -7,15 +7,6 @@
 import unittest
 from test import test_support
 
-def myunlink(path):
-    try:
-        os._unlink(path)
-    except:
-        import pdb;pdb.set_trace()
-
-#os._unlink = os.unlink
-#os.unlink = myunlink
-
 # cleanup earlier tests
 try:
     os.unlink(test_support.TESTFN)
@@ -44,14 +35,11 @@
         self._msgfiles = []
 
     def tearDown(self):
-        try:  
-            map(os.unlink, self._msgfiles)
-            os.rmdir(os.path.join(self._dir, "cur"))
-            os.rmdir(os.path.join(self._dir, "tmp"))
-            os.rmdir(os.path.join(self._dir, "new"))
-            os.rmdir(self._dir)
-        except:
-            import pdb;pdb.set_trace()
+        map(os.unlink, self._msgfiles)
+        os.rmdir(os.path.join(self._dir, "cur"))
+        os.rmdir(os.path.join(self._dir, "tmp"))
+        os.rmdir(os.path.join(self._dir, "new"))
+        os.rmdir(self._dir)
 
     def createMessage(self, dir, mbox=False):
         t = int(time.time() % 1000000)

From noreply at buildbot.pypy.org  Wed Apr 11 15:03:20 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 11 Apr 2012 15:03:20 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: work on the blog post
Message-ID: <20120411130320.3D61382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r4181:29ebad2bd7b6
Date: 2012-04-11 15:01 +0200
http://bitbucket.org/pypy/extradoc/changeset/29ebad2bd7b6/

Log:	work on the blog post

diff --git a/blog/draft/pycon-wrapup.rst b/blog/draft/pycon-wrapup.rst
--- a/blog/draft/pycon-wrapup.rst
+++ b/blog/draft/pycon-wrapup.rst
@@ -10,7 +10,8 @@
   PyPy and at least partly failing. He however did not fail to explain
   bits and pieces about PyPy's architecture. `Video`_ is available.
 
-* We gave tons of talks, including the `tutorial`_ and `why pypy by example`_.
+* We gave tons of talks, including the `tutorial`_, `why pypy by example`_
+  and `pypy's JIT architecturew`_
 
 * We had a giant influx of new commiters, easily doubling the amount of pull
   requests ever created for PyPy. The main topics for newcomers were numpy and
@@ -19,9 +20,16 @@
 * Guido argued in his keynote that Python is not too slow. In the meantime,
   we're trying to `prove him correct`_ :-)
 
-* XXX stuff stuff
+We would like to thank everyone who talked to us, shared ideas and especially
+those who participated in sprints - we're always happy to welcome newcomers!
 
-.. _`Video`: xxx
-.. _`tutorial`: xxx
-.. _`why pypy by example`: xxx
+I'm sure there is tons of things I forgot, but thank you all!
+
+Cheers,
+fijal
+
+.. _`Video`: http://pyvideo.org/video/659/keynote-david-beazley
+.. _`tutorial`: http://pyvideo.org/video/612/how-to-get-the-most-out-of-your-pypy
+.. _`why pypy by example`: http://pyvideo.org/video/661/why-pypy-by-example
+.. _`pypy's JIT architecturew`: http://pyvideo.org/video/662/how-the-pypy-jit-works
 .. _`prove him correct`: http://mrjoes.github.com/2011/12/15/sockjs-bench.html

From noreply at buildbot.pypy.org  Wed Apr 11 15:03:21 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 11 Apr 2012 15:03:21 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20120411130321.EE6AF82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r4182:c61a7069d337
Date: 2012-04-11 15:02 +0200
http://bitbucket.org/pypy/extradoc/changeset/c61a7069d337/

Log:	merge

diff --git a/blog/draft/py3k-activity-march.png b/blog/draft/py3k-activity-march.png
new file mode 100644
index 0000000000000000000000000000000000000000..23b5b2a2aa5a7284a06c01328c6c1b4561d2a063
GIT binary patch

[cut]

diff --git a/blog/draft/py3k-status-update-3.rst b/blog/draft/py3k-status-update-3.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/py3k-status-update-3.rst
@@ -0,0 +1,64 @@
+Py3k status update #3
+---------------------
+
+This is the third status update about my work on the `py3k branch`_, which I
+can work on thanks to all of the people who donated_ to the `py3k proposal`_.
+
+A lot of work has been done during the last month: as usual, the list of
+changes is too big to be reported in a detalied way, so this is just a summary
+of what happened.
+
+One of the most active areas was killing old and deprecated features. In
+particular, we killed support for the ``__cmp__`` special method and its
+counsins, the ``cmp`` builtin function and keyword argument for
+``list.sort()`` and ``sorted()``.  Killing is easy, but then you have to fix
+all the places which breaks because of this, including all the types which
+relied on ``__cmp__`` to be comparable,, fixing all the tests which tried to
+order objects which are no longer ordeable now, or implementing new behavior
+like forbidding calling ``hash()`` on objects which implement ``__eq__`` but
+not ``__hash__``.
+
+Among the other features, we killed lots of now-gone functions in the
+``operator`` module, the builtins ``apply()``, ``reduce()`` and ``buffer``,
+and the ``os.*`` functions to deal with temporary files, which has been
+deprecated in favour of the new ``tempfile`` module.
+
+The other topic which can't miss in a py3k status update is, as usual,
+string-vs-unicode. At this round, we fixed bugs in string formatting (in
+particular to teach ``format()`` to always use unicode strings) and various
+corner cases about when calling the (possibly overridden) ``__str__`` method
+on subclasses of ``str``. Believe me, you don't want to know the precise rules
+:-).
+
+Other features which we worked on and fixed tests include, but are not limited
+to, ``marshal``, ``hashlib``, ``zipimport``, ``_socket`` and ``itertools``,
+plus the habitual endless lists of tests which fail for shallow reasons such
+as the syntactic differences, ``int`` vs ``long``, ``range()`` vs
+``list(range())`` etc. As a result, the number of failing tests dropped_ from
+650 to 235: we are beginning to see the light at the end of the tunnel :-)
+
+Benjamin finished implementing Python 3 syntax. Most of it was small cleanups
+and tweaks to be compatible with CPython such as making ``True`` and ``False``
+keywords and preventing ``. . .`` from being parsed as ``Ellipsis``. Larger
+syntax additions included keyword only arguments and function annotations.
+
+Finally, we did some RPython fixes, so that it is possible again to translate
+PyPy in the py3k branch. However, the resuling binary is a strange beast which
+mixes python 2 and python 3 semantics, so it is unusable for anything but
+`showing friends how cool it's that`_.
+
+.. image:: py3k-activity-march.png
+
+I would like to underline that I was not alone in doing all this work. In
+particular, a lot of people joined the PyPy sprint at Pycon and worked on the
+branch, as you can clearly see in this activity graph. I would like to thank
+all who helped!
+
+cheers,
+Antonio and Benjamin
+
+.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html
+.. _`py3k proposal`: http://pypy.org/py3donate.html
+.. _`py3k branch`: https://bitbucket.org/pypy/pypy/src/py3k
+.. _`showing friends how cool it's that`: http://paste.pocoo.org/show/577006/
+.. _dropped: http://buildbot.pypy.org/summary?category=linux32&branch=py3k&recentrev=53956:3c8ac35c653a
diff --git a/sprintinfo/leipzig2012/people.txt b/sprintinfo/leipzig2012/people.txt
--- a/sprintinfo/leipzig2012/people.txt
+++ b/sprintinfo/leipzig2012/people.txt
@@ -9,5 +9,6 @@
     Name              Arrive/Depart     Accomodation 
 ==================== ============== =====================
 Armin Rigo           ?              ?
+Christian Tismer     ?              ?
 ==================== ============== =====================
 

From noreply at buildbot.pypy.org  Wed Apr 11 15:05:14 2012
From: noreply at buildbot.pypy.org (gutworth)
Date: Wed, 11 Apr 2012 15:05:14 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: edits
Message-ID: <20120411130514.E4C3982F4E@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: extradoc
Changeset: r4183:a9af41b489f2
Date: 2012-04-06 13:30 -0400
http://bitbucket.org/pypy/extradoc/changeset/a9af41b489f2/

Log:	edits

diff --git a/blog/draft/py3k-status-update-3.rst b/blog/draft/py3k-status-update-3.rst
--- a/blog/draft/py3k-status-update-3.rst
+++ b/blog/draft/py3k-status-update-3.rst
@@ -1,22 +1,22 @@
 Py3k status update #3
 ---------------------
 
-This is the third status update about my work on the `py3k branch`_, which I
+This is the third status update about our work on the `py3k branch`_, which we
 can work on thanks to all of the people who donated_ to the `py3k proposal`_.
 
 A lot of work has been done during the last month: as usual, the list of
-changes is too big to be reported in a detalied way, so this is just a summary
+changes is too big to be reported in a detailed way, so this is just a summary
 of what happened.
 
 One of the most active areas was killing old and deprecated features. In
 particular, we killed support for the ``__cmp__`` special method and its
-counsins, the ``cmp`` builtin function and keyword argument for
-``list.sort()`` and ``sorted()``.  Killing is easy, but then you have to fix
-all the places which breaks because of this, including all the types which
-relied on ``__cmp__`` to be comparable,, fixing all the tests which tried to
-order objects which are no longer ordeable now, or implementing new behavior
-like forbidding calling ``hash()`` on objects which implement ``__eq__`` but
-not ``__hash__``.
+counsins, the ``cmp`` builtin function and keyword argument for ``list.sort()``
+and ``sorted()``.  Killing is easy, but then you have to fix all the places
+which breaks because of this, including all the types which relied on
+``__cmp__`` to be comparable and all the tests which tried to order objects
+which are no longer ordeable. New behavior, like forbidding calling ``hash()``
+on objects which implement ``__eq__`` but not ``__hash__``, also has to be
+implemented.
 
 Among the other features, we killed lots of now-gone functions in the
 ``operator`` module, the builtins ``apply()``, ``reduce()`` and ``buffer``,

From noreply at buildbot.pypy.org  Wed Apr 11 15:05:16 2012
From: noreply at buildbot.pypy.org (gutworth)
Date: Wed, 11 Apr 2012 15:05:16 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge heads
Message-ID: <20120411130516.1D2D382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: extradoc
Changeset: r4184:bb1caf50c520
Date: 2012-04-11 09:05 -0400
http://bitbucket.org/pypy/extradoc/changeset/bb1caf50c520/

Log:	merge heads

diff --git a/blog/draft/pycon-wrapup.rst b/blog/draft/pycon-wrapup.rst
--- a/blog/draft/pycon-wrapup.rst
+++ b/blog/draft/pycon-wrapup.rst
@@ -10,7 +10,8 @@
   PyPy and at least partly failing. He however did not fail to explain
   bits and pieces about PyPy's architecture. `Video`_ is available.
 
-* We gave tons of talks, including the `tutorial`_ and `why pypy by example`_.
+* We gave tons of talks, including the `tutorial`_, `why pypy by example`_
+  and `pypy's JIT architecturew`_
 
 * We had a giant influx of new commiters, easily doubling the amount of pull
   requests ever created for PyPy. The main topics for newcomers were numpy and
@@ -19,9 +20,16 @@
 * Guido argued in his keynote that Python is not too slow. In the meantime,
   we're trying to `prove him correct`_ :-)
 
-* XXX stuff stuff
+We would like to thank everyone who talked to us, shared ideas and especially
+those who participated in sprints - we're always happy to welcome newcomers!
 
-.. _`Video`: xxx
-.. _`tutorial`: xxx
-.. _`why pypy by example`: xxx
+I'm sure there is tons of things I forgot, but thank you all!
+
+Cheers,
+fijal
+
+.. _`Video`: http://pyvideo.org/video/659/keynote-david-beazley
+.. _`tutorial`: http://pyvideo.org/video/612/how-to-get-the-most-out-of-your-pypy
+.. _`why pypy by example`: http://pyvideo.org/video/661/why-pypy-by-example
+.. _`pypy's JIT architecturew`: http://pyvideo.org/video/662/how-the-pypy-jit-works
 .. _`prove him correct`: http://mrjoes.github.com/2011/12/15/sockjs-bench.html

From noreply at buildbot.pypy.org  Wed Apr 11 15:23:57 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Wed, 11 Apr 2012 15:23:57 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Some cleanups.
Message-ID: <20120411132357.4BB8782F4E@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: extradoc
Changeset: r4185:958c29a7ede1
Date: 2012-04-11 09:23 -0400
http://bitbucket.org/pypy/extradoc/changeset/958c29a7ede1/

Log:	Some cleanups.

diff --git a/blog/draft/pycon-wrapup.rst b/blog/draft/pycon-wrapup.rst
--- a/blog/draft/pycon-wrapup.rst
+++ b/blog/draft/pycon-wrapup.rst
@@ -6,12 +6,13 @@
 
 From the PyPy perspective, a lot at PyCon was about PyPy. Listing things:
 
-* David Beazley did an excellent keynote on trying to dive head-first into
-  PyPy and at least partly failing. He however did not fail to explain
-  bits and pieces about PyPy's architecture. `Video`_ is available.
+* David Beazley presented an excellent keynote describing his experience
+  diving head-first into PyPy and at least partly failing. He, however, did
+  not fail to explain bits and pieces about PyPy's architecture.
+  `Video`_ is available.
 
 * We gave tons of talks, including the `tutorial`_, `why pypy by example`_
-  and `pypy's JIT architecturew`_
+  and `pypy's JIT architecture`_
 
 * We had a giant influx of new commiters, easily doubling the amount of pull
   requests ever created for PyPy. The main topics for newcomers were numpy and
@@ -31,5 +32,5 @@
 .. _`Video`: http://pyvideo.org/video/659/keynote-david-beazley
 .. _`tutorial`: http://pyvideo.org/video/612/how-to-get-the-most-out-of-your-pypy
 .. _`why pypy by example`: http://pyvideo.org/video/661/why-pypy-by-example
-.. _`pypy's JIT architecturew`: http://pyvideo.org/video/662/how-the-pypy-jit-works
+.. _`pypy's JIT architecture`: http://pyvideo.org/video/662/how-the-pypy-jit-works
 .. _`prove him correct`: http://mrjoes.github.com/2011/12/15/sockjs-bench.html

From noreply at buildbot.pypy.org  Wed Apr 11 15:30:33 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 15:30:33 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: finish test_mailbox
Message-ID: <20120411133033.658BB82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54286:5e1e95bc2659
Date: 2012-04-11 16:00 +0300
http://bitbucket.org/pypy/pypy/changeset/5e1e95bc2659/

Log:	finish test_mailbox

diff --git a/lib-python/modified-2.7/mailbox.py b/lib-python/modified-2.7/mailbox.py
--- a/lib-python/modified-2.7/mailbox.py
+++ b/lib-python/modified-2.7/mailbox.py
@@ -619,7 +619,9 @@
         """Write any pending changes to disk."""
         if not self._pending:
             return
-
+        if self._file.closed:
+            self._pending = False
+            return
         # In order to be writing anything out at all, self._toc must
         # already have been generated (and presumably has been modified
         # by adding or deleting an item).
@@ -747,6 +749,7 @@
         """Return a file-like representation or raise a KeyError."""
         start, stop = self._lookup(key)
         self._file.seek(start)
+            
         if not from_:
             self._file.readline()
         return _PartialFile(self._file, self._file.tell(), stop)
diff --git a/lib-python/modified-2.7/test/test_mailbox.py b/lib-python/modified-2.7/test/test_mailbox.py
--- a/lib-python/modified-2.7/test/test_mailbox.py
+++ b/lib-python/modified-2.7/test/test_mailbox.py
@@ -44,7 +44,7 @@
                 for name in dirs:
                     os.rmdir(os.path.join(path, name))
             os.rmdir(target)
-        elif os.pathtexists(target):
+        elif os.path.exists(target):
             os.remove(target)
 
 
@@ -60,6 +60,8 @@
 
     def tearDown(self):
         self._box.close()
+        if os.name == 'nt':
+            time.sleep(0.1) #Allow all syncing to take place
         self._delete_recursively(self._path)
 
     def test_add(self):
@@ -170,12 +172,12 @@
         # Get file representations of messages
         key0 = self._box.add(self._template % 0)
         key1 = self._box.add(_sample_message)
-        msg = self._box.get_file(key0)
-        self.assertEqual(msg.read().replace(os.linesep, '\n'), self._template % 0)
-        msg.close()
-        msg = self._box.get_file(key1)
-        self.assertEqual(msg.read().replace(os.linesep, '\n'), _sample_message)
-        msg.close()
+        msg0 = self._box.get_file(key0)
+        self.assertEqual(msg0.read().replace(os.linesep, '\n'), self._template % 0)
+        msg1 = self._box.get_file(key1)
+        self.assertEqual(msg1.read().replace(os.linesep, '\n'), _sample_message)
+        msg0.close()
+        msg1.close()
 
     def test_iterkeys(self):
         # Get keys using iterkeys()
@@ -789,6 +791,8 @@
 class _TestMboxMMDF(TestMailbox):
 
     def tearDown(self):
+        if os.name == 'nt':
+            time.sleep(0.1) #Allow os to sync files
         self._box.close()
         self._delete_recursively(self._path)
         for lock_remnant in glob.glob(self._path + '.*'):
@@ -1840,7 +1844,9 @@
         self.createMessage("cur")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 1)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
@@ -1848,7 +1854,9 @@
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 1)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
@@ -1857,8 +1865,12 @@
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 2)
-        self.assertIsNot(self.mbox.next(), None)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
@@ -1867,11 +1879,13 @@
         import email.parser
         fname = self.createMessage("cur", True)
         n = 0
-        for msg in mailbox.PortableUnixMailbox(open(fname),
+        fid = open(fname)
+        for msg in mailbox.PortableUnixMailbox(fid,
                                                email.parser.Parser().parse):
             n += 1
             self.assertEqual(msg["subject"], "Simple Test")
             self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+        fid.close()    
         self.assertEqual(n, 1)
 
 ## End: classes from the original module (for backward compatibility).

From noreply at buildbot.pypy.org  Wed Apr 11 15:30:34 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 15:30:34 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: allow capitests to find libpypy-c
Message-ID: <20120411133034.BA58D82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54287:e26139b9710c
Date: 2012-04-11 16:29 +0300
http://bitbucket.org/pypy/pypy/changeset/e26139b9710c/

Log:	allow capitests to find libpypy-c

diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -29,6 +29,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
                         '/EXPORT:init_testcapi']

From noreply at buildbot.pypy.org  Wed Apr 11 15:35:53 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Wed, 11 Apr 2012 15:35:53 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: (DanielHolth): is -> are.
Message-ID: <20120411133553.C226682F4E@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: extradoc
Changeset: r4186:3cd508c8a5c1
Date: 2012-04-11 09:35 -0400
http://bitbucket.org/pypy/extradoc/changeset/3cd508c8a5c1/

Log:	(DanielHolth): is -> are.

diff --git a/blog/draft/pycon-wrapup.rst b/blog/draft/pycon-wrapup.rst
--- a/blog/draft/pycon-wrapup.rst
+++ b/blog/draft/pycon-wrapup.rst
@@ -24,7 +24,7 @@
 We would like to thank everyone who talked to us, shared ideas and especially
 those who participated in sprints - we're always happy to welcome newcomers!
 
-I'm sure there is tons of things I forgot, but thank you all!
+I'm sure there are tons of things I forgot, but thank you all!
 
 Cheers,
 fijal

From noreply at buildbot.pypy.org  Wed Apr 11 15:44:24 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 15:44:24 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge win32-stdlib changes
Message-ID: <20120411134424.9C5B082F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54288:11f147df436c
Date: 2012-04-11 16:43 +0300
http://bitbucket.org/pypy/pypy/changeset/11f147df436c/

Log:	merge win32-stdlib changes

diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py
deleted file mode 100644
--- a/lib-python/2.7/mailbox.py
+++ /dev/null
@@ -1,2171 +0,0 @@
-#! /usr/bin/env python
-
-"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
-
-# Notes for authors of new mailbox subclasses:
-#
-# Remember to fsync() changes to disk before closing a modified file
-# or returning from a flush() method.  See functions _sync_flush() and
-# _sync_close().
-
-import sys
-import os
-import time
-import calendar
-import socket
-import errno
-import copy
-import email
-import email.message
-import email.generator
-import StringIO
-try:
-    if sys.platform == 'os2emx':
-        # OS/2 EMX fcntl() not adequate
-        raise ImportError
-    import fcntl
-except ImportError:
-    fcntl = None
-
-import warnings
-with warnings.catch_warnings():
-    if sys.py3kwarning:
-        warnings.filterwarnings("ignore", ".*rfc822 has been removed",
-                                DeprecationWarning)
-    import rfc822
-
-__all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
-            'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
-            'BabylMessage', 'MMDFMessage', 'UnixMailbox',
-            'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
-
-class Mailbox:
-    """A group of messages in a particular place."""
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize a Mailbox instance."""
-        self._path = os.path.abspath(os.path.expanduser(path))
-        self._factory = factory
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def __delitem__(self, key):
-        self.remove(key)
-
-    def discard(self, key):
-        """If the keyed message exists, remove it."""
-        try:
-            self.remove(key)
-        except KeyError:
-            pass
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def get(self, key, default=None):
-        """Return the keyed message, or default if it doesn't exist."""
-        try:
-            return self.__getitem__(key)
-        except KeyError:
-            return default
-
-    def __getitem__(self, key):
-        """Return the keyed message; raise KeyError if it doesn't exist."""
-        if not self._factory:
-            return self.get_message(key)
-        else:
-            return self._factory(self.get_file(key))
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def get_string(self, key):
-        """Return a string representation or raise a KeyError."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def get_file(self, key):
-        """Return a file-like representation or raise a KeyError."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def iterkeys(self):
-        """Return an iterator over keys."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def keys(self):
-        """Return a list of keys."""
-        return list(self.iterkeys())
-
-    def itervalues(self):
-        """Return an iterator over all messages."""
-        for key in self.iterkeys():
-            try:
-                value = self[key]
-            except KeyError:
-                continue
-            yield value
-
-    def __iter__(self):
-        return self.itervalues()
-
-    def values(self):
-        """Return a list of messages. Memory intensive."""
-        return list(self.itervalues())
-
-    def iteritems(self):
-        """Return an iterator over (key, message) tuples."""
-        for key in self.iterkeys():
-            try:
-                value = self[key]
-            except KeyError:
-                continue
-            yield (key, value)
-
-    def items(self):
-        """Return a list of (key, message) tuples. Memory intensive."""
-        return list(self.iteritems())
-
-    def has_key(self, key):
-        """Return True if the keyed message exists, False otherwise."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def __contains__(self, key):
-        return self.has_key(key)
-
-    def __len__(self):
-        """Return a count of messages in the mailbox."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def clear(self):
-        """Delete all messages."""
-        for key in self.iterkeys():
-            self.discard(key)
-
-    def pop(self, key, default=None):
-        """Delete the keyed message and return it, or default."""
-        try:
-            result = self[key]
-        except KeyError:
-            return default
-        self.discard(key)
-        return result
-
-    def popitem(self):
-        """Delete an arbitrary (key, message) pair and return it."""
-        for key in self.iterkeys():
-            return (key, self.pop(key))     # This is only run once.
-        else:
-            raise KeyError('No messages in mailbox')
-
-    def update(self, arg=None):
-        """Change the messages that correspond to certain keys."""
-        if hasattr(arg, 'iteritems'):
-            source = arg.iteritems()
-        elif hasattr(arg, 'items'):
-            source = arg.items()
-        else:
-            source = arg
-        bad_key = False
-        for key, message in source:
-            try:
-                self[key] = message
-            except KeyError:
-                bad_key = True
-        if bad_key:
-            raise KeyError('No message with key(s)')
-
-    def flush(self):
-        """Write any pending changes to the disk."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def lock(self):
-        """Lock the mailbox."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def unlock(self):
-        """Unlock the mailbox if it is locked."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def close(self):
-        """Flush and close the mailbox."""
-        raise NotImplementedError('Method must be implemented by subclass')
-
-    def _dump_message(self, message, target, mangle_from_=False):
-        # Most files are opened in binary mode to allow predictable seeking.
-        # To get native line endings on disk, the user-friendly \n line endings
-        # used in strings and by email.Message are translated here.
-        """Dump message contents to target file."""
-        if isinstance(message, email.message.Message):
-            buffer = StringIO.StringIO()
-            gen = email.generator.Generator(buffer, mangle_from_, 0)
-            gen.flatten(message)
-            buffer.seek(0)
-            target.write(buffer.read().replace('\n', os.linesep))
-        elif isinstance(message, str):
-            if mangle_from_:
-                message = message.replace('\nFrom ', '\n>From ')
-            message = message.replace('\n', os.linesep)
-            target.write(message)
-        elif hasattr(message, 'read'):
-            while True:
-                line = message.readline()
-                if line == '':
-                    break
-                if mangle_from_ and line.startswith('From '):
-                    line = '>From ' + line[5:]
-                line = line.replace('\n', os.linesep)
-                target.write(line)
-        else:
-            raise TypeError('Invalid message type: %s' % type(message))
-
-
-class Maildir(Mailbox):
-    """A qmail-style Maildir mailbox."""
-
-    colon = ':'
-
-    def __init__(self, dirname, factory=rfc822.Message, create=True):
-        """Initialize a Maildir instance."""
-        Mailbox.__init__(self, dirname, factory, create)
-        self._paths = {
-            'tmp': os.path.join(self._path, 'tmp'),
-            'new': os.path.join(self._path, 'new'),
-            'cur': os.path.join(self._path, 'cur'),
-            }
-        if not os.path.exists(self._path):
-            if create:
-                os.mkdir(self._path, 0700)
-                for path in self._paths.values():
-                    os.mkdir(path, 0o700)
-            else:
-                raise NoSuchMailboxError(self._path)
-        self._toc = {}
-        self._toc_mtimes = {}
-        for subdir in ('cur', 'new'):
-            self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir])
-        self._last_read = time.time()  # Records last time we read cur/new
-        self._skewfactor = 0.1         # Adjust if os/fs clocks are skewing
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        tmp_file = self._create_tmp()
-        try:
-            self._dump_message(message, tmp_file)
-        except BaseException:
-            tmp_file.close()
-            os.remove(tmp_file.name)
-            raise
-        _sync_close(tmp_file)
-        if isinstance(message, MaildirMessage):
-            subdir = message.get_subdir()
-            suffix = self.colon + message.get_info()
-            if suffix == self.colon:
-                suffix = ''
-        else:
-            subdir = 'new'
-            suffix = ''
-        uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
-        dest = os.path.join(self._path, subdir, uniq + suffix)
-        try:
-            if hasattr(os, 'link'):
-                os.link(tmp_file.name, dest)
-                os.remove(tmp_file.name)
-            else:
-                os.rename(tmp_file.name, dest)
-        except OSError, e:
-            os.remove(tmp_file.name)
-            if e.errno == errno.EEXIST:
-                raise ExternalClashError('Name clash with existing message: %s'
-                                         % dest)
-            else:
-                raise
-        if isinstance(message, MaildirMessage):
-            os.utime(dest, (os.path.getatime(dest), message.get_date()))
-        return uniq
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        os.remove(os.path.join(self._path, self._lookup(key)))
-
-    def discard(self, key):
-        """If the keyed message exists, remove it."""
-        # This overrides an inapplicable implementation in the superclass.
-        try:
-            self.remove(key)
-        except KeyError:
-            pass
-        except OSError, e:
-            if e.errno != errno.ENOENT:
-                raise
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        old_subpath = self._lookup(key)
-        temp_key = self.add(message)
-        temp_subpath = self._lookup(temp_key)
-        if isinstance(message, MaildirMessage):
-            # temp's subdir and suffix were specified by message.
-            dominant_subpath = temp_subpath
-        else:
-            # temp's subdir and suffix were defaults from add().
-            dominant_subpath = old_subpath
-        subdir = os.path.dirname(dominant_subpath)
-        if self.colon in dominant_subpath:
-            suffix = self.colon + dominant_subpath.split(self.colon)[-1]
-        else:
-            suffix = ''
-        self.discard(key)
-        new_path = os.path.join(self._path, subdir, key + suffix)
-        os.rename(os.path.join(self._path, temp_subpath), new_path)
-        if isinstance(message, MaildirMessage):
-            os.utime(new_path, (os.path.getatime(new_path),
-                                message.get_date()))
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        subpath = self._lookup(key)
-        f = open(os.path.join(self._path, subpath), 'r')
-        try:
-            if self._factory:
-                msg = self._factory(f)
-            else:
-                msg = MaildirMessage(f)
-        finally:
-            f.close()
-        subdir, name = os.path.split(subpath)
-        msg.set_subdir(subdir)
-        if self.colon in name:
-            msg.set_info(name.split(self.colon)[-1])
-        msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
-        return msg
-
-    def get_string(self, key):
-        """Return a string representation or raise a KeyError."""
-        f = open(os.path.join(self._path, self._lookup(key)), 'r')
-        try:
-            return f.read()
-        finally:
-            f.close()
-
-    def get_file(self, key):
-        """Return a file-like representation or raise a KeyError."""
-        f = open(os.path.join(self._path, self._lookup(key)), 'rb')
-        return _ProxyFile(f)
-
-    def iterkeys(self):
-        """Return an iterator over keys."""
-        self._refresh()
-        for key in self._toc:
-            try:
-                self._lookup(key)
-            except KeyError:
-                continue
-            yield key
-
-    def has_key(self, key):
-        """Return True if the keyed message exists, False otherwise."""
-        self._refresh()
-        return key in self._toc
-
-    def __len__(self):
-        """Return a count of messages in the mailbox."""
-        self._refresh()
-        return len(self._toc)
-
-    def flush(self):
-        """Write any pending changes to disk."""
-        # Maildir changes are always written immediately, so there's nothing
-        # to do.
-        pass
-
-    def lock(self):
-        """Lock the mailbox."""
-        return
-
-    def unlock(self):
-        """Unlock the mailbox if it is locked."""
-        return
-
-    def close(self):
-        """Flush and close the mailbox."""
-        return
-
-    def list_folders(self):
-        """Return a list of folder names."""
-        result = []
-        for entry in os.listdir(self._path):
-            if len(entry) > 1 and entry[0] == '.' and \
-               os.path.isdir(os.path.join(self._path, entry)):
-                result.append(entry[1:])
-        return result
-
-    def get_folder(self, folder):
-        """Return a Maildir instance for the named folder."""
-        return Maildir(os.path.join(self._path, '.' + folder),
-                       factory=self._factory,
-                       create=False)
-
-    def add_folder(self, folder):
-        """Create a folder and return a Maildir instance representing it."""
-        path = os.path.join(self._path, '.' + folder)
-        result = Maildir(path, factory=self._factory)
-        maildirfolder_path = os.path.join(path, 'maildirfolder')
-        if not os.path.exists(maildirfolder_path):
-            os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
-                0666))
-        return result
-
-    def remove_folder(self, folder):
-        """Delete the named folder, which must be empty."""
-        path = os.path.join(self._path, '.' + folder)
-        for entry in os.listdir(os.path.join(path, 'new')) + \
-                     os.listdir(os.path.join(path, 'cur')):
-            if len(entry) < 1 or entry[0] != '.':
-                raise NotEmptyError('Folder contains message(s): %s' % folder)
-        for entry in os.listdir(path):
-            if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
-               os.path.isdir(os.path.join(path, entry)):
-                raise NotEmptyError("Folder contains subdirectory '%s': %s" %
-                                    (folder, entry))
-        for root, dirs, files in os.walk(path, topdown=False):
-            for entry in files:
-                os.remove(os.path.join(root, entry))
-            for entry in dirs:
-                os.rmdir(os.path.join(root, entry))
-        os.rmdir(path)
-
-    def clean(self):
-        """Delete old files in "tmp"."""
-        now = time.time()
-        for entry in os.listdir(os.path.join(self._path, 'tmp')):
-            path = os.path.join(self._path, 'tmp', entry)
-            if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
-                os.remove(path)
-
-    _count = 1  # This is used to generate unique file names.
-
-    def _create_tmp(self):
-        """Create a file in the tmp subdirectory and open and return it."""
-        now = time.time()
-        hostname = socket.gethostname()
-        if '/' in hostname:
-            hostname = hostname.replace('/', r'\057')
-        if ':' in hostname:
-            hostname = hostname.replace(':', r'\072')
-        uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
-                                    Maildir._count, hostname)
-        path = os.path.join(self._path, 'tmp', uniq)
-        try:
-            os.stat(path)
-        except OSError, e:
-            if e.errno == errno.ENOENT:
-                Maildir._count += 1
-                try:
-                    return _create_carefully(path)
-                except OSError, e:
-                    if e.errno != errno.EEXIST:
-                        raise
-            else:
-                raise
-
-        # Fall through to here if stat succeeded or open raised EEXIST.
-        raise ExternalClashError('Name clash prevented file creation: %s' %
-                                 path)
-
-    def _refresh(self):
-        """Update table of contents mapping."""
-        # If it has been less than two seconds since the last _refresh() call,
-        # we have to unconditionally re-read the mailbox just in case it has
-        # been modified, because os.path.mtime() has a 2 sec resolution in the
-        # most common worst case (FAT) and a 1 sec resolution typically.  This
-        # results in a few unnecessary re-reads when _refresh() is called
-        # multiple times in that interval, but once the clock ticks over, we
-        # will only re-read as needed.  Because the filesystem might be being
-        # served by an independent system with its own clock, we record and
-        # compare with the mtimes from the filesystem.  Because the other
-        # system's clock might be skewing relative to our clock, we add an
-        # extra delta to our wait.  The default is one tenth second, but is an
-        # instance variable and so can be adjusted if dealing with a
-        # particularly skewed or irregular system.
-        if time.time() - self._last_read > 2 + self._skewfactor:
-            refresh = False
-            for subdir in self._toc_mtimes:
-                mtime = os.path.getmtime(self._paths[subdir])
-                if mtime > self._toc_mtimes[subdir]:
-                    refresh = True
-                self._toc_mtimes[subdir] = mtime
-            if not refresh:
-                return
-        # Refresh toc
-        self._toc = {}
-        for subdir in self._toc_mtimes:
-            path = self._paths[subdir]
-            for entry in os.listdir(path):
-                p = os.path.join(path, entry)
-                if os.path.isdir(p):
-                    continue
-                uniq = entry.split(self.colon)[0]
-                self._toc[uniq] = os.path.join(subdir, entry)
-        self._last_read = time.time()
-
-    def _lookup(self, key):
-        """Use TOC to return subpath for given key, or raise a KeyError."""
-        try:
-            if os.path.exists(os.path.join(self._path, self._toc[key])):
-                return self._toc[key]
-        except KeyError:
-            pass
-        self._refresh()
-        try:
-            return self._toc[key]
-        except KeyError:
-            raise KeyError('No message with key: %s' % key)
-
-    # This method is for backward compatibility only.
-    def next(self):
-        """Return the next message in a one-time iteration."""
-        if not hasattr(self, '_onetime_keys'):
-            self._onetime_keys = self.iterkeys()
-        while True:
-            try:
-                return self[self._onetime_keys.next()]
-            except StopIteration:
-                return None
-            except KeyError:
-                continue
-
-
-class _singlefileMailbox(Mailbox):
-    """A single-file mailbox."""
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize a single-file mailbox."""
-        Mailbox.__init__(self, path, factory, create)
-        try:
-            f = open(self._path, 'rb+')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                if create:
-                    f = open(self._path, 'wb+')
-                else:
-                    raise NoSuchMailboxError(self._path)
-            elif e.errno in (errno.EACCES, errno.EROFS):
-                f = open(self._path, 'rb')
-            else:
-                raise
-        self._file = f
-        self._toc = None
-        self._next_key = 0
-        self._pending = False   # No changes require rewriting the file.
-        self._locked = False
-        self._file_length = None        # Used to record mailbox size
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        self._lookup()
-        self._toc[self._next_key] = self._append_message(message)
-        self._next_key += 1
-        self._pending = True
-        return self._next_key - 1
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        self._lookup(key)
-        del self._toc[key]
-        self._pending = True
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        self._lookup(key)
-        self._toc[key] = self._append_message(message)
-        self._pending = True
-
-    def iterkeys(self):
-        """Return an iterator over keys."""
-        self._lookup()
-        for key in self._toc.keys():
-            yield key
-
-    def has_key(self, key):
-        """Return True if the keyed message exists, False otherwise."""
-        self._lookup()
-        return key in self._toc
-
-    def __len__(self):
-        """Return a count of messages in the mailbox."""
-        self._lookup()
-        return len(self._toc)
-
-    def lock(self):
-        """Lock the mailbox."""
-        if not self._locked:
-            _lock_file(self._file)
-            self._locked = True
-
-    def unlock(self):
-        """Unlock the mailbox if it is locked."""
-        if self._locked:
-            _unlock_file(self._file)
-            self._locked = False
-
-    def flush(self):
-        """Write any pending changes to disk."""
-        if not self._pending:
-            return
-
-        # In order to be writing anything out at all, self._toc must
-        # already have been generated (and presumably has been modified
-        # by adding or deleting an item).
-        assert self._toc is not None
-
-        # Check length of self._file; if it's changed, some other process
-        # has modified the mailbox since we scanned it.
-        self._file.seek(0, 2)
-        cur_len = self._file.tell()
-        if cur_len != self._file_length:
-            raise ExternalClashError('Size of mailbox file changed '
-                                     '(expected %i, found %i)' %
-                                     (self._file_length, cur_len))
-
-        new_file = _create_temporary(self._path)
-        try:
-            new_toc = {}
-            self._pre_mailbox_hook(new_file)
-            for key in sorted(self._toc.keys()):
-                start, stop = self._toc[key]
-                self._file.seek(start)
-                self._pre_message_hook(new_file)
-                new_start = new_file.tell()
-                while True:
-                    buffer = self._file.read(min(4096,
-                                                 stop - self._file.tell()))
-                    if buffer == '':
-                        break
-                    new_file.write(buffer)
-                new_toc[key] = (new_start, new_file.tell())
-                self._post_message_hook(new_file)
-        except:
-            new_file.close()
-            os.remove(new_file.name)
-            raise
-        _sync_close(new_file)
-        # self._file is about to get replaced, so no need to sync.
-        self._file.close()
-        try:
-            os.rename(new_file.name, self._path)
-        except OSError, e:
-            if e.errno == errno.EEXIST or \
-              (os.name == 'os2' and e.errno == errno.EACCES):
-                os.remove(self._path)
-                os.rename(new_file.name, self._path)
-            else:
-                raise
-        self._file = open(self._path, 'rb+')
-        self._toc = new_toc
-        self._pending = False
-        if self._locked:
-            _lock_file(self._file, dotlock=False)
-
-    def _pre_mailbox_hook(self, f):
-        """Called before writing the mailbox to file f."""
-        return
-
-    def _pre_message_hook(self, f):
-        """Called before writing each message to file f."""
-        return
-
-    def _post_message_hook(self, f):
-        """Called after writing each message to file f."""
-        return
-
-    def close(self):
-        """Flush and close the mailbox."""
-        self.flush()
-        if self._locked:
-            self.unlock()
-        self._file.close()  # Sync has been done by self.flush() above.
-
-    def _lookup(self, key=None):
-        """Return (start, stop) or raise KeyError."""
-        if self._toc is None:
-            self._generate_toc()
-        if key is not None:
-            try:
-                return self._toc[key]
-            except KeyError:
-                raise KeyError('No message with key: %s' % key)
-
-    def _append_message(self, message):
-        """Append message to mailbox and return (start, stop) offsets."""
-        self._file.seek(0, 2)
-        before = self._file.tell()
-        try:
-            self._pre_message_hook(self._file)
-            offsets = self._install_message(message)
-            self._post_message_hook(self._file)
-        except BaseException:
-            self._file.truncate(before)
-            raise
-        self._file.flush()
-        self._file_length = self._file.tell()  # Record current length of mailbox
-        return offsets
-
-
-
-class _mboxMMDF(_singlefileMailbox):
-    """An mbox or MMDF mailbox."""
-
-    _mangle_from_ = True
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        from_line = self._file.readline().replace(os.linesep, '')
-        string = self._file.read(stop - self._file.tell())
-        msg = self._message_factory(string.replace(os.linesep, '\n'))
-        msg.set_from(from_line[5:])
-        return msg
-
-    def get_string(self, key, from_=False):
-        """Return a string representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        if not from_:
-            self._file.readline()
-        string = self._file.read(stop - self._file.tell())
-        return string.replace(os.linesep, '\n')
-
-    def get_file(self, key, from_=False):
-        """Return a file-like representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        if not from_:
-            self._file.readline()
-        return _PartialFile(self._file, self._file.tell(), stop)
-
-    def _install_message(self, message):
-        """Format a message and blindly write to self._file."""
-        from_line = None
-        if isinstance(message, str) and message.startswith('From '):
-            newline = message.find('\n')
-            if newline != -1:
-                from_line = message[:newline]
-                message = message[newline + 1:]
-            else:
-                from_line = message
-                message = ''
-        elif isinstance(message, _mboxMMDFMessage):
-            from_line = 'From ' + message.get_from()
-        elif isinstance(message, email.message.Message):
-            from_line = message.get_unixfrom()  # May be None.
-        if from_line is None:
-            from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
-        start = self._file.tell()
-        self._file.write(from_line + os.linesep)
-        self._dump_message(message, self._file, self._mangle_from_)
-        stop = self._file.tell()
-        return (start, stop)
-
-
-class mbox(_mboxMMDF):
-    """A classic mbox mailbox."""
-
-    _mangle_from_ = True
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize an mbox mailbox."""
-        self._message_factory = mboxMessage
-        _mboxMMDF.__init__(self, path, factory, create)
-
-    def _pre_message_hook(self, f):
-        """Called before writing each message to file f."""
-        if f.tell() != 0:
-            f.write(os.linesep)
-
-    def _generate_toc(self):
-        """Generate key-to-(start, stop) table of contents."""
-        starts, stops = [], []
-        self._file.seek(0)
-        while True:
-            line_pos = self._file.tell()
-            line = self._file.readline()
-            if line.startswith('From '):
-                if len(stops) < len(starts):
-                    stops.append(line_pos - len(os.linesep))
-                starts.append(line_pos)
-            elif line == '':
-                stops.append(line_pos)
-                break
-        self._toc = dict(enumerate(zip(starts, stops)))
-        self._next_key = len(self._toc)
-        self._file_length = self._file.tell()
-
-
-class MMDF(_mboxMMDF):
-    """An MMDF mailbox."""
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize an MMDF mailbox."""
-        self._message_factory = MMDFMessage
-        _mboxMMDF.__init__(self, path, factory, create)
-
-    def _pre_message_hook(self, f):
-        """Called before writing each message to file f."""
-        f.write('\001\001\001\001' + os.linesep)
-
-    def _post_message_hook(self, f):
-        """Called after writing each message to file f."""
-        f.write(os.linesep + '\001\001\001\001' + os.linesep)
-
-    def _generate_toc(self):
-        """Generate key-to-(start, stop) table of contents."""
-        starts, stops = [], []
-        self._file.seek(0)
-        next_pos = 0
-        while True:
-            line_pos = next_pos
-            line = self._file.readline()
-            next_pos = self._file.tell()
-            if line.startswith('\001\001\001\001' + os.linesep):
-                starts.append(next_pos)
-                while True:
-                    line_pos = next_pos
-                    line = self._file.readline()
-                    next_pos = self._file.tell()
-                    if line == '\001\001\001\001' + os.linesep:
-                        stops.append(line_pos - len(os.linesep))
-                        break
-                    elif line == '':
-                        stops.append(line_pos)
-                        break
-            elif line == '':
-                break
-        self._toc = dict(enumerate(zip(starts, stops)))
-        self._next_key = len(self._toc)
-        self._file.seek(0, 2)
-        self._file_length = self._file.tell()
-
-
-class MH(Mailbox):
-    """An MH mailbox."""
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize an MH instance."""
-        Mailbox.__init__(self, path, factory, create)
-        if not os.path.exists(self._path):
-            if create:
-                os.mkdir(self._path, 0700)
-                os.close(os.open(os.path.join(self._path, '.mh_sequences'),
-                                 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
-            else:
-                raise NoSuchMailboxError(self._path)
-        self._locked = False
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        keys = self.keys()
-        if len(keys) == 0:
-            new_key = 1
-        else:
-            new_key = max(keys) + 1
-        new_path = os.path.join(self._path, str(new_key))
-        f = _create_carefully(new_path)
-        closed = False
-        try:
-            if self._locked:
-                _lock_file(f)
-            try:
-                try:
-                    self._dump_message(message, f)
-                except BaseException:
-                    # Unlock and close so it can be deleted on Windows
-                    if self._locked:
-                        _unlock_file(f)
-                    _sync_close(f)
-                    closed = True
-                    os.remove(new_path)
-                    raise
-                if isinstance(message, MHMessage):
-                    self._dump_sequences(message, new_key)
-            finally:
-                if self._locked:
-                    _unlock_file(f)
-        finally:
-            if not closed:
-                _sync_close(f)
-        return new_key
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        path = os.path.join(self._path, str(key))
-        try:
-            f = open(path, 'rb+')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        else:
-            f.close()
-            os.remove(path)
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        path = os.path.join(self._path, str(key))
-        try:
-            f = open(path, 'rb+')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        try:
-            if self._locked:
-                _lock_file(f)
-            try:
-                os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
-                self._dump_message(message, f)
-                if isinstance(message, MHMessage):
-                    self._dump_sequences(message, key)
-            finally:
-                if self._locked:
-                    _unlock_file(f)
-        finally:
-            _sync_close(f)
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        try:
-            if self._locked:
-                f = open(os.path.join(self._path, str(key)), 'r+')
-            else:
-                f = open(os.path.join(self._path, str(key)), 'r')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        try:
-            if self._locked:
-                _lock_file(f)
-            try:
-                msg = MHMessage(f)
-            finally:
-                if self._locked:
-                    _unlock_file(f)
-        finally:
-            f.close()
-        for name, key_list in self.get_sequences().iteritems():
-            if key in key_list:
-                msg.add_sequence(name)
-        return msg
-
-    def get_string(self, key):
-        """Return a string representation or raise a KeyError."""
-        try:
-            if self._locked:
-                f = open(os.path.join(self._path, str(key)), 'r+')
-            else:
-                f = open(os.path.join(self._path, str(key)), 'r')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        try:
-            if self._locked:
-                _lock_file(f)
-            try:
-                return f.read()
-            finally:
-                if self._locked:
-                    _unlock_file(f)
-        finally:
-            f.close()
-
-    def get_file(self, key):
-        """Return a file-like representation or raise a KeyError."""
-        try:
-            f = open(os.path.join(self._path, str(key)), 'rb')
-        except IOError, e:
-            if e.errno == errno.ENOENT:
-                raise KeyError('No message with key: %s' % key)
-            else:
-                raise
-        return _ProxyFile(f)
-
-    def iterkeys(self):
-        """Return an iterator over keys."""
-        return iter(sorted(int(entry) for entry in os.listdir(self._path)
-                                      if entry.isdigit()))
-
-    def has_key(self, key):
-        """Return True if the keyed message exists, False otherwise."""
-        return os.path.exists(os.path.join(self._path, str(key)))
-
-    def __len__(self):
-        """Return a count of messages in the mailbox."""
-        return len(list(self.iterkeys()))
-
-    def lock(self):
-        """Lock the mailbox."""
-        if not self._locked:
-            self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
-            _lock_file(self._file)
-            self._locked = True
-
-    def unlock(self):
-        """Unlock the mailbox if it is locked."""
-        if self._locked:
-            _unlock_file(self._file)
-            _sync_close(self._file)
-            del self._file
-            self._locked = False
-
-    def flush(self):
-        """Write any pending changes to the disk."""
-        return
-
-    def close(self):
-        """Flush and close the mailbox."""
-        if self._locked:
-            self.unlock()
-
-    def list_folders(self):
-        """Return a list of folder names."""
-        result = []
-        for entry in os.listdir(self._path):
-            if os.path.isdir(os.path.join(self._path, entry)):
-                result.append(entry)
-        return result
-
-    def get_folder(self, folder):
-        """Return an MH instance for the named folder."""
-        return MH(os.path.join(self._path, folder),
-                  factory=self._factory, create=False)
-
-    def add_folder(self, folder):
-        """Create a folder and return an MH instance representing it."""
-        return MH(os.path.join(self._path, folder),
-                  factory=self._factory)
-
-    def remove_folder(self, folder):
-        """Delete the named folder, which must be empty."""
-        path = os.path.join(self._path, folder)
-        entries = os.listdir(path)
-        if entries == ['.mh_sequences']:
-            os.remove(os.path.join(path, '.mh_sequences'))
-        elif entries == []:
-            pass
-        else:
-            raise NotEmptyError('Folder not empty: %s' % self._path)
-        os.rmdir(path)
-
-    def get_sequences(self):
-        """Return a name-to-key-list dictionary to define each sequence."""
-        results = {}
-        f = open(os.path.join(self._path, '.mh_sequences'), 'r')
-        try:
-            all_keys = set(self.keys())
-            for line in f:
-                try:
-                    name, contents = line.split(':')
-                    keys = set()
-                    for spec in contents.split():
-                        if spec.isdigit():
-                            keys.add(int(spec))
-                        else:
-                            start, stop = (int(x) for x in spec.split('-'))
-                            keys.update(range(start, stop + 1))
-                    results[name] = [key for key in sorted(keys) \
-                                         if key in all_keys]
-                    if len(results[name]) == 0:
-                        del results[name]
-                except ValueError:
-                    raise FormatError('Invalid sequence specification: %s' %
-                                      line.rstrip())
-        finally:
-            f.close()
-        return results
-
-    def set_sequences(self, sequences):
-        """Set sequences using the given name-to-key-list dictionary."""
-        f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
-        try:
-            os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
-            for name, keys in sequences.iteritems():
-                if len(keys) == 0:
-                    continue
-                f.write('%s:' % name)
-                prev = None
-                completing = False
-                for key in sorted(set(keys)):
-                    if key - 1 == prev:
-                        if not completing:
-                            completing = True
-                            f.write('-')
-                    elif completing:
-                        completing = False
-                        f.write('%s %s' % (prev, key))
-                    else:
-                        f.write(' %s' % key)
-                    prev = key
-                if completing:
-                    f.write(str(prev) + '\n')
-                else:
-                    f.write('\n')
-        finally:
-            _sync_close(f)
-
-    def pack(self):
-        """Re-name messages to eliminate numbering gaps. Invalidates keys."""
-        sequences = self.get_sequences()
-        prev = 0
-        changes = []
-        for key in self.iterkeys():
-            if key - 1 != prev:
-                changes.append((key, prev + 1))
-                if hasattr(os, 'link'):
-                    os.link(os.path.join(self._path, str(key)),
-                            os.path.join(self._path, str(prev + 1)))
-                    os.unlink(os.path.join(self._path, str(key)))
-                else:
-                    os.rename(os.path.join(self._path, str(key)),
-                              os.path.join(self._path, str(prev + 1)))
-            prev += 1
-        self._next_key = prev + 1
-        if len(changes) == 0:
-            return
-        for name, key_list in sequences.items():
-            for old, new in changes:
-                if old in key_list:
-                    key_list[key_list.index(old)] = new
-        self.set_sequences(sequences)
-
-    def _dump_sequences(self, message, key):
-        """Inspect a new MHMessage and update sequences appropriately."""
-        pending_sequences = message.get_sequences()
-        all_sequences = self.get_sequences()
-        for name, key_list in all_sequences.iteritems():
-            if name in pending_sequences:
-                key_list.append(key)
-            elif key in key_list:
-                del key_list[key_list.index(key)]
-        for sequence in pending_sequences:
-            if sequence not in all_sequences:
-                all_sequences[sequence] = [key]
-        self.set_sequences(all_sequences)
-
-
-class Babyl(_singlefileMailbox):
-    """An Rmail-style Babyl mailbox."""
-
-    _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
-                                 'forwarded', 'edited', 'resent'))
-
-    def __init__(self, path, factory=None, create=True):
-        """Initialize a Babyl mailbox."""
-        _singlefileMailbox.__init__(self, path, factory, create)
-        self._labels = {}
-
-    def add(self, message):
-        """Add message and return assigned key."""
-        key = _singlefileMailbox.add(self, message)
-        if isinstance(message, BabylMessage):
-            self._labels[key] = message.get_labels()
-        return key
-
-    def remove(self, key):
-        """Remove the keyed message; raise KeyError if it doesn't exist."""
-        _singlefileMailbox.remove(self, key)
-        if key in self._labels:
-            del self._labels[key]
-
-    def __setitem__(self, key, message):
-        """Replace the keyed message; raise KeyError if it doesn't exist."""
-        _singlefileMailbox.__setitem__(self, key, message)
-        if isinstance(message, BabylMessage):
-            self._labels[key] = message.get_labels()
-
-    def get_message(self, key):
-        """Return a Message representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        self._file.readline()   # Skip '1,' line specifying labels.
-        original_headers = StringIO.StringIO()
-        while True:
-            line = self._file.readline()
-            if line == '*** EOOH ***' + os.linesep or line == '':
-                break
-            original_headers.write(line.replace(os.linesep, '\n'))
-        visible_headers = StringIO.StringIO()
-        while True:
-            line = self._file.readline()
-            if line == os.linesep or line == '':
-                break
-            visible_headers.write(line.replace(os.linesep, '\n'))
-        body = self._file.read(stop - self._file.tell()).replace(os.linesep,
-                                                                 '\n')
-        msg = BabylMessage(original_headers.getvalue() + body)
-        msg.set_visible(visible_headers.getvalue())
-        if key in self._labels:
-            msg.set_labels(self._labels[key])
-        return msg
-
-    def get_string(self, key):
-        """Return a string representation or raise a KeyError."""
-        start, stop = self._lookup(key)
-        self._file.seek(start)
-        self._file.readline()   # Skip '1,' line specifying labels.
-        original_headers = StringIO.StringIO()
-        while True:
-            line = self._file.readline()
-            if line == '*** EOOH ***' + os.linesep or line == '':
-                break
-            original_headers.write(line.replace(os.linesep, '\n'))
-        while True:
-            line = self._file.readline()
-            if line == os.linesep or line == '':
-                break
-        return original_headers.getvalue() + \
-               self._file.read(stop - self._file.tell()).replace(os.linesep,
-                                                                 '\n')
-
-    def get_file(self, key):
-        """Return a file-like representation or raise a KeyError."""
-        return StringIO.StringIO(self.get_string(key).replace('\n',
-                                                              os.linesep))
-
-    def get_labels(self):
-        """Return a list of user-defined labels in the mailbox."""
-        self._lookup()
-        labels = set()
-        for label_list in self._labels.values():
-            labels.update(label_list)
-        labels.difference_update(self._special_labels)
-        return list(labels)
-
-    def _generate_toc(self):
-        """Generate key-to-(start, stop) table of contents."""
-        starts, stops = [], []
-        self._file.seek(0)
-        next_pos = 0
-        label_lists = []
-        while True:
-            line_pos = next_pos
-            line = self._file.readline()
-            next_pos = self._file.tell()
-            if line == '\037\014' + os.linesep:
-                if len(stops) < len(starts):
-                    stops.append(line_pos - len(os.linesep))
-                starts.append(next_pos)
-                labels = [label.strip() for label
-                                        in self._file.readline()[1:].split(',')
-                                        if label.strip() != '']
-                label_lists.append(labels)
-            elif line == '\037' or line == '\037' + os.linesep:
-                if len(stops) < len(starts):
-                    stops.append(line_pos - len(os.linesep))
-            elif line == '':
-                stops.append(line_pos - len(os.linesep))
-                break
-        self._toc = dict(enumerate(zip(starts, stops)))
-        self._labels = dict(enumerate(label_lists))
-        self._next_key = len(self._toc)
-        self._file.seek(0, 2)
-        self._file_length = self._file.tell()
-
-    def _pre_mailbox_hook(self, f):
-        """Called before writing the mailbox to file f."""
-        f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
-                (os.linesep, os.linesep, ','.join(self.get_labels()),
-                 os.linesep))
-
-    def _pre_message_hook(self, f):
-        """Called before writing each message to file f."""
-        f.write('\014' + os.linesep)
-
-    def _post_message_hook(self, f):
-        """Called after writing each message to file f."""
-        f.write(os.linesep + '\037')
-
-    def _install_message(self, message):
-        """Write message contents and return (start, stop)."""
-        start = self._file.tell()
-        if isinstance(message, BabylMessage):
-            special_labels = []
-            labels = []
-            for label in message.get_labels():
-                if label in self._special_labels:
-                    special_labels.append(label)
-                else:
-                    labels.append(label)
-            self._file.write('1')
-            for label in special_labels:
-                self._file.write(', ' + label)
-            self._file.write(',,')
-            for label in labels:
-                self._file.write(' ' + label + ',')
-            self._file.write(os.linesep)
-        else:
-            self._file.write('1,,' + os.linesep)
-        if isinstance(message, email.message.Message):
-            orig_buffer = StringIO.StringIO()
-            orig_generator = email.generator.Generator(orig_buffer, False, 0)
-            orig_generator.flatten(message)
-            orig_buffer.seek(0)
-            while True:
-                line = orig_buffer.readline()
-                self._file.write(line.replace('\n', os.linesep))
-                if line == '\n' or line == '':
-                    break
-            self._file.write('*** EOOH ***' + os.linesep)
-            if isinstance(message, BabylMessage):
-                vis_buffer = StringIO.StringIO()
-                vis_generator = email.generator.Generator(vis_buffer, False, 0)
-                vis_generator.flatten(message.get_visible())
-                while True:
-                    line = vis_buffer.readline()
-                    self._file.write(line.replace('\n', os.linesep))
-                    if line == '\n' or line == '':
-                        break
-            else:
-                orig_buffer.seek(0)
-                while True:
-                    line = orig_buffer.readline()
-                    self._file.write(line.replace('\n', os.linesep))
-                    if line == '\n' or line == '':
-                        break
-            while True:
-                buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
-                if buffer == '':
-                    break
-                self._file.write(buffer.replace('\n', os.linesep))
-        elif isinstance(message, str):
-            body_start = message.find('\n\n') + 2
-            if body_start - 2 != -1:
-                self._file.write(message[:body_start].replace('\n',
-                                                              os.linesep))
-                self._file.write('*** EOOH ***' + os.linesep)
-                self._file.write(message[:body_start].replace('\n',
-                                                              os.linesep))
-                self._file.write(message[body_start:].replace('\n',
-                                                              os.linesep))
-            else:
-                self._file.write('*** EOOH ***' + os.linesep + os.linesep)
-                self._file.write(message.replace('\n', os.linesep))
-        elif hasattr(message, 'readline'):
-            original_pos = message.tell()
-            first_pass = True
-            while True:
-                line = message.readline()
-                self._file.write(line.replace('\n', os.linesep))
-                if line == '\n' or line == '':
-                    self._file.write('*** EOOH ***' + os.linesep)
-                    if first_pass:
-                        first_pass = False
-                        message.seek(original_pos)
-                    else:
-                        break
-            while True:
-                buffer = message.read(4096)     # Buffer size is arbitrary.
-                if buffer == '':
-                    break
-                self._file.write(buffer.replace('\n', os.linesep))
-        else:
-            raise TypeError('Invalid message type: %s' % type(message))
-        stop = self._file.tell()
-        return (start, stop)
-
-
-class Message(email.message.Message):
-    """Message with mailbox-format-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize a Message instance."""
-        if isinstance(message, email.message.Message):
-            self._become_message(copy.deepcopy(message))
-            if isinstance(message, Message):
-                message._explain_to(self)
-        elif isinstance(message, str):
-            self._become_message(email.message_from_string(message))
-        elif hasattr(message, "read"):
-            self._become_message(email.message_from_file(message))
-        elif message is None:
-            email.message.Message.__init__(self)
-        else:
-            raise TypeError('Invalid message type: %s' % type(message))
-
-    def _become_message(self, message):
-        """Assume the non-format-specific state of message."""
-        for name in ('_headers', '_unixfrom', '_payload', '_charset',
-                     'preamble', 'epilogue', 'defects', '_default_type'):
-            self.__dict__[name] = message.__dict__[name]
-
-    def _explain_to(self, message):
-        """Copy format-specific state to message insofar as possible."""
-        if isinstance(message, Message):
-            return  # There's nothing format-specific to explain.
-        else:
-            raise TypeError('Cannot convert to specified type')
-
-
-class MaildirMessage(Message):
-    """Message with Maildir-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize a MaildirMessage instance."""
-        self._subdir = 'new'
-        self._info = ''
-        self._date = time.time()
-        Message.__init__(self, message)
-
-    def get_subdir(self):
-        """Return 'new' or 'cur'."""
-        return self._subdir
-
-    def set_subdir(self, subdir):
-        """Set subdir to 'new' or 'cur'."""
-        if subdir == 'new' or subdir == 'cur':
-            self._subdir = subdir
-        else:
-            raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
-
-    def get_flags(self):
-        """Return as a string the flags that are set."""
-        if self._info.startswith('2,'):
-            return self._info[2:]
-        else:
-            return ''
-
-    def set_flags(self, flags):
-        """Set the given flags and unset all others."""
-        self._info = '2,' + ''.join(sorted(flags))
-
-    def add_flag(self, flag):
-        """Set the given flag(s) without changing others."""
-        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
-
-    def remove_flag(self, flag):
-        """Unset the given string flag(s) without changing others."""
-        if self.get_flags() != '':
-            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
-
-    def get_date(self):
-        """Return delivery date of message, in seconds since the epoch."""
-        return self._date
-
-    def set_date(self, date):
-        """Set delivery date of message, in seconds since the epoch."""
-        try:
-            self._date = float(date)
-        except ValueError:
-            raise TypeError("can't convert to float: %s" % date)
-
-    def get_info(self):
-        """Get the message's "info" as a string."""
-        return self._info
-
-    def set_info(self, info):
-        """Set the message's "info" string."""
-        if isinstance(info, str):
-            self._info = info
-        else:
-            raise TypeError('info must be a string: %s' % type(info))
-
-    def _explain_to(self, message):
-        """Copy Maildir-specific state to message insofar as possible."""
-        if isinstance(message, MaildirMessage):
-            message.set_flags(self.get_flags())
-            message.set_subdir(self.get_subdir())
-            message.set_date(self.get_date())
-        elif isinstance(message, _mboxMMDFMessage):
-            flags = set(self.get_flags())
-            if 'S' in flags:
-                message.add_flag('R')
-            if self.get_subdir() == 'cur':
-                message.add_flag('O')
-            if 'T' in flags:
-                message.add_flag('D')
-            if 'F' in flags:
-                message.add_flag('F')
-            if 'R' in flags:
-                message.add_flag('A')
-            message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
-        elif isinstance(message, MHMessage):
-            flags = set(self.get_flags())
-            if 'S' not in flags:
-                message.add_sequence('unseen')
-            if 'R' in flags:
-                message.add_sequence('replied')
-            if 'F' in flags:
-                message.add_sequence('flagged')
-        elif isinstance(message, BabylMessage):
-            flags = set(self.get_flags())
-            if 'S' not in flags:
-                message.add_label('unseen')
-            if 'T' in flags:
-                message.add_label('deleted')
-            if 'R' in flags:
-                message.add_label('answered')
-            if 'P' in flags:
-                message.add_label('forwarded')
-        elif isinstance(message, Message):
-            pass
-        else:
-            raise TypeError('Cannot convert to specified type: %s' %
-                            type(message))
-
-
-class _mboxMMDFMessage(Message):
-    """Message with mbox- or MMDF-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize an mboxMMDFMessage instance."""
-        self.set_from('MAILER-DAEMON', True)
-        if isinstance(message, email.message.Message):
-            unixfrom = message.get_unixfrom()
-            if unixfrom is not None and unixfrom.startswith('From '):
-                self.set_from(unixfrom[5:])
-        Message.__init__(self, message)
-
-    def get_from(self):
-        """Return contents of "From " line."""
-        return self._from
-
-    def set_from(self, from_, time_=None):
-        """Set "From " line, formatting and appending time_ if specified."""
-        if time_ is not None:
-            if time_ is True:
-                time_ = time.gmtime()
-            from_ += ' ' + time.asctime(time_)
-        self._from = from_
-
-    def get_flags(self):
-        """Return as a string the flags that are set."""
-        return self.get('Status', '') + self.get('X-Status', '')
-
-    def set_flags(self, flags):
-        """Set the given flags and unset all others."""
-        flags = set(flags)
-        status_flags, xstatus_flags = '', ''
-        for flag in ('R', 'O'):
-            if flag in flags:
-                status_flags += flag
-                flags.remove(flag)
-        for flag in ('D', 'F', 'A'):
-            if flag in flags:
-                xstatus_flags += flag
-                flags.remove(flag)
-        xstatus_flags += ''.join(sorted(flags))
-        try:
-            self.replace_header('Status', status_flags)
-        except KeyError:
-            self.add_header('Status', status_flags)
-        try:
-            self.replace_header('X-Status', xstatus_flags)
-        except KeyError:
-            self.add_header('X-Status', xstatus_flags)
-
-    def add_flag(self, flag):
-        """Set the given flag(s) without changing others."""
-        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
-
-    def remove_flag(self, flag):
-        """Unset the given string flag(s) without changing others."""
-        if 'Status' in self or 'X-Status' in self:
-            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
-
-    def _explain_to(self, message):
-        """Copy mbox- or MMDF-specific state to message insofar as possible."""
-        if isinstance(message, MaildirMessage):
-            flags = set(self.get_flags())
-            if 'O' in flags:
-                message.set_subdir('cur')
-            if 'F' in flags:
-                message.add_flag('F')
-            if 'A' in flags:
-                message.add_flag('R')
-            if 'R' in flags:
-                message.add_flag('S')
-            if 'D' in flags:
-                message.add_flag('T')
-            del message['status']
-            del message['x-status']
-            maybe_date = ' '.join(self.get_from().split()[-5:])
-            try:
-                message.set_date(calendar.timegm(time.strptime(maybe_date,
-                                                      '%a %b %d %H:%M:%S %Y')))
-            except (ValueError, OverflowError):
-                pass
-        elif isinstance(message, _mboxMMDFMessage):
-            message.set_flags(self.get_flags())
-            message.set_from(self.get_from())
-        elif isinstance(message, MHMessage):
-            flags = set(self.get_flags())
-            if 'R' not in flags:
-                message.add_sequence('unseen')
-            if 'A' in flags:
-                message.add_sequence('replied')
-            if 'F' in flags:
-                message.add_sequence('flagged')
-            del message['status']
-            del message['x-status']
-        elif isinstance(message, BabylMessage):
-            flags = set(self.get_flags())
-            if 'R' not in flags:
-                message.add_label('unseen')
-            if 'D' in flags:
-                message.add_label('deleted')
-            if 'A' in flags:
-                message.add_label('answered')
-            del message['status']
-            del message['x-status']
-        elif isinstance(message, Message):
-            pass
-        else:
-            raise TypeError('Cannot convert to specified type: %s' %
-                            type(message))
-
-
-class mboxMessage(_mboxMMDFMessage):
-    """Message with mbox-specific properties."""
-
-
-class MHMessage(Message):
-    """Message with MH-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize an MHMessage instance."""
-        self._sequences = []
-        Message.__init__(self, message)
-
-    def get_sequences(self):
-        """Return a list of sequences that include the message."""
-        return self._sequences[:]
-
-    def set_sequences(self, sequences):
-        """Set the list of sequences that include the message."""
-        self._sequences = list(sequences)
-
-    def add_sequence(self, sequence):
-        """Add sequence to list of sequences including the message."""
-        if isinstance(sequence, str):
-            if not sequence in self._sequences:
-                self._sequences.append(sequence)
-        else:
-            raise TypeError('sequence must be a string: %s' % type(sequence))
-
-    def remove_sequence(self, sequence):
-        """Remove sequence from the list of sequences including the message."""
-        try:
-            self._sequences.remove(sequence)
-        except ValueError:
-            pass
-
-    def _explain_to(self, message):
-        """Copy MH-specific state to message insofar as possible."""
-        if isinstance(message, MaildirMessage):
-            sequences = set(self.get_sequences())
-            if 'unseen' in sequences:
-                message.set_subdir('cur')
-            else:
-                message.set_subdir('cur')
-                message.add_flag('S')
-            if 'flagged' in sequences:
-                message.add_flag('F')
-            if 'replied' in sequences:
-                message.add_flag('R')
-        elif isinstance(message, _mboxMMDFMessage):
-            sequences = set(self.get_sequences())
-            if 'unseen' not in sequences:
-                message.add_flag('RO')
-            else:
-                message.add_flag('O')
-            if 'flagged' in sequences:
-                message.add_flag('F')
-            if 'replied' in sequences:
-                message.add_flag('A')
-        elif isinstance(message, MHMessage):
-            for sequence in self.get_sequences():
-                message.add_sequence(sequence)
-        elif isinstance(message, BabylMessage):
-            sequences = set(self.get_sequences())
-            if 'unseen' in sequences:
-                message.add_label('unseen')
-            if 'replied' in sequences:
-                message.add_label('answered')
-        elif isinstance(message, Message):
-            pass
-        else:
-            raise TypeError('Cannot convert to specified type: %s' %
-                            type(message))
-
-
-class BabylMessage(Message):
-    """Message with Babyl-specific properties."""
-
-    def __init__(self, message=None):
-        """Initialize an BabylMessage instance."""
-        self._labels = []
-        self._visible = Message()
-        Message.__init__(self, message)
-
-    def get_labels(self):
-        """Return a list of labels on the message."""
-        return self._labels[:]
-
-    def set_labels(self, labels):
-        """Set the list of labels on the message."""
-        self._labels = list(labels)
-
-    def add_label(self, label):
-        """Add label to list of labels on the message."""
-        if isinstance(label, str):
-            if label not in self._labels:
-                self._labels.append(label)
-        else:
-            raise TypeError('label must be a string: %s' % type(label))
-
-    def remove_label(self, label):
-        """Remove label from the list of labels on the message."""
-        try:
-            self._labels.remove(label)
-        except ValueError:
-            pass
-
-    def get_visible(self):
-        """Return a Message representation of visible headers."""
-        return Message(self._visible)
-
-    def set_visible(self, visible):
-        """Set the Message representation of visible headers."""
-        self._visible = Message(visible)
-
-    def update_visible(self):
-        """Update and/or sensibly generate a set of visible headers."""
-        for header in self._visible.keys():
-            if header in self:
-                self._visible.replace_header(header, self[header])
-            else:
-                del self._visible[header]
-        for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
-            if header in self and header not in self._visible:
-                self._visible[header] = self[header]
-
-    def _explain_to(self, message):
-        """Copy Babyl-specific state to message insofar as possible."""
-        if isinstance(message, MaildirMessage):
-            labels = set(self.get_labels())
-            if 'unseen' in labels:
-                message.set_subdir('cur')
-            else:
-                message.set_subdir('cur')
-                message.add_flag('S')
-            if 'forwarded' in labels or 'resent' in labels:
-                message.add_flag('P')
-            if 'answered' in labels:
-                message.add_flag('R')
-            if 'deleted' in labels:
-                message.add_flag('T')
-        elif isinstance(message, _mboxMMDFMessage):
-            labels = set(self.get_labels())
-            if 'unseen' not in labels:
-                message.add_flag('RO')
-            else:
-                message.add_flag('O')
-            if 'deleted' in labels:
-                message.add_flag('D')
-            if 'answered' in labels:
-                message.add_flag('A')
-        elif isinstance(message, MHMessage):
-            labels = set(self.get_labels())
-            if 'unseen' in labels:
-                message.add_sequence('unseen')
-            if 'answered' in labels:
-                message.add_sequence('replied')
-        elif isinstance(message, BabylMessage):
-            message.set_visible(self.get_visible())
-            for label in self.get_labels():
-                message.add_label(label)
-        elif isinstance(message, Message):
-            pass
-        else:
-            raise TypeError('Cannot convert to specified type: %s' %
-                            type(message))
-
-
-class MMDFMessage(_mboxMMDFMessage):
-    """Message with MMDF-specific properties."""
-
-
-class _ProxyFile:
-    """A read-only wrapper of a file."""
-
-    def __init__(self, f, pos=None):
-        """Initialize a _ProxyFile."""
-        self._file = f
-        if pos is None:
-            self._pos = f.tell()
-        else:
-            self._pos = pos
-
-    def read(self, size=None):
-        """Read bytes."""
-        return self._read(size, self._file.read)
-
-    def readline(self, size=None):
-        """Read a line."""
-        return self._read(size, self._file.readline)
-
-    def readlines(self, sizehint=None):
-        """Read multiple lines."""
-        result = []
-        for line in self:
-            result.append(line)
-            if sizehint is not None:
-                sizehint -= len(line)
-                if sizehint <= 0:
-                    break
-        return result
-
-    def __iter__(self):
-        """Iterate over lines."""
-        return iter(self.readline, "")
-
-    def tell(self):
-        """Return the position."""
-        return self._pos
-
-    def seek(self, offset, whence=0):
-        """Change position."""
-        if whence == 1:
-            self._file.seek(self._pos)
-        self._file.seek(offset, whence)
-        self._pos = self._file.tell()
-
-    def close(self):
-        """Close the file."""
-        del self._file
-
-    def _read(self, size, read_method):
-        """Read size bytes using read_method."""
-        if size is None:
-            size = -1
-        self._file.seek(self._pos)
-        result = read_method(size)
-        self._pos = self._file.tell()
-        return result
-
-
-class _PartialFile(_ProxyFile):
-    """A read-only wrapper of part of a file."""
-
-    def __init__(self, f, start=None, stop=None):
-        """Initialize a _PartialFile."""
-        _ProxyFile.__init__(self, f, start)
-        self._start = start
-        self._stop = stop
-
-    def tell(self):
-        """Return the position with respect to start."""
-        return _ProxyFile.tell(self) - self._start
-
-    def seek(self, offset, whence=0):
-        """Change position, possibly with respect to start or stop."""
-        if whence == 0:
-            self._pos = self._start
-            whence = 1
-        elif whence == 2:
-            self._pos = self._stop
-            whence = 1
-        _ProxyFile.seek(self, offset, whence)
-
-    def _read(self, size, read_method):
-        """Read size bytes using read_method, honoring start and stop."""
-        remaining = self._stop - self._pos
-        if remaining <= 0:
-            return ''
-        if size is None or size < 0 or size > remaining:
-            size = remaining
-        return _ProxyFile._read(self, size, read_method)
-
-
-def _lock_file(f, dotlock=True):
-    """Lock file f using lockf and dot locking."""
-    dotlock_done = False
-    try:
-        if fcntl:
-            try:
-                fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
-            except IOError, e:
-                if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
-                    raise ExternalClashError('lockf: lock unavailable: %s' %
-                                             f.name)
-                else:
-                    raise
-        if dotlock:
-            try:
-                pre_lock = _create_temporary(f.name + '.lock')
-                pre_lock.close()
-            except IOError, e:
-                if e.errno in (errno.EACCES, errno.EROFS):
-                    return  # Without write access, just skip dotlocking.
-                else:
-                    raise
-            try:
-                if hasattr(os, 'link'):
-                    os.link(pre_lock.name, f.name + '.lock')
-                    dotlock_done = True
-                    os.unlink(pre_lock.name)
-                else:
-                    os.rename(pre_lock.name, f.name + '.lock')
-                    dotlock_done = True
-            except OSError, e:
-                if e.errno == errno.EEXIST or \
-                  (os.name == 'os2' and e.errno == errno.EACCES):
-                    os.remove(pre_lock.name)
-                    raise ExternalClashError('dot lock unavailable: %s' %
-                                             f.name)
-                else:
-                    raise
-    except:
-        if fcntl:
-            fcntl.lockf(f, fcntl.LOCK_UN)
-        if dotlock_done:
-            os.remove(f.name + '.lock')
-        raise
-
-def _unlock_file(f):
-    """Unlock file f using lockf and dot locking."""
-    if fcntl:
-        fcntl.lockf(f, fcntl.LOCK_UN)
-    if os.path.exists(f.name + '.lock'):
-        os.remove(f.name + '.lock')
-
-def _create_carefully(path):
-    """Create a file if it doesn't exist and open for reading and writing."""
-    fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
-    try:
-        return open(path, 'rb+')
-    finally:
-        os.close(fd)
-
-def _create_temporary(path):
-    """Create a temp file based on path and open for reading and writing."""
-    return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
-                                              socket.gethostname(),
-                                              os.getpid()))
-
-def _sync_flush(f):
-    """Ensure changes to file f are physically on disk."""
-    f.flush()
-    if hasattr(os, 'fsync'):
-        os.fsync(f.fileno())
-
-def _sync_close(f):
-    """Close file f, ensuring all changes are physically on disk."""
-    _sync_flush(f)
-    f.close()
-
-## Start: classes from the original module (for backward compatibility).
-
-# Note that the Maildir class, whose name is unchanged, itself offers a next()
-# method for backward compatibility.
-
-class _Mailbox:
-
-    def __init__(self, fp, factory=rfc822.Message):
-        self.fp = fp
-        self.seekp = 0
-        self.factory = factory
-
-    def __iter__(self):
-        return iter(self.next, None)
-
-    def next(self):
-        while 1:
-            self.fp.seek(self.seekp)
-            try:
-                self._search_start()
-            except EOFError:
-                self.seekp = self.fp.tell()
-                return None
-            start = self.fp.tell()
-            self._search_end()
-            self.seekp = stop = self.fp.tell()
-            if start != stop:
-                break
-        return self.factory(_PartialFile(self.fp, start, stop))
-
-# Recommended to use PortableUnixMailbox instead!
-class UnixMailbox(_Mailbox):
-
-    def _search_start(self):
-        while 1:
-            pos = self.fp.tell()
-            line = self.fp.readline()
-            if not line:
-                raise EOFError
-            if line[:5] == 'From ' and self._isrealfromline(line):
-                self.fp.seek(pos)
-                return
-
-    def _search_end(self):
-        self.fp.readline()      # Throw away header line
-        while 1:
-            pos = self.fp.tell()
-            line = self.fp.readline()
-            if not line:
-                return
-            if line[:5] == 'From ' and self._isrealfromline(line):
-                self.fp.seek(pos)
-                return
-
-    # An overridable mechanism to test for From-line-ness.  You can either
-    # specify a different regular expression or define a whole new
-    # _isrealfromline() method.  Note that this only gets called for lines
-    # starting with the 5 characters "From ".
-    #
-    # BAW: According to
-    #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
-    # the only portable, reliable way to find message delimiters in a BSD (i.e
-    # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
-    # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
-    # like a good idea, in practice, there are too many variations for more
-    # strict parsing of the line to be completely accurate.
-    #
-    # _strict_isrealfromline() is the old version which tries to do stricter
-    # parsing of the From_ line.  _portable_isrealfromline() simply returns
-    # true, since it's never called if the line doesn't already start with
-    # "From ".
-    #
-    # This algorithm, and the way it interacts with _search_start() and
-    # _search_end() may not be completely correct, because it doesn't check
-    # that the two characters preceding "From " are \n\n or the beginning of
-    # the file.  Fixing this would require a more extensive rewrite than is
-    # necessary.  For convenience, we've added a PortableUnixMailbox class
-    # which does no checking of the format of the 'From' line.
-
-    _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
-                        r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
-                        r"[^\s]*\s*"
-                        "$")
-    _regexp = None
-
-    def _strict_isrealfromline(self, line):
-        if not self._regexp:
-            import re
-            self._regexp = re.compile(self._fromlinepattern)
-        return self._regexp.match(line)
-
-    def _portable_isrealfromline(self, line):
-        return True
-
-    _isrealfromline = _strict_isrealfromline
-
-
-class PortableUnixMailbox(UnixMailbox):
-    _isrealfromline = UnixMailbox._portable_isrealfromline
-
-
-class MmdfMailbox(_Mailbox):
-
-    def _search_start(self):
-        while 1:
-            line = self.fp.readline()
-            if not line:
-                raise EOFError
-            if line[:5] == '\001\001\001\001\n':
-                return
-
-    def _search_end(self):
-        while 1:
-            pos = self.fp.tell()
-            line = self.fp.readline()
-            if not line:
-                return
-            if line == '\001\001\001\001\n':
-                self.fp.seek(pos)
-                return
-
-
-class MHMailbox:
-
-    def __init__(self, dirname, factory=rfc822.Message):
-        import re
-        pat = re.compile('^[1-9][0-9]*$')
-        self.dirname = dirname
-        # the three following lines could be combined into:
-        # list = map(long, filter(pat.match, os.listdir(self.dirname)))
-        list = os.listdir(self.dirname)
-        list = filter(pat.match, list)
-        list = map(long, list)
-        list.sort()
-        # This only works in Python 1.6 or later;
-        # before that str() added 'L':
-        self.boxes = map(str, list)
-        self.boxes.reverse()
-        self.factory = factory
-
-    def __iter__(self):
-        return iter(self.next, None)
-
-    def next(self):
-        if not self.boxes:
-            return None
-        fn = self.boxes.pop()
-        fp = open(os.path.join(self.dirname, fn))
-        msg = self.factory(fp)
-        try:
-            msg._mh_msgno = fn
-        except (AttributeError, TypeError):
-            pass
-        return msg
-
-
-class BabylMailbox(_Mailbox):
-
-    def _search_start(self):
-        while 1:
-            line = self.fp.readline()
-            if not line:
-                raise EOFError
-            if line == '*** EOOH ***\n':
-                return
-
-    def _search_end(self):
-        while 1:
-            pos = self.fp.tell()
-            line = self.fp.readline()
-            if not line:
-                return
-            if line == '\037\014\n' or line == '\037':
-                self.fp.seek(pos)
-                return
-
-## End: classes from the original module (for backward compatibility).
-
-
-class Error(Exception):
-    """Raised for module-specific errors."""
-
-class NoSuchMailboxError(Error):
-    """The specified mailbox does not exist and won't be created."""
-
-class NotEmptyError(Error):
-    """The specified mailbox is not empty and deletion was requested."""
-
-class ExternalClashError(Error):
-    """Another process caused an action to fail."""
-
-class FormatError(Error):
-    """A file appears to have an invalid format."""
diff --git a/lib-python/modified-2.7/mailbox.py b/lib-python/modified-2.7/mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/mailbox.py
@@ -0,0 +1,2179 @@
+#! /usr/bin/env python
+
+"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
+
+# Notes for authors of new mailbox subclasses:
+#
+# Remember to fsync() changes to disk before closing a modified file
+# or returning from a flush() method.  See functions _sync_flush() and
+# _sync_close().
+
+import sys
+import os
+import time
+import calendar
+import socket
+import errno
+import copy
+import email
+import email.message
+import email.generator
+import StringIO
+try:
+    if sys.platform == 'os2emx':
+        # OS/2 EMX fcntl() not adequate
+        raise ImportError
+    import fcntl
+except ImportError:
+    fcntl = None
+
+import warnings
+with warnings.catch_warnings():
+    if sys.py3kwarning:
+        warnings.filterwarnings("ignore", ".*rfc822 has been removed",
+                                DeprecationWarning)
+    import rfc822
+
+__all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
+            'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
+            'BabylMessage', 'MMDFMessage', 'UnixMailbox',
+            'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
+
+class Mailbox:
+    """A group of messages in a particular place."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a Mailbox instance."""
+        self._path = os.path.abspath(os.path.expanduser(path))
+        self._factory = factory
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def __delitem__(self, key):
+        self.remove(key)
+
+    def discard(self, key):
+        """If the keyed message exists, remove it."""
+        try:
+            self.remove(key)
+        except KeyError:
+            pass
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get(self, key, default=None):
+        """Return the keyed message, or default if it doesn't exist."""
+        try:
+            return self.__getitem__(key)
+        except KeyError:
+            return default
+
+    def __getitem__(self, key):
+        """Return the keyed message; raise KeyError if it doesn't exist."""
+        if not self._factory:
+            return self.get_message(key)
+        else:
+            return self._factory(self.get_file(key))
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def keys(self):
+        """Return a list of keys."""
+        return list(self.iterkeys())
+
+    def itervalues(self):
+        """Return an iterator over all messages."""
+        for key in self.iterkeys():
+            try:
+                value = self[key]
+            except KeyError:
+                continue
+            yield value
+
+    def __iter__(self):
+        return self.itervalues()
+
+    def values(self):
+        """Return a list of messages. Memory intensive."""
+        return list(self.itervalues())
+
+    def iteritems(self):
+        """Return an iterator over (key, message) tuples."""
+        for key in self.iterkeys():
+            try:
+                value = self[key]
+            except KeyError:
+                continue
+            yield (key, value)
+
+    def items(self):
+        """Return a list of (key, message) tuples. Memory intensive."""
+        return list(self.iteritems())
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def __contains__(self, key):
+        return self.has_key(key)
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def clear(self):
+        """Delete all messages."""
+        for key in self.iterkeys():
+            self.discard(key)
+
+    def pop(self, key, default=None):
+        """Delete the keyed message and return it, or default."""
+        try:
+            result = self[key]
+        except KeyError:
+            return default
+        self.discard(key)
+        return result
+
+    def popitem(self):
+        """Delete an arbitrary (key, message) pair and return it."""
+        for key in self.iterkeys():
+            return (key, self.pop(key))     # This is only run once.
+        else:
+            raise KeyError('No messages in mailbox')
+
+    def update(self, arg=None):
+        """Change the messages that correspond to certain keys."""
+        if hasattr(arg, 'iteritems'):
+            source = arg.iteritems()
+        elif hasattr(arg, 'items'):
+            source = arg.items()
+        else:
+            source = arg
+        bad_key = False
+        for key, message in source:
+            try:
+                self[key] = message
+            except KeyError:
+                bad_key = True
+        if bad_key:
+            raise KeyError('No message with key(s)')
+
+    def flush(self):
+        """Write any pending changes to the disk."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def lock(self):
+        """Lock the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def close(self):
+        """Flush and close the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def _dump_message(self, message, target, mangle_from_=False):
+        # Most files are opened in binary mode to allow predictable seeking.
+        # To get native line endings on disk, the user-friendly \n line endings
+        # used in strings and by email.Message are translated here.
+        """Dump message contents to target file."""
+        if isinstance(message, email.message.Message):
+            buffer = StringIO.StringIO()
+            gen = email.generator.Generator(buffer, mangle_from_, 0)
+            gen.flatten(message)
+            buffer.seek(0)
+            target.write(buffer.read().replace('\n', os.linesep))
+        elif isinstance(message, str):
+            if mangle_from_:
+                message = message.replace('\nFrom ', '\n>From ')
+            message = message.replace('\n', os.linesep)
+            target.write(message)
+        elif hasattr(message, 'read'):
+            while True:
+                line = message.readline()
+                if line == '':
+                    break
+                if mangle_from_ and line.startswith('From '):
+                    line = '>From ' + line[5:]
+                line = line.replace('\n', os.linesep)
+                target.write(line)
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+
+
+class Maildir(Mailbox):
+    """A qmail-style Maildir mailbox."""
+
+    colon = ':'
+
+    def __init__(self, dirname, factory=rfc822.Message, create=True):
+        """Initialize a Maildir instance."""
+        Mailbox.__init__(self, dirname, factory, create)
+        self._paths = {
+            'tmp': os.path.join(self._path, 'tmp'),
+            'new': os.path.join(self._path, 'new'),
+            'cur': os.path.join(self._path, 'cur'),
+            }
+        if not os.path.exists(self._path):
+            if create:
+                os.mkdir(self._path, 0700)
+                for path in self._paths.values():
+                    os.mkdir(path, 0o700)
+            else:
+                raise NoSuchMailboxError(self._path)
+        self._toc = {}
+        self._toc_mtimes = {}
+        for subdir in ('cur', 'new'):
+            self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir])
+        self._last_read = time.time()  # Records last time we read cur/new
+        self._skewfactor = 0.1         # Adjust if os/fs clocks are skewing
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        tmp_file = self._create_tmp()
+        try:
+            self._dump_message(message, tmp_file)
+        except BaseException:
+            tmp_file.close()
+            os.remove(tmp_file.name)
+            raise
+        _sync_close(tmp_file)
+        if isinstance(message, MaildirMessage):
+            subdir = message.get_subdir()
+            suffix = self.colon + message.get_info()
+            if suffix == self.colon:
+                suffix = ''
+        else:
+            subdir = 'new'
+            suffix = ''
+        uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
+        dest = os.path.join(self._path, subdir, uniq + suffix)
+        try:
+            if hasattr(os, 'link'):
+                os.link(tmp_file.name, dest)
+                os.remove(tmp_file.name)
+            else:
+                os.rename(tmp_file.name, dest)
+        except OSError, e:
+            os.remove(tmp_file.name)
+            if e.errno == errno.EEXIST:
+                raise ExternalClashError('Name clash with existing message: %s'
+                                         % dest)
+            else:
+                raise
+        if isinstance(message, MaildirMessage):
+            os.utime(dest, (os.path.getatime(dest), message.get_date()))
+        return uniq
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        os.remove(os.path.join(self._path, self._lookup(key)))
+
+    def discard(self, key):
+        """If the keyed message exists, remove it."""
+        # This overrides an inapplicable implementation in the superclass.
+        try:
+            self.remove(key)
+        except KeyError:
+            pass
+        except OSError, e:
+            if e.errno != errno.ENOENT:
+                raise
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        old_subpath = self._lookup(key)
+        temp_key = self.add(message)
+        temp_subpath = self._lookup(temp_key)
+        if isinstance(message, MaildirMessage):
+            # temp's subdir and suffix were specified by message.
+            dominant_subpath = temp_subpath
+        else:
+            # temp's subdir and suffix were defaults from add().
+            dominant_subpath = old_subpath
+        subdir = os.path.dirname(dominant_subpath)
+        if self.colon in dominant_subpath:
+            suffix = self.colon + dominant_subpath.split(self.colon)[-1]
+        else:
+            suffix = ''
+        self.discard(key)
+        new_path = os.path.join(self._path, subdir, key + suffix)
+        os.rename(os.path.join(self._path, temp_subpath), new_path)
+        if isinstance(message, MaildirMessage):
+            os.utime(new_path, (os.path.getatime(new_path),
+                                message.get_date()))
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        subpath = self._lookup(key)
+        f = open(os.path.join(self._path, subpath), 'r')
+        try:
+            if self._factory:
+                msg = self._factory(f)
+            else:
+                msg = MaildirMessage(f)
+        finally:
+            f.close()
+        subdir, name = os.path.split(subpath)
+        msg.set_subdir(subdir)
+        if self.colon in name:
+            msg.set_info(name.split(self.colon)[-1])
+        msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        f = open(os.path.join(self._path, self._lookup(key)), 'r')
+        try:
+            return f.read()
+        finally:
+            f.close()
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        f = open(os.path.join(self._path, self._lookup(key)), 'rb')
+        return _ProxyFile(f)
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        self._refresh()
+        for key in self._toc:
+            try:
+                self._lookup(key)
+            except KeyError:
+                continue
+            yield key
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        self._refresh()
+        return key in self._toc
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        self._refresh()
+        return len(self._toc)
+
+    def flush(self):
+        """Write any pending changes to disk."""
+        # Maildir changes are always written immediately, so there's nothing
+        # to do.
+        pass
+
+    def lock(self):
+        """Lock the mailbox."""
+        return
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        return
+
+    def list_folders(self):
+        """Return a list of folder names."""
+        result = []
+        for entry in os.listdir(self._path):
+            if len(entry) > 1 and entry[0] == '.' and \
+               os.path.isdir(os.path.join(self._path, entry)):
+                result.append(entry[1:])
+        return result
+
+    def get_folder(self, folder):
+        """Return a Maildir instance for the named folder."""
+        return Maildir(os.path.join(self._path, '.' + folder),
+                       factory=self._factory,
+                       create=False)
+
+    def add_folder(self, folder):
+        """Create a folder and return a Maildir instance representing it."""
+        path = os.path.join(self._path, '.' + folder)
+        result = Maildir(path, factory=self._factory)
+        maildirfolder_path = os.path.join(path, 'maildirfolder')
+        if not os.path.exists(maildirfolder_path):
+            os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
+                0666))
+        return result
+
+    def remove_folder(self, folder):
+        """Delete the named folder, which must be empty."""
+        path = os.path.join(self._path, '.' + folder)
+        for entry in os.listdir(os.path.join(path, 'new')) + \
+                     os.listdir(os.path.join(path, 'cur')):
+            if len(entry) < 1 or entry[0] != '.':
+                raise NotEmptyError('Folder contains message(s): %s' % folder)
+        for entry in os.listdir(path):
+            if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
+               os.path.isdir(os.path.join(path, entry)):
+                raise NotEmptyError("Folder contains subdirectory '%s': %s" %
+                                    (folder, entry))
+        for root, dirs, files in os.walk(path, topdown=False):
+            for entry in files:
+                os.remove(os.path.join(root, entry))
+            for entry in dirs:
+                os.rmdir(os.path.join(root, entry))
+        os.rmdir(path)
+
+    def clean(self):
+        """Delete old files in "tmp"."""
+        now = time.time()
+        for entry in os.listdir(os.path.join(self._path, 'tmp')):
+            path = os.path.join(self._path, 'tmp', entry)
+            if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
+                os.remove(path)
+
+    _count = 1  # This is used to generate unique file names.
+
+    def _create_tmp(self):
+        """Create a file in the tmp subdirectory and open and return it."""
+        now = time.time()
+        hostname = socket.gethostname()
+        if '/' in hostname:
+            hostname = hostname.replace('/', r'\057')
+        if ':' in hostname:
+            hostname = hostname.replace(':', r'\072')
+        uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
+                                    Maildir._count, hostname)
+        path = os.path.join(self._path, 'tmp', uniq)
+        try:
+            os.stat(path)
+        except OSError, e:
+            if e.errno == errno.ENOENT:
+                Maildir._count += 1
+                try:
+                    return _create_carefully(path)
+                except OSError, e:
+                    if e.errno != errno.EEXIST:
+                        raise
+            else:
+                raise
+
+        # Fall through to here if stat succeeded or open raised EEXIST.
+        raise ExternalClashError('Name clash prevented file creation: %s' %
+                                 path)
+
+    def _refresh(self):
+        """Update table of contents mapping."""
+        # If it has been less than two seconds since the last _refresh() call,
+        # we have to unconditionally re-read the mailbox just in case it has
+        # been modified, because os.path.mtime() has a 2 sec resolution in the
+        # most common worst case (FAT) and a 1 sec resolution typically.  This
+        # results in a few unnecessary re-reads when _refresh() is called
+        # multiple times in that interval, but once the clock ticks over, we
+        # will only re-read as needed.  Because the filesystem might be being
+        # served by an independent system with its own clock, we record and
+        # compare with the mtimes from the filesystem.  Because the other
+        # system's clock might be skewing relative to our clock, we add an
+        # extra delta to our wait.  The default is one tenth second, but is an
+        # instance variable and so can be adjusted if dealing with a
+        # particularly skewed or irregular system.
+        if time.time() - self._last_read > 2 + self._skewfactor:
+            refresh = False
+            for subdir in self._toc_mtimes:
+                mtime = os.path.getmtime(self._paths[subdir])
+                if mtime > self._toc_mtimes[subdir]:
+                    refresh = True
+                self._toc_mtimes[subdir] = mtime
+            if not refresh:
+                return
+        # Refresh toc
+        self._toc = {}
+        for subdir in self._toc_mtimes:
+            path = self._paths[subdir]
+            for entry in os.listdir(path):
+                p = os.path.join(path, entry)
+                if os.path.isdir(p):
+                    continue
+                uniq = entry.split(self.colon)[0]
+                self._toc[uniq] = os.path.join(subdir, entry)
+        self._last_read = time.time()
+
+    def _lookup(self, key):
+        """Use TOC to return subpath for given key, or raise a KeyError."""
+        try:
+            if os.path.exists(os.path.join(self._path, self._toc[key])):
+                return self._toc[key]
+        except KeyError:
+            pass
+        self._refresh()
+        try:
+            return self._toc[key]
+        except KeyError:
+            raise KeyError('No message with key: %s' % key)
+
+    # This method is for backward compatibility only.
+    def next(self):
+        """Return the next message in a one-time iteration."""
+        if not hasattr(self, '_onetime_keys'):
+            self._onetime_keys = self.iterkeys()
+        while True:
+            try:
+                return self[self._onetime_keys.next()]
+            except StopIteration:
+                return None
+            except KeyError:
+                continue
+
+
+class _singlefileMailbox(Mailbox):
+    """A single-file mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a single-file mailbox."""
+        Mailbox.__init__(self, path, factory, create)
+        try:
+            f = open(self._path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                if create:
+                    f = open(self._path, 'wb+')
+                else:
+                    raise NoSuchMailboxError(self._path)
+            elif e.errno in (errno.EACCES, errno.EROFS):
+                f = open(self._path, 'rb')
+            else:
+                raise
+        self._file = f
+        self._toc = None
+        self._next_key = 0
+        self._pending = False   # No changes require rewriting the file.
+        self._locked = False
+        self._file_length = None        # Used to record mailbox size
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        self._lookup()
+        self._toc[self._next_key] = self._append_message(message)
+        self._next_key += 1
+        self._pending = True
+        return self._next_key - 1
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        self._lookup(key)
+        del self._toc[key]
+        self._pending = True
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        self._lookup(key)
+        self._toc[key] = self._append_message(message)
+        self._pending = True
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        self._lookup()
+        for key in self._toc.keys():
+            yield key
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        self._lookup()
+        return key in self._toc
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        self._lookup()
+        return len(self._toc)
+
+    def lock(self):
+        """Lock the mailbox."""
+        if not self._locked:
+            _lock_file(self._file)
+            self._locked = True
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        if self._locked:
+            _unlock_file(self._file)
+            self._locked = False
+
+    def flush(self):
+        """Write any pending changes to disk."""
+        if not self._pending:
+            return
+        if self._file.closed:
+            self._pending = False
+            return
+        # In order to be writing anything out at all, self._toc must
+        # already have been generated (and presumably has been modified
+        # by adding or deleting an item).
+        assert self._toc is not None
+
+        # Check length of self._file; if it's changed, some other process
+        # has modified the mailbox since we scanned it.
+        self._file.seek(0, 2)
+        cur_len = self._file.tell()
+        if cur_len != self._file_length:
+            raise ExternalClashError('Size of mailbox file changed '
+                                     '(expected %i, found %i)' %
+                                     (self._file_length, cur_len))
+
+        new_file = _create_temporary(self._path)
+        try:
+            new_toc = {}
+            self._pre_mailbox_hook(new_file)
+            for key in sorted(self._toc.keys()):
+                start, stop = self._toc[key]
+                self._file.seek(start)
+                self._pre_message_hook(new_file)
+                new_start = new_file.tell()
+                while True:
+                    buffer = self._file.read(min(4096,
+                                                 stop - self._file.tell()))
+                    if buffer == '':
+                        break
+                    new_file.write(buffer)
+                new_toc[key] = (new_start, new_file.tell())
+                self._post_message_hook(new_file)
+        except:
+            new_file.close()
+            os.remove(new_file.name)
+            raise
+        _sync_close(new_file)
+        # self._file is about to get replaced, so no need to sync.
+        self._file.close()
+        try:
+            os.rename(new_file.name, self._path)
+        except OSError, e:
+            if e.errno == errno.EEXIST or \
+              (os.name == 'os2' and e.errno == errno.EACCES):
+                os.remove(self._path)
+                os.rename(new_file.name, self._path)
+            else:
+                raise
+        self._file = open(self._path, 'rb+')
+        self._toc = new_toc
+        self._pending = False
+        if self._locked:
+            _lock_file(self._file, dotlock=False)
+
+    def _pre_mailbox_hook(self, f):
+        """Called before writing the mailbox to file f."""
+        return
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        return
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        self.flush()
+        if self._locked:
+            self.unlock()
+        self._file.close()  # Sync has been done by self.flush() above.
+
+    def _lookup(self, key=None):
+        """Return (start, stop) or raise KeyError."""
+        if self._toc is None:
+            self._generate_toc()
+        if key is not None:
+            try:
+                return self._toc[key]
+            except KeyError:
+                raise KeyError('No message with key: %s' % key)
+
+    def _append_message(self, message):
+        """Append message to mailbox and return (start, stop) offsets."""
+        self._file.seek(0, 2)
+        before = self._file.tell()
+        try:
+            self._pre_message_hook(self._file)
+            offsets = self._install_message(message)
+            self._post_message_hook(self._file)
+        except BaseException:
+            self._file.truncate(before)
+            raise
+        self._file.flush()
+        self._file_length = self._file.tell()  # Record current length of mailbox
+        return offsets
+
+
+
+class _mboxMMDF(_singlefileMailbox):
+    """An mbox or MMDF mailbox."""
+
+    _mangle_from_ = True
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        from_line = self._file.readline().replace(os.linesep, '')
+        string = self._file.read(stop - self._file.tell())
+        msg = self._message_factory(string.replace(os.linesep, '\n'))
+        msg.set_from(from_line[5:])
+        return msg
+
+    def get_string(self, key, from_=False):
+        """Return a string representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        if not from_:
+            self._file.readline()
+        string = self._file.read(stop - self._file.tell())
+        return string.replace(os.linesep, '\n')
+
+    def get_file(self, key, from_=False):
+        """Return a file-like representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+            
+        if not from_:
+            self._file.readline()
+        return _PartialFile(self._file, self._file.tell(), stop)
+
+    def _install_message(self, message):
+        """Format a message and blindly write to self._file."""
+        from_line = None
+        if isinstance(message, str) and message.startswith('From '):
+            newline = message.find('\n')
+            if newline != -1:
+                from_line = message[:newline]
+                message = message[newline + 1:]
+            else:
+                from_line = message
+                message = ''
+        elif isinstance(message, _mboxMMDFMessage):
+            from_line = 'From ' + message.get_from()
+        elif isinstance(message, email.message.Message):
+            from_line = message.get_unixfrom()  # May be None.
+        if from_line is None:
+            from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
+        start = self._file.tell()
+        self._file.write(from_line + os.linesep)
+        self._dump_message(message, self._file, self._mangle_from_)
+        stop = self._file.tell()
+        return (start, stop)
+
+
+class mbox(_mboxMMDF):
+    """A classic mbox mailbox."""
+
+    _mangle_from_ = True
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an mbox mailbox."""
+        self._message_factory = mboxMessage
+        _mboxMMDF.__init__(self, path, factory, create)
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        if f.tell() != 0:
+            f.write(os.linesep)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        while True:
+            line_pos = self._file.tell()
+            line = self._file.readline()
+            if line.startswith('From '):
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+                starts.append(line_pos)
+            elif line == '':
+                stops.append(line_pos)
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._next_key = len(self._toc)
+        self._file_length = self._file.tell()
+
+
+class MMDF(_mboxMMDF):
+    """An MMDF mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an MMDF mailbox."""
+        self._message_factory = MMDFMessage
+        _mboxMMDF.__init__(self, path, factory, create)
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        f.write('\001\001\001\001' + os.linesep)
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        f.write(os.linesep + '\001\001\001\001' + os.linesep)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        next_pos = 0
+        while True:
+            line_pos = next_pos
+            line = self._file.readline()
+            next_pos = self._file.tell()
+            if line.startswith('\001\001\001\001' + os.linesep):
+                starts.append(next_pos)
+                while True:
+                    line_pos = next_pos
+                    line = self._file.readline()
+                    next_pos = self._file.tell()
+                    if line == '\001\001\001\001' + os.linesep:
+                        stops.append(line_pos - len(os.linesep))
+                        break
+                    elif line == '':
+                        stops.append(line_pos)
+                        break
+            elif line == '':
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._next_key = len(self._toc)
+        self._file.seek(0, 2)
+        self._file_length = self._file.tell()
+
+
+class MH(Mailbox):
+    """An MH mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an MH instance."""
+        Mailbox.__init__(self, path, factory, create)
+        if not os.path.exists(self._path):
+            if create:
+                os.mkdir(self._path, 0700)
+                os.close(os.open(os.path.join(self._path, '.mh_sequences'),
+                                 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
+            else:
+                raise NoSuchMailboxError(self._path)
+        self._locked = False
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        keys = self.keys()
+        if len(keys) == 0:
+            new_key = 1
+        else:
+            new_key = max(keys) + 1
+        new_path = os.path.join(self._path, str(new_key))
+        f = _create_carefully(new_path)
+        closed = False
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                try:
+                    self._dump_message(message, f)
+                except BaseException:
+                    # Unlock and close so it can be deleted on Windows
+                    if self._locked:
+                        _unlock_file(f)
+                    _sync_close(f)
+                    closed = True
+                    os.remove(new_path)
+                    raise
+                if isinstance(message, MHMessage):
+                    self._dump_sequences(message, new_key)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            if not closed:
+                _sync_close(f)
+        return new_key
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        path = os.path.join(self._path, str(key))
+        try:
+            f = open(path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        else:
+            f.close()
+            os.remove(path)
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        path = os.path.join(self._path, str(key))
+        try:
+            f = open(path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
+                self._dump_message(message, f)
+                if isinstance(message, MHMessage):
+                    self._dump_sequences(message, key)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            _sync_close(f)
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        try:
+            if self._locked:
+                f = open(os.path.join(self._path, str(key)), 'r+')
+            else:
+                f = open(os.path.join(self._path, str(key)), 'r')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                msg = MHMessage(f)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            f.close()
+        for name, key_list in self.get_sequences().iteritems():
+            if key in key_list:
+                msg.add_sequence(name)
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        try:
+            if self._locked:
+                f = open(os.path.join(self._path, str(key)), 'r+')
+            else:
+                f = open(os.path.join(self._path, str(key)), 'r')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                return f.read()
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            f.close()
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        try:
+            f = open(os.path.join(self._path, str(key)), 'rb')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        return _ProxyFile(f)
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        return iter(sorted(int(entry) for entry in os.listdir(self._path)
+                                      if entry.isdigit()))
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        return os.path.exists(os.path.join(self._path, str(key)))
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        return len(list(self.iterkeys()))
+
+    def lock(self):
+        """Lock the mailbox."""
+        if not self._locked:
+            self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
+            _lock_file(self._file)
+            self._locked = True
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        if self._locked:
+            _unlock_file(self._file)
+            _sync_close(self._file)
+            del self._file
+            self._locked = False
+
+    def flush(self):
+        """Write any pending changes to the disk."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        if self._locked:
+            self.unlock()
+
+    def list_folders(self):
+        """Return a list of folder names."""
+        result = []
+        for entry in os.listdir(self._path):
+            if os.path.isdir(os.path.join(self._path, entry)):
+                result.append(entry)
+        return result
+
+    def get_folder(self, folder):
+        """Return an MH instance for the named folder."""
+        return MH(os.path.join(self._path, folder),
+                  factory=self._factory, create=False)
+
+    def add_folder(self, folder):
+        """Create a folder and return an MH instance representing it."""
+        return MH(os.path.join(self._path, folder),
+                  factory=self._factory)
+
+    def remove_folder(self, folder):
+        """Delete the named folder, which must be empty."""
+        path = os.path.join(self._path, folder)
+        entries = os.listdir(path)
+        if entries == ['.mh_sequences']:
+            os.remove(os.path.join(path, '.mh_sequences'))
+        elif entries == []:
+            pass
+        else:
+            raise NotEmptyError('Folder not empty: %s' % self._path)
+        os.rmdir(path)
+
+    def get_sequences(self):
+        """Return a name-to-key-list dictionary to define each sequence."""
+        results = {}
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r')
+        try:
+            all_keys = set(self.keys())
+            for line in f:
+                try:
+                    name, contents = line.split(':')
+                    keys = set()
+                    for spec in contents.split():
+                        if spec.isdigit():
+                            keys.add(int(spec))
+                        else:
+                            start, stop = (int(x) for x in spec.split('-'))
+                            keys.update(range(start, stop + 1))
+                    results[name] = [key for key in sorted(keys) \
+                                         if key in all_keys]
+                    if len(results[name]) == 0:
+                        del results[name]
+                except ValueError:
+                    raise FormatError('Invalid sequence specification: %s' %
+                                      line.rstrip())
+        finally:
+            f.close()
+        return results
+
+    def set_sequences(self, sequences):
+        """Set sequences using the given name-to-key-list dictionary."""
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
+        try:
+            os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
+            for name, keys in sequences.iteritems():
+                if len(keys) == 0:
+                    continue
+                f.write('%s:' % name)
+                prev = None
+                completing = False
+                for key in sorted(set(keys)):
+                    if key - 1 == prev:
+                        if not completing:
+                            completing = True
+                            f.write('-')
+                    elif completing:
+                        completing = False
+                        f.write('%s %s' % (prev, key))
+                    else:
+                        f.write(' %s' % key)
+                    prev = key
+                if completing:
+                    f.write(str(prev) + '\n')
+                else:
+                    f.write('\n')
+        finally:
+            _sync_close(f)
+
+    def pack(self):
+        """Re-name messages to eliminate numbering gaps. Invalidates keys."""
+        sequences = self.get_sequences()
+        prev = 0
+        changes = []
+        for key in self.iterkeys():
+            if key - 1 != prev:
+                changes.append((key, prev + 1))
+                if hasattr(os, 'link'):
+                    os.link(os.path.join(self._path, str(key)),
+                            os.path.join(self._path, str(prev + 1)))
+                    os.unlink(os.path.join(self._path, str(key)))
+                else:
+                    os.rename(os.path.join(self._path, str(key)),
+                              os.path.join(self._path, str(prev + 1)))
+            prev += 1
+        self._next_key = prev + 1
+        if len(changes) == 0:
+            return
+        for name, key_list in sequences.items():
+            for old, new in changes:
+                if old in key_list:
+                    key_list[key_list.index(old)] = new
+        self.set_sequences(sequences)
+
+    def _dump_sequences(self, message, key):
+        """Inspect a new MHMessage and update sequences appropriately."""
+        pending_sequences = message.get_sequences()
+        all_sequences = self.get_sequences()
+        for name, key_list in all_sequences.iteritems():
+            if name in pending_sequences:
+                key_list.append(key)
+            elif key in key_list:
+                del key_list[key_list.index(key)]
+        for sequence in pending_sequences:
+            if sequence not in all_sequences:
+                all_sequences[sequence] = [key]
+        self.set_sequences(all_sequences)
+
+
+class Babyl(_singlefileMailbox):
+    """An Rmail-style Babyl mailbox."""
+
+    _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
+                                 'forwarded', 'edited', 'resent'))
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a Babyl mailbox."""
+        _singlefileMailbox.__init__(self, path, factory, create)
+        self._labels = {}
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        key = _singlefileMailbox.add(self, message)
+        if isinstance(message, BabylMessage):
+            self._labels[key] = message.get_labels()
+        return key
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        _singlefileMailbox.remove(self, key)
+        if key in self._labels:
+            del self._labels[key]
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        _singlefileMailbox.__setitem__(self, key, message)
+        if isinstance(message, BabylMessage):
+            self._labels[key] = message.get_labels()
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        self._file.readline()   # Skip '1,' line specifying labels.
+        original_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == '*** EOOH ***' + os.linesep or line == '':
+                break
+            original_headers.write(line.replace(os.linesep, '\n'))
+        visible_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == os.linesep or line == '':
+                break
+            visible_headers.write(line.replace(os.linesep, '\n'))
+        body = self._file.read(stop - self._file.tell()).replace(os.linesep,
+                                                                 '\n')
+        msg = BabylMessage(original_headers.getvalue() + body)
+        msg.set_visible(visible_headers.getvalue())
+        if key in self._labels:
+            msg.set_labels(self._labels[key])
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        self._file.readline()   # Skip '1,' line specifying labels.
+        original_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == '*** EOOH ***' + os.linesep or line == '':
+                break
+            original_headers.write(line.replace(os.linesep, '\n'))
+        while True:
+            line = self._file.readline()
+            if line == os.linesep or line == '':
+                break
+        return original_headers.getvalue() + \
+               self._file.read(stop - self._file.tell()).replace(os.linesep,
+                                                                 '\n')
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        return StringIO.StringIO(self.get_string(key).replace('\n',
+                                                              os.linesep))
+
+    def get_labels(self):
+        """Return a list of user-defined labels in the mailbox."""
+        self._lookup()
+        labels = set()
+        for label_list in self._labels.values():
+            labels.update(label_list)
+        labels.difference_update(self._special_labels)
+        return list(labels)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        next_pos = 0
+        label_lists = []
+        while True:
+            line_pos = next_pos
+            line = self._file.readline()
+            next_pos = self._file.tell()
+            if line == '\037\014' + os.linesep:
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+                starts.append(next_pos)
+                labels = [label.strip() for label
+                                        in self._file.readline()[1:].split(',')
+                                        if label.strip() != '']
+                label_lists.append(labels)
+            elif line == '\037' or line == '\037' + os.linesep:
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+            elif line == '':
+                stops.append(line_pos - len(os.linesep))
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._labels = dict(enumerate(label_lists))
+        self._next_key = len(self._toc)
+        self._file.seek(0, 2)
+        self._file_length = self._file.tell()
+
+    def _pre_mailbox_hook(self, f):
+        """Called before writing the mailbox to file f."""
+        f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
+                (os.linesep, os.linesep, ','.join(self.get_labels()),
+                 os.linesep))
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        f.write('\014' + os.linesep)
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        f.write(os.linesep + '\037')
+
+    def _install_message(self, message):
+        """Write message contents and return (start, stop)."""
+        start = self._file.tell()
+        if isinstance(message, BabylMessage):
+            special_labels = []
+            labels = []
+            for label in message.get_labels():
+                if label in self._special_labels:
+                    special_labels.append(label)
+                else:
+                    labels.append(label)
+            self._file.write('1')
+            for label in special_labels:
+                self._file.write(', ' + label)
+            self._file.write(',,')
+            for label in labels:
+                self._file.write(' ' + label + ',')
+            self._file.write(os.linesep)
+        else:
+            self._file.write('1,,' + os.linesep)
+        if isinstance(message, email.message.Message):
+            orig_buffer = StringIO.StringIO()
+            orig_generator = email.generator.Generator(orig_buffer, False, 0)
+            orig_generator.flatten(message)
+            orig_buffer.seek(0)
+            while True:
+                line = orig_buffer.readline()
+                self._file.write(line.replace('\n', os.linesep))
+                if line == '\n' or line == '':
+                    break
+            self._file.write('*** EOOH ***' + os.linesep)
+            if isinstance(message, BabylMessage):
+                vis_buffer = StringIO.StringIO()
+                vis_generator = email.generator.Generator(vis_buffer, False, 0)
+                vis_generator.flatten(message.get_visible())
+                while True:
+                    line = vis_buffer.readline()
+                    self._file.write(line.replace('\n', os.linesep))
+                    if line == '\n' or line == '':
+                        break
+            else:
+                orig_buffer.seek(0)
+                while True:
+                    line = orig_buffer.readline()
+                    self._file.write(line.replace('\n', os.linesep))
+                    if line == '\n' or line == '':
+                        break
+            while True:
+                buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
+                if buffer == '':
+                    break
+                self._file.write(buffer.replace('\n', os.linesep))
+        elif isinstance(message, str):
+            body_start = message.find('\n\n') + 2
+            if body_start - 2 != -1:
+                self._file.write(message[:body_start].replace('\n',
+                                                              os.linesep))
+                self._file.write('*** EOOH ***' + os.linesep)
+                self._file.write(message[:body_start].replace('\n',
+                                                              os.linesep))
+                self._file.write(message[body_start:].replace('\n',
+                                                              os.linesep))
+            else:
+                self._file.write('*** EOOH ***' + os.linesep + os.linesep)
+                self._file.write(message.replace('\n', os.linesep))
+        elif hasattr(message, 'readline'):
+            original_pos = message.tell()
+            first_pass = True
+            while True:
+                line = message.readline()
+                self._file.write(line.replace('\n', os.linesep))
+                if line == '\n' or line == '':
+                    self._file.write('*** EOOH ***' + os.linesep)
+                    if first_pass:
+                        first_pass = False
+                        message.seek(original_pos)
+                    else:
+                        break
+            while True:
+                buffer = message.read(4096)     # Buffer size is arbitrary.
+                if buffer == '':
+                    break
+                self._file.write(buffer.replace('\n', os.linesep))
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+        stop = self._file.tell()
+        return (start, stop)
+
+
+class Message(email.message.Message):
+    """Message with mailbox-format-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize a Message instance."""
+        if isinstance(message, email.message.Message):
+            self._become_message(copy.deepcopy(message))
+            if isinstance(message, Message):
+                message._explain_to(self)
+        elif isinstance(message, str):
+            self._become_message(email.message_from_string(message))
+        elif hasattr(message, "read"):
+            self._become_message(email.message_from_file(message))
+        elif message is None:
+            email.message.Message.__init__(self)
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+
+    def _become_message(self, message):
+        """Assume the non-format-specific state of message."""
+        for name in ('_headers', '_unixfrom', '_payload', '_charset',
+                     'preamble', 'epilogue', 'defects', '_default_type'):
+            self.__dict__[name] = message.__dict__[name]
+
+    def _explain_to(self, message):
+        """Copy format-specific state to message insofar as possible."""
+        if isinstance(message, Message):
+            return  # There's nothing format-specific to explain.
+        else:
+            raise TypeError('Cannot convert to specified type')
+
+
+class MaildirMessage(Message):
+    """Message with Maildir-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize a MaildirMessage instance."""
+        self._subdir = 'new'
+        self._info = ''
+        self._date = time.time()
+        Message.__init__(self, message)
+
+    def get_subdir(self):
+        """Return 'new' or 'cur'."""
+        return self._subdir
+
+    def set_subdir(self, subdir):
+        """Set subdir to 'new' or 'cur'."""
+        if subdir == 'new' or subdir == 'cur':
+            self._subdir = subdir
+        else:
+            raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
+
+    def get_flags(self):
+        """Return as a string the flags that are set."""
+        if self._info.startswith('2,'):
+            return self._info[2:]
+        else:
+            return ''
+
+    def set_flags(self, flags):
+        """Set the given flags and unset all others."""
+        self._info = '2,' + ''.join(sorted(flags))
+
+    def add_flag(self, flag):
+        """Set the given flag(s) without changing others."""
+        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+    def remove_flag(self, flag):
+        """Unset the given string flag(s) without changing others."""
+        if self.get_flags() != '':
+            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+    def get_date(self):
+        """Return delivery date of message, in seconds since the epoch."""
+        return self._date
+
+    def set_date(self, date):
+        """Set delivery date of message, in seconds since the epoch."""
+        try:
+            self._date = float(date)
+        except ValueError:
+            raise TypeError("can't convert to float: %s" % date)
+
+    def get_info(self):
+        """Get the message's "info" as a string."""
+        return self._info
+
+    def set_info(self, info):
+        """Set the message's "info" string."""
+        if isinstance(info, str):
+            self._info = info
+        else:
+            raise TypeError('info must be a string: %s' % type(info))
+
+    def _explain_to(self, message):
+        """Copy Maildir-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            message.set_flags(self.get_flags())
+            message.set_subdir(self.get_subdir())
+            message.set_date(self.get_date())
+        elif isinstance(message, _mboxMMDFMessage):
+            flags = set(self.get_flags())
+            if 'S' in flags:
+                message.add_flag('R')
+            if self.get_subdir() == 'cur':
+                message.add_flag('O')
+            if 'T' in flags:
+                message.add_flag('D')
+            if 'F' in flags:
+                message.add_flag('F')
+            if 'R' in flags:
+                message.add_flag('A')
+            message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
+        elif isinstance(message, MHMessage):
+            flags = set(self.get_flags())
+            if 'S' not in flags:
+                message.add_sequence('unseen')
+            if 'R' in flags:
+                message.add_sequence('replied')
+            if 'F' in flags:
+                message.add_sequence('flagged')
+        elif isinstance(message, BabylMessage):
+            flags = set(self.get_flags())
+            if 'S' not in flags:
+                message.add_label('unseen')
+            if 'T' in flags:
+                message.add_label('deleted')
+            if 'R' in flags:
+                message.add_label('answered')
+            if 'P' in flags:
+                message.add_label('forwarded')
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class _mboxMMDFMessage(Message):
+    """Message with mbox- or MMDF-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an mboxMMDFMessage instance."""
+        self.set_from('MAILER-DAEMON', True)
+        if isinstance(message, email.message.Message):
+            unixfrom = message.get_unixfrom()
+            if unixfrom is not None and unixfrom.startswith('From '):
+                self.set_from(unixfrom[5:])
+        Message.__init__(self, message)
+
+    def get_from(self):
+        """Return contents of "From " line."""
+        return self._from
+
+    def set_from(self, from_, time_=None):
+        """Set "From " line, formatting and appending time_ if specified."""
+        if time_ is not None:
+            if time_ is True:
+                time_ = time.gmtime()
+            from_ += ' ' + time.asctime(time_)
+        self._from = from_
+
+    def get_flags(self):
+        """Return as a string the flags that are set."""
+        return self.get('Status', '') + self.get('X-Status', '')
+
+    def set_flags(self, flags):
+        """Set the given flags and unset all others."""
+        flags = set(flags)
+        status_flags, xstatus_flags = '', ''
+        for flag in ('R', 'O'):
+            if flag in flags:
+                status_flags += flag
+                flags.remove(flag)
+        for flag in ('D', 'F', 'A'):
+            if flag in flags:
+                xstatus_flags += flag
+                flags.remove(flag)
+        xstatus_flags += ''.join(sorted(flags))
+        try:
+            self.replace_header('Status', status_flags)
+        except KeyError:
+            self.add_header('Status', status_flags)
+        try:
+            self.replace_header('X-Status', xstatus_flags)
+        except KeyError:
+            self.add_header('X-Status', xstatus_flags)
+
+    def add_flag(self, flag):
+        """Set the given flag(s) without changing others."""
+        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+    def remove_flag(self, flag):
+        """Unset the given string flag(s) without changing others."""
+        if 'Status' in self or 'X-Status' in self:
+            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+    def _explain_to(self, message):
+        """Copy mbox- or MMDF-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            flags = set(self.get_flags())
+            if 'O' in flags:
+                message.set_subdir('cur')
+            if 'F' in flags:
+                message.add_flag('F')
+            if 'A' in flags:
+                message.add_flag('R')
+            if 'R' in flags:
+                message.add_flag('S')
+            if 'D' in flags:
+                message.add_flag('T')
+            del message['status']
+            del message['x-status']
+            maybe_date = ' '.join(self.get_from().split()[-5:])
+            try:
+                message.set_date(calendar.timegm(time.strptime(maybe_date,
+                                                      '%a %b %d %H:%M:%S %Y')))
+            except (ValueError, OverflowError):
+                pass
+        elif isinstance(message, _mboxMMDFMessage):
+            message.set_flags(self.get_flags())
+            message.set_from(self.get_from())
+        elif isinstance(message, MHMessage):
+            flags = set(self.get_flags())
+            if 'R' not in flags:
+                message.add_sequence('unseen')
+            if 'A' in flags:
+                message.add_sequence('replied')
+            if 'F' in flags:
+                message.add_sequence('flagged')
+            del message['status']
+            del message['x-status']
+        elif isinstance(message, BabylMessage):
+            flags = set(self.get_flags())
+            if 'R' not in flags:
+                message.add_label('unseen')
+            if 'D' in flags:
+                message.add_label('deleted')
+            if 'A' in flags:
+                message.add_label('answered')
+            del message['status']
+            del message['x-status']
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class mboxMessage(_mboxMMDFMessage):
+    """Message with mbox-specific properties."""
+
+
+class MHMessage(Message):
+    """Message with MH-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an MHMessage instance."""
+        self._sequences = []
+        Message.__init__(self, message)
+
+    def get_sequences(self):
+        """Return a list of sequences that include the message."""
+        return self._sequences[:]
+
+    def set_sequences(self, sequences):
+        """Set the list of sequences that include the message."""
+        self._sequences = list(sequences)
+
+    def add_sequence(self, sequence):
+        """Add sequence to list of sequences including the message."""
+        if isinstance(sequence, str):
+            if not sequence in self._sequences:
+                self._sequences.append(sequence)
+        else:
+            raise TypeError('sequence must be a string: %s' % type(sequence))
+
+    def remove_sequence(self, sequence):
+        """Remove sequence from the list of sequences including the message."""
+        try:
+            self._sequences.remove(sequence)
+        except ValueError:
+            pass
+
+    def _explain_to(self, message):
+        """Copy MH-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' in sequences:
+                message.set_subdir('cur')
+            else:
+                message.set_subdir('cur')
+                message.add_flag('S')
+            if 'flagged' in sequences:
+                message.add_flag('F')
+            if 'replied' in sequences:
+                message.add_flag('R')
+        elif isinstance(message, _mboxMMDFMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' not in sequences:
+                message.add_flag('RO')
+            else:
+                message.add_flag('O')
+            if 'flagged' in sequences:
+                message.add_flag('F')
+            if 'replied' in sequences:
+                message.add_flag('A')
+        elif isinstance(message, MHMessage):
+            for sequence in self.get_sequences():
+                message.add_sequence(sequence)
+        elif isinstance(message, BabylMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' in sequences:
+                message.add_label('unseen')
+            if 'replied' in sequences:
+                message.add_label('answered')
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class BabylMessage(Message):
+    """Message with Babyl-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an BabylMessage instance."""
+        self._labels = []
+        self._visible = Message()
+        Message.__init__(self, message)
+
+    def get_labels(self):
+        """Return a list of labels on the message."""
+        return self._labels[:]
+
+    def set_labels(self, labels):
+        """Set the list of labels on the message."""
+        self._labels = list(labels)
+
+    def add_label(self, label):
+        """Add label to list of labels on the message."""
+        if isinstance(label, str):
+            if label not in self._labels:
+                self._labels.append(label)
+        else:
+            raise TypeError('label must be a string: %s' % type(label))
+
+    def remove_label(self, label):
+        """Remove label from the list of labels on the message."""
+        try:
+            self._labels.remove(label)
+        except ValueError:
+            pass
+
+    def get_visible(self):
+        """Return a Message representation of visible headers."""
+        return Message(self._visible)
+
+    def set_visible(self, visible):
+        """Set the Message representation of visible headers."""
+        self._visible = Message(visible)
+
+    def update_visible(self):
+        """Update and/or sensibly generate a set of visible headers."""
+        for header in self._visible.keys():
+            if header in self:
+                self._visible.replace_header(header, self[header])
+            else:
+                del self._visible[header]
+        for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
+            if header in self and header not in self._visible:
+                self._visible[header] = self[header]
+
+    def _explain_to(self, message):
+        """Copy Babyl-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            labels = set(self.get_labels())
+            if 'unseen' in labels:
+                message.set_subdir('cur')
+            else:
+                message.set_subdir('cur')
+                message.add_flag('S')
+            if 'forwarded' in labels or 'resent' in labels:
+                message.add_flag('P')
+            if 'answered' in labels:
+                message.add_flag('R')
+            if 'deleted' in labels:
+                message.add_flag('T')
+        elif isinstance(message, _mboxMMDFMessage):
+            labels = set(self.get_labels())
+            if 'unseen' not in labels:
+                message.add_flag('RO')
+            else:
+                message.add_flag('O')
+            if 'deleted' in labels:
+                message.add_flag('D')
+            if 'answered' in labels:
+                message.add_flag('A')
+        elif isinstance(message, MHMessage):
+            labels = set(self.get_labels())
+            if 'unseen' in labels:
+                message.add_sequence('unseen')
+            if 'answered' in labels:
+                message.add_sequence('replied')
+        elif isinstance(message, BabylMessage):
+            message.set_visible(self.get_visible())
+            for label in self.get_labels():
+                message.add_label(label)
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class MMDFMessage(_mboxMMDFMessage):
+    """Message with MMDF-specific properties."""
+
+
+class _ProxyFile:
+    """A read-only wrapper of a file."""
+
+    def __init__(self, f, pos=None):
+        """Initialize a _ProxyFile."""
+        self._file = f
+        if pos is None:
+            self._pos = f.tell()
+        else:
+            self._pos = pos
+
+    def __del__(self):
+        if hasattr(self,'_file'):
+            self.close()
+
+    def read(self, size=None):
+        """Read bytes."""
+        return self._read(size, self._file.read)
+
+    def readline(self, size=None):
+        """Read a line."""
+        return self._read(size, self._file.readline)
+
+    def readlines(self, sizehint=None):
+        """Read multiple lines."""
+        result = []
+        for line in self:
+            result.append(line)
+            if sizehint is not None:
+                sizehint -= len(line)
+                if sizehint <= 0:
+                    break
+        return result
+
+    def __iter__(self):
+        """Iterate over lines."""
+        return iter(self.readline, "")
+
+    def tell(self):
+        """Return the position."""
+        return self._pos
+
+    def seek(self, offset, whence=0):
+        """Change position."""
+        if whence == 1:
+            self._file.seek(self._pos)
+        self._file.seek(offset, whence)
+        self._pos = self._file.tell()
+
+    def close(self):
+        """Close the file."""
+        self._file.close()
+        del self._file
+
+    def _read(self, size, read_method):
+        """Read size bytes using read_method."""
+        if size is None:
+            size = -1
+        self._file.seek(self._pos)
+        result = read_method(size)
+        self._pos = self._file.tell()
+        return result
+
+
+class _PartialFile(_ProxyFile):
+    """A read-only wrapper of part of a file."""
+
+    def __init__(self, f, start=None, stop=None):
+        """Initialize a _PartialFile."""
+        _ProxyFile.__init__(self, f, start)
+        self._start = start
+        self._stop = stop
+
+    def tell(self):
+        """Return the position with respect to start."""
+        return _ProxyFile.tell(self) - self._start
+
+    def seek(self, offset, whence=0):
+        """Change position, possibly with respect to start or stop."""
+        if whence == 0:
+            self._pos = self._start
+            whence = 1
+        elif whence == 2:
+            self._pos = self._stop
+            whence = 1
+        _ProxyFile.seek(self, offset, whence)
+
+    def _read(self, size, read_method):
+        """Read size bytes using read_method, honoring start and stop."""
+        remaining = self._stop - self._pos
+        if remaining <= 0:
+            return ''
+        if size is None or size < 0 or size > remaining:
+            size = remaining
+        return _ProxyFile._read(self, size, read_method)
+
+
+def _lock_file(f, dotlock=True):
+    """Lock file f using lockf and dot locking."""
+    dotlock_done = False
+    try:
+        if fcntl:
+            try:
+                fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except IOError, e:
+                if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
+                    raise ExternalClashError('lockf: lock unavailable: %s' %
+                                             f.name)
+                else:
+                    raise
+        if dotlock:
+            try:
+                pre_lock = _create_temporary(f.name + '.lock')
+                pre_lock.close()
+            except IOError, e:
+                if e.errno in (errno.EACCES, errno.EROFS):
+                    return  # Without write access, just skip dotlocking.
+                else:
+                    raise
+            try:
+                if hasattr(os, 'link'):
+                    os.link(pre_lock.name, f.name + '.lock')
+                    dotlock_done = True
+                    os.unlink(pre_lock.name)
+                else:
+                    os.rename(pre_lock.name, f.name + '.lock')
+                    dotlock_done = True
+            except OSError, e:
+                if e.errno == errno.EEXIST or \
+                  (os.name == 'os2' and e.errno == errno.EACCES):
+                    os.remove(pre_lock.name)
+                    raise ExternalClashError('dot lock unavailable: %s' %
+                                             f.name)
+                else:
+                    raise
+    except:
+        if fcntl:
+            fcntl.lockf(f, fcntl.LOCK_UN)
+        if dotlock_done:
+            os.remove(f.name + '.lock')
+        raise
+
+def _unlock_file(f):
+    """Unlock file f using lockf and dot locking."""
+    if fcntl:
+        fcntl.lockf(f, fcntl.LOCK_UN)
+    if os.path.exists(f.name + '.lock'):
+        os.remove(f.name + '.lock')
+
+def _create_carefully(path):
+    """Create a file if it doesn't exist and open for reading and writing."""
+    fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
+    try:
+        return open(path, 'rb+')
+    finally:
+        os.close(fd)
+
+def _create_temporary(path):
+    """Create a temp file based on path and open for reading and writing."""
+    return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
+                                              socket.gethostname(),
+                                              os.getpid()))
+
+def _sync_flush(f):
+    """Ensure changes to file f are physically on disk."""
+    f.flush()
+    if hasattr(os, 'fsync'):
+        os.fsync(f.fileno())
+
+def _sync_close(f):
+    """Close file f, ensuring all changes are physically on disk."""
+    _sync_flush(f)
+    f.close()
+
+## Start: classes from the original module (for backward compatibility).
+
+# Note that the Maildir class, whose name is unchanged, itself offers a next()
+# method for backward compatibility.
+
+class _Mailbox:
+
+    def __init__(self, fp, factory=rfc822.Message):
+        self.fp = fp
+        self.seekp = 0
+        self.factory = factory
+
+    def __iter__(self):
+        return iter(self.next, None)
+
+    def next(self):
+        while 1:
+            self.fp.seek(self.seekp)
+            try:
+                self._search_start()
+            except EOFError:
+                self.seekp = self.fp.tell()
+                return None
+            start = self.fp.tell()
+            self._search_end()
+            self.seekp = stop = self.fp.tell()
+            if start != stop:
+                break
+        return self.factory(_PartialFile(self.fp, start, stop))
+
+# Recommended to use PortableUnixMailbox instead!
+class UnixMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line[:5] == 'From ' and self._isrealfromline(line):
+                self.fp.seek(pos)
+                return
+
+    def _search_end(self):
+        self.fp.readline()      # Throw away header line
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line[:5] == 'From ' and self._isrealfromline(line):
+                self.fp.seek(pos)
+                return
+
+    # An overridable mechanism to test for From-line-ness.  You can either
+    # specify a different regular expression or define a whole new
+    # _isrealfromline() method.  Note that this only gets called for lines
+    # starting with the 5 characters "From ".
+    #
+    # BAW: According to
+    #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
+    # the only portable, reliable way to find message delimiters in a BSD (i.e
+    # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
+    # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
+    # like a good idea, in practice, there are too many variations for more
+    # strict parsing of the line to be completely accurate.
+    #
+    # _strict_isrealfromline() is the old version which tries to do stricter
+    # parsing of the From_ line.  _portable_isrealfromline() simply returns
+    # true, since it's never called if the line doesn't already start with
+    # "From ".
+    #
+    # This algorithm, and the way it interacts with _search_start() and
+    # _search_end() may not be completely correct, because it doesn't check
+    # that the two characters preceding "From " are \n\n or the beginning of
+    # the file.  Fixing this would require a more extensive rewrite than is
+    # necessary.  For convenience, we've added a PortableUnixMailbox class
+    # which does no checking of the format of the 'From' line.
+
+    _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
+                        r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
+                        r"[^\s]*\s*"
+                        "$")
+    _regexp = None
+
+    def _strict_isrealfromline(self, line):
+        if not self._regexp:
+            import re
+            self._regexp = re.compile(self._fromlinepattern)
+        return self._regexp.match(line)
+
+    def _portable_isrealfromline(self, line):
+        return True
+
+    _isrealfromline = _strict_isrealfromline
+
+
+class PortableUnixMailbox(UnixMailbox):
+    _isrealfromline = UnixMailbox._portable_isrealfromline
+
+
+class MmdfMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line[:5] == '\001\001\001\001\n':
+                return
+
+    def _search_end(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line == '\001\001\001\001\n':
+                self.fp.seek(pos)
+                return
+
+
+class MHMailbox:
+
+    def __init__(self, dirname, factory=rfc822.Message):
+        import re
+        pat = re.compile('^[1-9][0-9]*$')
+        self.dirname = dirname
+        # the three following lines could be combined into:
+        # list = map(long, filter(pat.match, os.listdir(self.dirname)))
+        list = os.listdir(self.dirname)
+        list = filter(pat.match, list)
+        list = map(long, list)
+        list.sort()
+        # This only works in Python 1.6 or later;
+        # before that str() added 'L':
+        self.boxes = map(str, list)
+        self.boxes.reverse()
+        self.factory = factory
+
+    def __iter__(self):
+        return iter(self.next, None)
+
+    def next(self):
+        if not self.boxes:
+            return None
+        fn = self.boxes.pop()
+        fp = open(os.path.join(self.dirname, fn))
+        msg = self.factory(fp)
+        try:
+            msg._mh_msgno = fn
+        except (AttributeError, TypeError):
+            pass
+        return msg
+
+
+class BabylMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line == '*** EOOH ***\n':
+                return
+
+    def _search_end(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line == '\037\014\n' or line == '\037':
+                self.fp.seek(pos)
+                return
+
+## End: classes from the original module (for backward compatibility).
+
+
+class Error(Exception):
+    """Raised for module-specific errors."""
+
+class NoSuchMailboxError(Error):
+    """The specified mailbox does not exist and won't be created."""
+
+class NotEmptyError(Error):
+    """The specified mailbox is not empty and deletion was requested."""
+
+class ExternalClashError(Error):
+    """Another process caused an action to fail."""
+
+class FormatError(Error):
+    """A file appears to have an invalid format."""
diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/modified-2.7/test/test_mailbox.py
rename from lib-python/2.7/test/test_mailbox.py
rename to lib-python/modified-2.7/test/test_mailbox.py
--- a/lib-python/2.7/test/test_mailbox.py
+++ b/lib-python/modified-2.7/test/test_mailbox.py
@@ -60,6 +60,8 @@
 
     def tearDown(self):
         self._box.close()
+        if os.name == 'nt':
+            time.sleep(0.1) #Allow all syncing to take place
         self._delete_recursively(self._path)
 
     def test_add(self):
@@ -137,6 +139,7 @@
         msg = self._box.get(key1)
         self.assertEqual(msg['from'], 'foo')
         self.assertEqual(msg.fp.read(), '1')
+        msg.fp.close()
 
     def test_getitem(self):
         # Retrieve message using __getitem__()
@@ -169,10 +172,12 @@
         # Get file representations of messages
         key0 = self._box.add(self._template % 0)
         key1 = self._box.add(_sample_message)
-        self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'),
-                         self._template % 0)
-        self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'),
-                         _sample_message)
+        msg0 = self._box.get_file(key0)
+        self.assertEqual(msg0.read().replace(os.linesep, '\n'), self._template % 0)
+        msg1 = self._box.get_file(key1)
+        self.assertEqual(msg1.read().replace(os.linesep, '\n'), _sample_message)
+        msg0.close()
+        msg1.close()
 
     def test_iterkeys(self):
         # Get keys using iterkeys()
@@ -786,6 +791,8 @@
 class _TestMboxMMDF(TestMailbox):
 
     def tearDown(self):
+        if os.name == 'nt':
+            time.sleep(0.1) #Allow os to sync files
         self._box.close()
         self._delete_recursively(self._path)
         for lock_remnant in glob.glob(self._path + '.*'):
@@ -1837,7 +1844,9 @@
         self.createMessage("cur")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 1)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
@@ -1845,7 +1854,9 @@
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 1)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
@@ -1854,8 +1865,12 @@
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         #self.assertTrue(len(self.mbox.boxes) == 2)
-        self.assertIsNot(self.mbox.next(), None)
-        self.assertIsNot(self.mbox.next(), None)
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
+        msg = self.mbox.next()
+        self.assertIsNot(msg, None)
+        msg.fp.close()
         self.assertIs(self.mbox.next(), None)
         self.assertIs(self.mbox.next(), None)
 
@@ -1864,11 +1879,13 @@
         import email.parser
         fname = self.createMessage("cur", True)
         n = 0
-        for msg in mailbox.PortableUnixMailbox(open(fname),
+        fid = open(fname)
+        for msg in mailbox.PortableUnixMailbox(fid,
                                                email.parser.Parser().parse):
             n += 1
             self.assertEqual(msg["subject"], "Simple Test")
             self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+        fid.close()    
         self.assertEqual(n, 1)
 
 ## End: classes from the original module (for backward compatibility).
diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/modified-2.7/test/test_old_mailbox.py
rename from lib-python/2.7/test/test_old_mailbox.py
rename to lib-python/modified-2.7/test/test_old_mailbox.py
--- a/lib-python/2.7/test/test_old_mailbox.py
+++ b/lib-python/modified-2.7/test/test_old_mailbox.py
@@ -73,7 +73,9 @@
         self.createMessage("cur")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 1)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 
@@ -81,7 +83,9 @@
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 1)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 
@@ -90,8 +94,12 @@
         self.createMessage("new")
         self.mbox = mailbox.Maildir(test_support.TESTFN)
         self.assertTrue(len(self.mbox) == 2)
-        self.assertTrue(self.mbox.next() is not None)
-        self.assertTrue(self.mbox.next() is not None)
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
+        msg = self.mbox.next()
+        self.assertTrue(msg is not None)
+        msg.fp.close()
         self.assertTrue(self.mbox.next() is None)
         self.assertTrue(self.mbox.next() is None)
 
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/modified-2.7/test/test_os.py
rename from lib-python/2.7/test/test_os.py
rename to lib-python/modified-2.7/test/test_os.py
--- a/lib-python/2.7/test/test_os.py
+++ b/lib-python/modified-2.7/test/test_os.py
@@ -74,7 +74,8 @@
         self.assertFalse(os.path.exists(name),
                     "file already exists for temporary file")
         # make sure we can create the file
-        open(name, "w")
+        fid = open(name, "w")
+        fid.close()
         self.files.append(name)
 
     def test_tempnam(self):
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -29,6 +29,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
                         '/EXPORT:init_testcapi']
diff --git a/py/bin/_findpy.py b/py/bin/_findpy.py
new file mode 100644
--- /dev/null
+++ b/py/bin/_findpy.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+#
+# find and import a version of 'py'
+#
+import sys
+import os
+from os.path import dirname as opd, exists, join, basename, abspath
+
+def searchpy(current):
+    while 1:
+        last = current
+        initpy = join(current, '__init__.py')
+        if not exists(initpy):
+            pydir = join(current, 'py')
+            # recognize py-package and ensure it is importable
+            if exists(pydir) and exists(join(pydir, '__init__.py')):
+                #for p in sys.path:
+                #    if p == current:
+                #        return True
+                if current != sys.path[0]:  # if we are already first, then ok
+                    sys.stderr.write("inserting into sys.path: %s\n" % current)
+                    sys.path.insert(0, current)
+                return True
+        current = opd(current)
+        if last == current:
+            return False
+
+if not searchpy(abspath(os.curdir)):
+    if not searchpy(opd(abspath(sys.argv[0]))):
+        if not searchpy(opd(__file__)):
+            pass # let's hope it is just on sys.path
+
+import py
+import pytest
+
+if __name__ == '__main__':
+    print ("py lib is at %s" % py.__file__)
diff --git a/py/bin/py.test b/py/bin/py.test
new file mode 100755
--- /dev/null
+++ b/py/bin/py.test
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from _findpy import pytest
+raise SystemExit(pytest.main())
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -483,7 +483,7 @@
                 return a_str.strip(' ')
             elif n == 1:
                 return a_str.rstrip(' ')
-            else: 
+            else:
                 return a_str.lstrip(' ')
         s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
         assert s.no_nul
@@ -3737,6 +3737,25 @@
         s = a.build_types(f, [int])
         assert s.listdef.listitem.range_step == 0
 
+    def test_specialize_arg_memo(self):
+        @objectmodel.specialize.memo()
+        def g(n):
+            return n
+        @objectmodel.specialize.arg(0)
+        def f(i):
+            return g(i)
+        def main(i):
+            if i == 2:
+                return f(i)
+            elif i == 3:
+                return f(i)
+            else:
+                raise NotImplementedError
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(main, [int])
+        assert isinstance(s, annmodel.SomeInteger)
+
 
 def g(n):
     return [0,1,2,n]
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2253,24 +2253,6 @@
     """Concat two strings giving a new Unicode string."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
-def PyUnicode_Split(space, s, sep, maxsplit):
-    """Split a string giving a list of Unicode strings.  If sep is NULL, splitting
-    will be done at all whitespace substrings.  Otherwise, splits occur at the given
-    separator.  At most maxsplit splits will be done.  If negative, no limit is
-    set.  Separators are not included in the resulting list.
-
-    This function used an int type for maxsplit. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyUnicode_Splitlines(space, s, keepend):
-    """Split a Unicode string at line breaks, returning a list of Unicode strings.
-    CRLF is considered to be one line break.  If keepend is 0, the Line break
-    characters are not included in the resulting strings."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.CCHARP], PyObject)
 def PyUnicode_Translate(space, str, table, errors):
     """Translate a string by applying a character mapping table to it and return the
@@ -2287,29 +2269,6 @@
     use the default error handling."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t, error=-2)
-def PyUnicode_Find(space, str, substr, start, end, direction):
-    """Return the first position of substr in str*[*start:end] using the given
-    direction (direction == 1 means to do a forward search, direction == -1 a
-    backward search).  The return value is the index of the first match; a value of
-    -1 indicates that no match was found, and -2 indicates that an error
-    occurred and an exception has been set.
-
-    This function used an int type for start and end. This
-    might require changes in your code for properly supporting 64-bit
-    systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
-def PyUnicode_Count(space, str, substr, start, end):
-    """Return the number of non-overlapping occurrences of substr in
-    str[start:end].  Return -1 if an error occurred.
-
-    This function returned an int type and used an int
-    type for start and end. This might require changes in your code for
-    properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
 def PyUnicode_RichCompare(space, left, right, op):
     """Rich compare two unicode strings and return one of the following:
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -457,3 +457,29 @@
         assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+
+    def test_count(self, space, api):
+        w_str = space.wrap(u"abcabdab")
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, -1) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, 2) == 1
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), -5, 30) == 2
+
+    def test_find(self, space, api):
+        w_str = space.wrap(u"abcabcd")
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, 1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, 1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
+
+    def test_split(self, space, api):
+        w_str = space.wrap(u"a\nb\nc\nd")
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
+        assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 0)))
+        assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 1)))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -598,3 +598,44 @@
     else:
         return stringtype.stringendswith(str, substr, start, end)
 
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
+def PyUnicode_Count(space, w_str, w_substr, start, end):
+    """Return the number of non-overlapping occurrences of substr in
+    str[start:end].  Return -1 if an error occurred."""
+    w_count = space.call_method(w_str, "count", w_substr,
+                                space.wrap(start), space.wrap(end))
+    return space.int_w(w_count)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
+             Py_ssize_t, error=-2)
+def PyUnicode_Find(space, w_str, w_substr, start, end, direction):
+    """Return the first position of substr in str*[*start:end] using
+    the given direction (direction == 1 means to do a forward search,
+    direction == -1 a backward search).  The return value is the index
+    of the first match; a value of -1 indicates that no match was
+    found, and -2 indicates that an error occurred and an exception
+    has been set."""
+    if rffi.cast(lltype.Signed, direction) > 0:
+        w_pos = space.call_method(w_str, "find", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    else:
+        w_pos = space.call_method(w_str, "rfind", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    return space.int_w(w_pos)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
+def PyUnicode_Split(space, w_str, w_sep, maxsplit):
+    """Split a string giving a list of Unicode strings.  If sep is
+    NULL, splitting will be done at all whitespace substrings.
+    Otherwise, splits occur at the given separator.  At most maxsplit
+    splits will be done.  If negative, no limit is set.  Separators
+    are not included in the resulting list."""
+    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyUnicode_Splitlines(space, w_str, keepend):
+    """Split a Unicode string at line breaks, returning a list of
+    Unicode strings.  CRLF is considered to be one line break.  If
+    keepend is 0, the Line break characters are not included in the
+    resulting strings."""
+    return space.call_method(w_str, "splitlines", space.wrap(keepend))
diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -434,6 +434,13 @@
         self.lastblock = block
         self.pushvalue(w_result)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # This opcode was added with pypy-1.8.  Here is a simpler
+        # version, enough for annotation.
+        last_val = self.popvalue()
+        self.pushvalue(self.space.newlist([]))
+        self.pushvalue(last_val)
+
     # XXX Unimplemented 2.7 opcodes ----------------
 
     # Set literals, set comprehensions

From noreply at buildbot.pypy.org  Wed Apr 11 20:42:36 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 11 Apr 2012 20:42:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: PyUnicode_Split(): support NULL for the
	separator.
Message-ID: <20120411184236.D21C582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54289:ad03b1c52876
Date: 2012-04-11 20:41 +0200
http://bitbucket.org/pypy/pypy/changeset/ad03b1c52876/

Log:	PyUnicode_Split(): support NULL for the separator.

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -479,6 +479,8 @@
                 api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
         assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
                 api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
         assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
                 api.PyUnicode_Splitlines(w_str, 0)))
         assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -630,6 +630,8 @@
     Otherwise, splits occur at the given separator.  At most maxsplit
     splits will be done.  If negative, no limit is set.  Separators
     are not included in the resulting list."""
+    if w_sep is None:
+        w_sep = space.w_None
     return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
 
 @cpython_api([PyObject, rffi.INT_real], PyObject)

From noreply at buildbot.pypy.org  Wed Apr 11 21:15:21 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 21:15:21 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: help find the library,
	wherever it may be
Message-ID: <20120411191521.9FF3182F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54290:d00b67396a4e
Date: 2012-04-11 20:18 +0300
http://bitbucket.org/pypy/pypy/changeset/d00b67396a4e/

Log:	help find the library, wherever it may be

diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -34,6 +34,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST'] # needed for VC10
     else:

From noreply at buildbot.pypy.org  Wed Apr 11 21:15:22 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 21:15:22 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: silence warnings
Message-ID: <20120411191522.DE0CA82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54291:8117f3454026
Date: 2012-04-11 21:06 +0300
http://bitbucket.org/pypy/pypy/changeset/8117f3454026/

Log:	silence warnings

diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -21,7 +21,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC']
     res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -16,7 +16,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
     res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],

From noreply at buildbot.pypy.org  Wed Apr 11 21:15:24 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 21:15:24 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: restore files removed by mistake
Message-ID: <20120411191524.651B582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54292:d1833eeffd0d
Date: 2012-04-11 22:07 +0300
http://bitbucket.org/pypy/pypy/changeset/d1833eeffd0d/

Log:	restore files removed by mistake

diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/mailbox.py
@@ -0,0 +1,2171 @@
+#! /usr/bin/env python
+
+"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
+
+# Notes for authors of new mailbox subclasses:
+#
+# Remember to fsync() changes to disk before closing a modified file
+# or returning from a flush() method.  See functions _sync_flush() and
+# _sync_close().
+
+import sys
+import os
+import time
+import calendar
+import socket
+import errno
+import copy
+import email
+import email.message
+import email.generator
+import StringIO
+try:
+    if sys.platform == 'os2emx':
+        # OS/2 EMX fcntl() not adequate
+        raise ImportError
+    import fcntl
+except ImportError:
+    fcntl = None
+
+import warnings
+with warnings.catch_warnings():
+    if sys.py3kwarning:
+        warnings.filterwarnings("ignore", ".*rfc822 has been removed",
+                                DeprecationWarning)
+    import rfc822
+
+__all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
+            'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
+            'BabylMessage', 'MMDFMessage', 'UnixMailbox',
+            'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
+
+class Mailbox:
+    """A group of messages in a particular place."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a Mailbox instance."""
+        self._path = os.path.abspath(os.path.expanduser(path))
+        self._factory = factory
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def __delitem__(self, key):
+        self.remove(key)
+
+    def discard(self, key):
+        """If the keyed message exists, remove it."""
+        try:
+            self.remove(key)
+        except KeyError:
+            pass
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get(self, key, default=None):
+        """Return the keyed message, or default if it doesn't exist."""
+        try:
+            return self.__getitem__(key)
+        except KeyError:
+            return default
+
+    def __getitem__(self, key):
+        """Return the keyed message; raise KeyError if it doesn't exist."""
+        if not self._factory:
+            return self.get_message(key)
+        else:
+            return self._factory(self.get_file(key))
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def keys(self):
+        """Return a list of keys."""
+        return list(self.iterkeys())
+
+    def itervalues(self):
+        """Return an iterator over all messages."""
+        for key in self.iterkeys():
+            try:
+                value = self[key]
+            except KeyError:
+                continue
+            yield value
+
+    def __iter__(self):
+        return self.itervalues()
+
+    def values(self):
+        """Return a list of messages. Memory intensive."""
+        return list(self.itervalues())
+
+    def iteritems(self):
+        """Return an iterator over (key, message) tuples."""
+        for key in self.iterkeys():
+            try:
+                value = self[key]
+            except KeyError:
+                continue
+            yield (key, value)
+
+    def items(self):
+        """Return a list of (key, message) tuples. Memory intensive."""
+        return list(self.iteritems())
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def __contains__(self, key):
+        return self.has_key(key)
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def clear(self):
+        """Delete all messages."""
+        for key in self.iterkeys():
+            self.discard(key)
+
+    def pop(self, key, default=None):
+        """Delete the keyed message and return it, or default."""
+        try:
+            result = self[key]
+        except KeyError:
+            return default
+        self.discard(key)
+        return result
+
+    def popitem(self):
+        """Delete an arbitrary (key, message) pair and return it."""
+        for key in self.iterkeys():
+            return (key, self.pop(key))     # This is only run once.
+        else:
+            raise KeyError('No messages in mailbox')
+
+    def update(self, arg=None):
+        """Change the messages that correspond to certain keys."""
+        if hasattr(arg, 'iteritems'):
+            source = arg.iteritems()
+        elif hasattr(arg, 'items'):
+            source = arg.items()
+        else:
+            source = arg
+        bad_key = False
+        for key, message in source:
+            try:
+                self[key] = message
+            except KeyError:
+                bad_key = True
+        if bad_key:
+            raise KeyError('No message with key(s)')
+
+    def flush(self):
+        """Write any pending changes to the disk."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def lock(self):
+        """Lock the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def close(self):
+        """Flush and close the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def _dump_message(self, message, target, mangle_from_=False):
+        # Most files are opened in binary mode to allow predictable seeking.
+        # To get native line endings on disk, the user-friendly \n line endings
+        # used in strings and by email.Message are translated here.
+        """Dump message contents to target file."""
+        if isinstance(message, email.message.Message):
+            buffer = StringIO.StringIO()
+            gen = email.generator.Generator(buffer, mangle_from_, 0)
+            gen.flatten(message)
+            buffer.seek(0)
+            target.write(buffer.read().replace('\n', os.linesep))
+        elif isinstance(message, str):
+            if mangle_from_:
+                message = message.replace('\nFrom ', '\n>From ')
+            message = message.replace('\n', os.linesep)
+            target.write(message)
+        elif hasattr(message, 'read'):
+            while True:
+                line = message.readline()
+                if line == '':
+                    break
+                if mangle_from_ and line.startswith('From '):
+                    line = '>From ' + line[5:]
+                line = line.replace('\n', os.linesep)
+                target.write(line)
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+
+
+class Maildir(Mailbox):
+    """A qmail-style Maildir mailbox."""
+
+    colon = ':'
+
+    def __init__(self, dirname, factory=rfc822.Message, create=True):
+        """Initialize a Maildir instance."""
+        Mailbox.__init__(self, dirname, factory, create)
+        self._paths = {
+            'tmp': os.path.join(self._path, 'tmp'),
+            'new': os.path.join(self._path, 'new'),
+            'cur': os.path.join(self._path, 'cur'),
+            }
+        if not os.path.exists(self._path):
+            if create:
+                os.mkdir(self._path, 0700)
+                for path in self._paths.values():
+                    os.mkdir(path, 0o700)
+            else:
+                raise NoSuchMailboxError(self._path)
+        self._toc = {}
+        self._toc_mtimes = {}
+        for subdir in ('cur', 'new'):
+            self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir])
+        self._last_read = time.time()  # Records last time we read cur/new
+        self._skewfactor = 0.1         # Adjust if os/fs clocks are skewing
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        tmp_file = self._create_tmp()
+        try:
+            self._dump_message(message, tmp_file)
+        except BaseException:
+            tmp_file.close()
+            os.remove(tmp_file.name)
+            raise
+        _sync_close(tmp_file)
+        if isinstance(message, MaildirMessage):
+            subdir = message.get_subdir()
+            suffix = self.colon + message.get_info()
+            if suffix == self.colon:
+                suffix = ''
+        else:
+            subdir = 'new'
+            suffix = ''
+        uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
+        dest = os.path.join(self._path, subdir, uniq + suffix)
+        try:
+            if hasattr(os, 'link'):
+                os.link(tmp_file.name, dest)
+                os.remove(tmp_file.name)
+            else:
+                os.rename(tmp_file.name, dest)
+        except OSError, e:
+            os.remove(tmp_file.name)
+            if e.errno == errno.EEXIST:
+                raise ExternalClashError('Name clash with existing message: %s'
+                                         % dest)
+            else:
+                raise
+        if isinstance(message, MaildirMessage):
+            os.utime(dest, (os.path.getatime(dest), message.get_date()))
+        return uniq
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        os.remove(os.path.join(self._path, self._lookup(key)))
+
+    def discard(self, key):
+        """If the keyed message exists, remove it."""
+        # This overrides an inapplicable implementation in the superclass.
+        try:
+            self.remove(key)
+        except KeyError:
+            pass
+        except OSError, e:
+            if e.errno != errno.ENOENT:
+                raise
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        old_subpath = self._lookup(key)
+        temp_key = self.add(message)
+        temp_subpath = self._lookup(temp_key)
+        if isinstance(message, MaildirMessage):
+            # temp's subdir and suffix were specified by message.
+            dominant_subpath = temp_subpath
+        else:
+            # temp's subdir and suffix were defaults from add().
+            dominant_subpath = old_subpath
+        subdir = os.path.dirname(dominant_subpath)
+        if self.colon in dominant_subpath:
+            suffix = self.colon + dominant_subpath.split(self.colon)[-1]
+        else:
+            suffix = ''
+        self.discard(key)
+        new_path = os.path.join(self._path, subdir, key + suffix)
+        os.rename(os.path.join(self._path, temp_subpath), new_path)
+        if isinstance(message, MaildirMessage):
+            os.utime(new_path, (os.path.getatime(new_path),
+                                message.get_date()))
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        subpath = self._lookup(key)
+        f = open(os.path.join(self._path, subpath), 'r')
+        try:
+            if self._factory:
+                msg = self._factory(f)
+            else:
+                msg = MaildirMessage(f)
+        finally:
+            f.close()
+        subdir, name = os.path.split(subpath)
+        msg.set_subdir(subdir)
+        if self.colon in name:
+            msg.set_info(name.split(self.colon)[-1])
+        msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        f = open(os.path.join(self._path, self._lookup(key)), 'r')
+        try:
+            return f.read()
+        finally:
+            f.close()
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        f = open(os.path.join(self._path, self._lookup(key)), 'rb')
+        return _ProxyFile(f)
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        self._refresh()
+        for key in self._toc:
+            try:
+                self._lookup(key)
+            except KeyError:
+                continue
+            yield key
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        self._refresh()
+        return key in self._toc
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        self._refresh()
+        return len(self._toc)
+
+    def flush(self):
+        """Write any pending changes to disk."""
+        # Maildir changes are always written immediately, so there's nothing
+        # to do.
+        pass
+
+    def lock(self):
+        """Lock the mailbox."""
+        return
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        return
+
+    def list_folders(self):
+        """Return a list of folder names."""
+        result = []
+        for entry in os.listdir(self._path):
+            if len(entry) > 1 and entry[0] == '.' and \
+               os.path.isdir(os.path.join(self._path, entry)):
+                result.append(entry[1:])
+        return result
+
+    def get_folder(self, folder):
+        """Return a Maildir instance for the named folder."""
+        return Maildir(os.path.join(self._path, '.' + folder),
+                       factory=self._factory,
+                       create=False)
+
+    def add_folder(self, folder):
+        """Create a folder and return a Maildir instance representing it."""
+        path = os.path.join(self._path, '.' + folder)
+        result = Maildir(path, factory=self._factory)
+        maildirfolder_path = os.path.join(path, 'maildirfolder')
+        if not os.path.exists(maildirfolder_path):
+            os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
+                0666))
+        return result
+
+    def remove_folder(self, folder):
+        """Delete the named folder, which must be empty."""
+        path = os.path.join(self._path, '.' + folder)
+        for entry in os.listdir(os.path.join(path, 'new')) + \
+                     os.listdir(os.path.join(path, 'cur')):
+            if len(entry) < 1 or entry[0] != '.':
+                raise NotEmptyError('Folder contains message(s): %s' % folder)
+        for entry in os.listdir(path):
+            if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
+               os.path.isdir(os.path.join(path, entry)):
+                raise NotEmptyError("Folder contains subdirectory '%s': %s" %
+                                    (folder, entry))
+        for root, dirs, files in os.walk(path, topdown=False):
+            for entry in files:
+                os.remove(os.path.join(root, entry))
+            for entry in dirs:
+                os.rmdir(os.path.join(root, entry))
+        os.rmdir(path)
+
+    def clean(self):
+        """Delete old files in "tmp"."""
+        now = time.time()
+        for entry in os.listdir(os.path.join(self._path, 'tmp')):
+            path = os.path.join(self._path, 'tmp', entry)
+            if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
+                os.remove(path)
+
+    _count = 1  # This is used to generate unique file names.
+
+    def _create_tmp(self):
+        """Create a file in the tmp subdirectory and open and return it."""
+        now = time.time()
+        hostname = socket.gethostname()
+        if '/' in hostname:
+            hostname = hostname.replace('/', r'\057')
+        if ':' in hostname:
+            hostname = hostname.replace(':', r'\072')
+        uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
+                                    Maildir._count, hostname)
+        path = os.path.join(self._path, 'tmp', uniq)
+        try:
+            os.stat(path)
+        except OSError, e:
+            if e.errno == errno.ENOENT:
+                Maildir._count += 1
+                try:
+                    return _create_carefully(path)
+                except OSError, e:
+                    if e.errno != errno.EEXIST:
+                        raise
+            else:
+                raise
+
+        # Fall through to here if stat succeeded or open raised EEXIST.
+        raise ExternalClashError('Name clash prevented file creation: %s' %
+                                 path)
+
+    def _refresh(self):
+        """Update table of contents mapping."""
+        # If it has been less than two seconds since the last _refresh() call,
+        # we have to unconditionally re-read the mailbox just in case it has
+        # been modified, because os.path.mtime() has a 2 sec resolution in the
+        # most common worst case (FAT) and a 1 sec resolution typically.  This
+        # results in a few unnecessary re-reads when _refresh() is called
+        # multiple times in that interval, but once the clock ticks over, we
+        # will only re-read as needed.  Because the filesystem might be being
+        # served by an independent system with its own clock, we record and
+        # compare with the mtimes from the filesystem.  Because the other
+        # system's clock might be skewing relative to our clock, we add an
+        # extra delta to our wait.  The default is one tenth second, but is an
+        # instance variable and so can be adjusted if dealing with a
+        # particularly skewed or irregular system.
+        if time.time() - self._last_read > 2 + self._skewfactor:
+            refresh = False
+            for subdir in self._toc_mtimes:
+                mtime = os.path.getmtime(self._paths[subdir])
+                if mtime > self._toc_mtimes[subdir]:
+                    refresh = True
+                self._toc_mtimes[subdir] = mtime
+            if not refresh:
+                return
+        # Refresh toc
+        self._toc = {}
+        for subdir in self._toc_mtimes:
+            path = self._paths[subdir]
+            for entry in os.listdir(path):
+                p = os.path.join(path, entry)
+                if os.path.isdir(p):
+                    continue
+                uniq = entry.split(self.colon)[0]
+                self._toc[uniq] = os.path.join(subdir, entry)
+        self._last_read = time.time()
+
+    def _lookup(self, key):
+        """Use TOC to return subpath for given key, or raise a KeyError."""
+        try:
+            if os.path.exists(os.path.join(self._path, self._toc[key])):
+                return self._toc[key]
+        except KeyError:
+            pass
+        self._refresh()
+        try:
+            return self._toc[key]
+        except KeyError:
+            raise KeyError('No message with key: %s' % key)
+
+    # This method is for backward compatibility only.
+    def next(self):
+        """Return the next message in a one-time iteration."""
+        if not hasattr(self, '_onetime_keys'):
+            self._onetime_keys = self.iterkeys()
+        while True:
+            try:
+                return self[self._onetime_keys.next()]
+            except StopIteration:
+                return None
+            except KeyError:
+                continue
+
+
+class _singlefileMailbox(Mailbox):
+    """A single-file mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a single-file mailbox."""
+        Mailbox.__init__(self, path, factory, create)
+        try:
+            f = open(self._path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                if create:
+                    f = open(self._path, 'wb+')
+                else:
+                    raise NoSuchMailboxError(self._path)
+            elif e.errno in (errno.EACCES, errno.EROFS):
+                f = open(self._path, 'rb')
+            else:
+                raise
+        self._file = f
+        self._toc = None
+        self._next_key = 0
+        self._pending = False   # No changes require rewriting the file.
+        self._locked = False
+        self._file_length = None        # Used to record mailbox size
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        self._lookup()
+        self._toc[self._next_key] = self._append_message(message)
+        self._next_key += 1
+        self._pending = True
+        return self._next_key - 1
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        self._lookup(key)
+        del self._toc[key]
+        self._pending = True
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        self._lookup(key)
+        self._toc[key] = self._append_message(message)
+        self._pending = True
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        self._lookup()
+        for key in self._toc.keys():
+            yield key
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        self._lookup()
+        return key in self._toc
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        self._lookup()
+        return len(self._toc)
+
+    def lock(self):
+        """Lock the mailbox."""
+        if not self._locked:
+            _lock_file(self._file)
+            self._locked = True
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        if self._locked:
+            _unlock_file(self._file)
+            self._locked = False
+
+    def flush(self):
+        """Write any pending changes to disk."""
+        if not self._pending:
+            return
+
+        # In order to be writing anything out at all, self._toc must
+        # already have been generated (and presumably has been modified
+        # by adding or deleting an item).
+        assert self._toc is not None
+
+        # Check length of self._file; if it's changed, some other process
+        # has modified the mailbox since we scanned it.
+        self._file.seek(0, 2)
+        cur_len = self._file.tell()
+        if cur_len != self._file_length:
+            raise ExternalClashError('Size of mailbox file changed '
+                                     '(expected %i, found %i)' %
+                                     (self._file_length, cur_len))
+
+        new_file = _create_temporary(self._path)
+        try:
+            new_toc = {}
+            self._pre_mailbox_hook(new_file)
+            for key in sorted(self._toc.keys()):
+                start, stop = self._toc[key]
+                self._file.seek(start)
+                self._pre_message_hook(new_file)
+                new_start = new_file.tell()
+                while True:
+                    buffer = self._file.read(min(4096,
+                                                 stop - self._file.tell()))
+                    if buffer == '':
+                        break
+                    new_file.write(buffer)
+                new_toc[key] = (new_start, new_file.tell())
+                self._post_message_hook(new_file)
+        except:
+            new_file.close()
+            os.remove(new_file.name)
+            raise
+        _sync_close(new_file)
+        # self._file is about to get replaced, so no need to sync.
+        self._file.close()
+        try:
+            os.rename(new_file.name, self._path)
+        except OSError, e:
+            if e.errno == errno.EEXIST or \
+              (os.name == 'os2' and e.errno == errno.EACCES):
+                os.remove(self._path)
+                os.rename(new_file.name, self._path)
+            else:
+                raise
+        self._file = open(self._path, 'rb+')
+        self._toc = new_toc
+        self._pending = False
+        if self._locked:
+            _lock_file(self._file, dotlock=False)
+
+    def _pre_mailbox_hook(self, f):
+        """Called before writing the mailbox to file f."""
+        return
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        return
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        self.flush()
+        if self._locked:
+            self.unlock()
+        self._file.close()  # Sync has been done by self.flush() above.
+
+    def _lookup(self, key=None):
+        """Return (start, stop) or raise KeyError."""
+        if self._toc is None:
+            self._generate_toc()
+        if key is not None:
+            try:
+                return self._toc[key]
+            except KeyError:
+                raise KeyError('No message with key: %s' % key)
+
+    def _append_message(self, message):
+        """Append message to mailbox and return (start, stop) offsets."""
+        self._file.seek(0, 2)
+        before = self._file.tell()
+        try:
+            self._pre_message_hook(self._file)
+            offsets = self._install_message(message)
+            self._post_message_hook(self._file)
+        except BaseException:
+            self._file.truncate(before)
+            raise
+        self._file.flush()
+        self._file_length = self._file.tell()  # Record current length of mailbox
+        return offsets
+
+
+
+class _mboxMMDF(_singlefileMailbox):
+    """An mbox or MMDF mailbox."""
+
+    _mangle_from_ = True
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        from_line = self._file.readline().replace(os.linesep, '')
+        string = self._file.read(stop - self._file.tell())
+        msg = self._message_factory(string.replace(os.linesep, '\n'))
+        msg.set_from(from_line[5:])
+        return msg
+
+    def get_string(self, key, from_=False):
+        """Return a string representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        if not from_:
+            self._file.readline()
+        string = self._file.read(stop - self._file.tell())
+        return string.replace(os.linesep, '\n')
+
+    def get_file(self, key, from_=False):
+        """Return a file-like representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        if not from_:
+            self._file.readline()
+        return _PartialFile(self._file, self._file.tell(), stop)
+
+    def _install_message(self, message):
+        """Format a message and blindly write to self._file."""
+        from_line = None
+        if isinstance(message, str) and message.startswith('From '):
+            newline = message.find('\n')
+            if newline != -1:
+                from_line = message[:newline]
+                message = message[newline + 1:]
+            else:
+                from_line = message
+                message = ''
+        elif isinstance(message, _mboxMMDFMessage):
+            from_line = 'From ' + message.get_from()
+        elif isinstance(message, email.message.Message):
+            from_line = message.get_unixfrom()  # May be None.
+        if from_line is None:
+            from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
+        start = self._file.tell()
+        self._file.write(from_line + os.linesep)
+        self._dump_message(message, self._file, self._mangle_from_)
+        stop = self._file.tell()
+        return (start, stop)
+
+
+class mbox(_mboxMMDF):
+    """A classic mbox mailbox."""
+
+    _mangle_from_ = True
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an mbox mailbox."""
+        self._message_factory = mboxMessage
+        _mboxMMDF.__init__(self, path, factory, create)
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        if f.tell() != 0:
+            f.write(os.linesep)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        while True:
+            line_pos = self._file.tell()
+            line = self._file.readline()
+            if line.startswith('From '):
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+                starts.append(line_pos)
+            elif line == '':
+                stops.append(line_pos)
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._next_key = len(self._toc)
+        self._file_length = self._file.tell()
+
+
+class MMDF(_mboxMMDF):
+    """An MMDF mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an MMDF mailbox."""
+        self._message_factory = MMDFMessage
+        _mboxMMDF.__init__(self, path, factory, create)
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        f.write('\001\001\001\001' + os.linesep)
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        f.write(os.linesep + '\001\001\001\001' + os.linesep)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        next_pos = 0
+        while True:
+            line_pos = next_pos
+            line = self._file.readline()
+            next_pos = self._file.tell()
+            if line.startswith('\001\001\001\001' + os.linesep):
+                starts.append(next_pos)
+                while True:
+                    line_pos = next_pos
+                    line = self._file.readline()
+                    next_pos = self._file.tell()
+                    if line == '\001\001\001\001' + os.linesep:
+                        stops.append(line_pos - len(os.linesep))
+                        break
+                    elif line == '':
+                        stops.append(line_pos)
+                        break
+            elif line == '':
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._next_key = len(self._toc)
+        self._file.seek(0, 2)
+        self._file_length = self._file.tell()
+
+
+class MH(Mailbox):
+    """An MH mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an MH instance."""
+        Mailbox.__init__(self, path, factory, create)
+        if not os.path.exists(self._path):
+            if create:
+                os.mkdir(self._path, 0700)
+                os.close(os.open(os.path.join(self._path, '.mh_sequences'),
+                                 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
+            else:
+                raise NoSuchMailboxError(self._path)
+        self._locked = False
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        keys = self.keys()
+        if len(keys) == 0:
+            new_key = 1
+        else:
+            new_key = max(keys) + 1
+        new_path = os.path.join(self._path, str(new_key))
+        f = _create_carefully(new_path)
+        closed = False
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                try:
+                    self._dump_message(message, f)
+                except BaseException:
+                    # Unlock and close so it can be deleted on Windows
+                    if self._locked:
+                        _unlock_file(f)
+                    _sync_close(f)
+                    closed = True
+                    os.remove(new_path)
+                    raise
+                if isinstance(message, MHMessage):
+                    self._dump_sequences(message, new_key)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            if not closed:
+                _sync_close(f)
+        return new_key
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        path = os.path.join(self._path, str(key))
+        try:
+            f = open(path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        else:
+            f.close()
+            os.remove(path)
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        path = os.path.join(self._path, str(key))
+        try:
+            f = open(path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
+                self._dump_message(message, f)
+                if isinstance(message, MHMessage):
+                    self._dump_sequences(message, key)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            _sync_close(f)
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        try:
+            if self._locked:
+                f = open(os.path.join(self._path, str(key)), 'r+')
+            else:
+                f = open(os.path.join(self._path, str(key)), 'r')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                msg = MHMessage(f)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            f.close()
+        for name, key_list in self.get_sequences().iteritems():
+            if key in key_list:
+                msg.add_sequence(name)
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        try:
+            if self._locked:
+                f = open(os.path.join(self._path, str(key)), 'r+')
+            else:
+                f = open(os.path.join(self._path, str(key)), 'r')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                return f.read()
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            f.close()
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        try:
+            f = open(os.path.join(self._path, str(key)), 'rb')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        return _ProxyFile(f)
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        return iter(sorted(int(entry) for entry in os.listdir(self._path)
+                                      if entry.isdigit()))
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        return os.path.exists(os.path.join(self._path, str(key)))
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        return len(list(self.iterkeys()))
+
+    def lock(self):
+        """Lock the mailbox."""
+        if not self._locked:
+            self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
+            _lock_file(self._file)
+            self._locked = True
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        if self._locked:
+            _unlock_file(self._file)
+            _sync_close(self._file)
+            del self._file
+            self._locked = False
+
+    def flush(self):
+        """Write any pending changes to the disk."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        if self._locked:
+            self.unlock()
+
+    def list_folders(self):
+        """Return a list of folder names."""
+        result = []
+        for entry in os.listdir(self._path):
+            if os.path.isdir(os.path.join(self._path, entry)):
+                result.append(entry)
+        return result
+
+    def get_folder(self, folder):
+        """Return an MH instance for the named folder."""
+        return MH(os.path.join(self._path, folder),
+                  factory=self._factory, create=False)
+
+    def add_folder(self, folder):
+        """Create a folder and return an MH instance representing it."""
+        return MH(os.path.join(self._path, folder),
+                  factory=self._factory)
+
+    def remove_folder(self, folder):
+        """Delete the named folder, which must be empty."""
+        path = os.path.join(self._path, folder)
+        entries = os.listdir(path)
+        if entries == ['.mh_sequences']:
+            os.remove(os.path.join(path, '.mh_sequences'))
+        elif entries == []:
+            pass
+        else:
+            raise NotEmptyError('Folder not empty: %s' % self._path)
+        os.rmdir(path)
+
+    def get_sequences(self):
+        """Return a name-to-key-list dictionary to define each sequence."""
+        results = {}
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r')
+        try:
+            all_keys = set(self.keys())
+            for line in f:
+                try:
+                    name, contents = line.split(':')
+                    keys = set()
+                    for spec in contents.split():
+                        if spec.isdigit():
+                            keys.add(int(spec))
+                        else:
+                            start, stop = (int(x) for x in spec.split('-'))
+                            keys.update(range(start, stop + 1))
+                    results[name] = [key for key in sorted(keys) \
+                                         if key in all_keys]
+                    if len(results[name]) == 0:
+                        del results[name]
+                except ValueError:
+                    raise FormatError('Invalid sequence specification: %s' %
+                                      line.rstrip())
+        finally:
+            f.close()
+        return results
+
+    def set_sequences(self, sequences):
+        """Set sequences using the given name-to-key-list dictionary."""
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
+        try:
+            os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
+            for name, keys in sequences.iteritems():
+                if len(keys) == 0:
+                    continue
+                f.write('%s:' % name)
+                prev = None
+                completing = False
+                for key in sorted(set(keys)):
+                    if key - 1 == prev:
+                        if not completing:
+                            completing = True
+                            f.write('-')
+                    elif completing:
+                        completing = False
+                        f.write('%s %s' % (prev, key))
+                    else:
+                        f.write(' %s' % key)
+                    prev = key
+                if completing:
+                    f.write(str(prev) + '\n')
+                else:
+                    f.write('\n')
+        finally:
+            _sync_close(f)
+
+    def pack(self):
+        """Re-name messages to eliminate numbering gaps. Invalidates keys."""
+        sequences = self.get_sequences()
+        prev = 0
+        changes = []
+        for key in self.iterkeys():
+            if key - 1 != prev:
+                changes.append((key, prev + 1))
+                if hasattr(os, 'link'):
+                    os.link(os.path.join(self._path, str(key)),
+                            os.path.join(self._path, str(prev + 1)))
+                    os.unlink(os.path.join(self._path, str(key)))
+                else:
+                    os.rename(os.path.join(self._path, str(key)),
+                              os.path.join(self._path, str(prev + 1)))
+            prev += 1
+        self._next_key = prev + 1
+        if len(changes) == 0:
+            return
+        for name, key_list in sequences.items():
+            for old, new in changes:
+                if old in key_list:
+                    key_list[key_list.index(old)] = new
+        self.set_sequences(sequences)
+
+    def _dump_sequences(self, message, key):
+        """Inspect a new MHMessage and update sequences appropriately."""
+        pending_sequences = message.get_sequences()
+        all_sequences = self.get_sequences()
+        for name, key_list in all_sequences.iteritems():
+            if name in pending_sequences:
+                key_list.append(key)
+            elif key in key_list:
+                del key_list[key_list.index(key)]
+        for sequence in pending_sequences:
+            if sequence not in all_sequences:
+                all_sequences[sequence] = [key]
+        self.set_sequences(all_sequences)
+
+
+class Babyl(_singlefileMailbox):
+    """An Rmail-style Babyl mailbox."""
+
+    _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
+                                 'forwarded', 'edited', 'resent'))
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a Babyl mailbox."""
+        _singlefileMailbox.__init__(self, path, factory, create)
+        self._labels = {}
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        key = _singlefileMailbox.add(self, message)
+        if isinstance(message, BabylMessage):
+            self._labels[key] = message.get_labels()
+        return key
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        _singlefileMailbox.remove(self, key)
+        if key in self._labels:
+            del self._labels[key]
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        _singlefileMailbox.__setitem__(self, key, message)
+        if isinstance(message, BabylMessage):
+            self._labels[key] = message.get_labels()
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        self._file.readline()   # Skip '1,' line specifying labels.
+        original_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == '*** EOOH ***' + os.linesep or line == '':
+                break
+            original_headers.write(line.replace(os.linesep, '\n'))
+        visible_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == os.linesep or line == '':
+                break
+            visible_headers.write(line.replace(os.linesep, '\n'))
+        body = self._file.read(stop - self._file.tell()).replace(os.linesep,
+                                                                 '\n')
+        msg = BabylMessage(original_headers.getvalue() + body)
+        msg.set_visible(visible_headers.getvalue())
+        if key in self._labels:
+            msg.set_labels(self._labels[key])
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        self._file.readline()   # Skip '1,' line specifying labels.
+        original_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == '*** EOOH ***' + os.linesep or line == '':
+                break
+            original_headers.write(line.replace(os.linesep, '\n'))
+        while True:
+            line = self._file.readline()
+            if line == os.linesep or line == '':
+                break
+        return original_headers.getvalue() + \
+               self._file.read(stop - self._file.tell()).replace(os.linesep,
+                                                                 '\n')
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        return StringIO.StringIO(self.get_string(key).replace('\n',
+                                                              os.linesep))
+
+    def get_labels(self):
+        """Return a list of user-defined labels in the mailbox."""
+        self._lookup()
+        labels = set()
+        for label_list in self._labels.values():
+            labels.update(label_list)
+        labels.difference_update(self._special_labels)
+        return list(labels)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        next_pos = 0
+        label_lists = []
+        while True:
+            line_pos = next_pos
+            line = self._file.readline()
+            next_pos = self._file.tell()
+            if line == '\037\014' + os.linesep:
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+                starts.append(next_pos)
+                labels = [label.strip() for label
+                                        in self._file.readline()[1:].split(',')
+                                        if label.strip() != '']
+                label_lists.append(labels)
+            elif line == '\037' or line == '\037' + os.linesep:
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+            elif line == '':
+                stops.append(line_pos - len(os.linesep))
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._labels = dict(enumerate(label_lists))
+        self._next_key = len(self._toc)
+        self._file.seek(0, 2)
+        self._file_length = self._file.tell()
+
+    def _pre_mailbox_hook(self, f):
+        """Called before writing the mailbox to file f."""
+        f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
+                (os.linesep, os.linesep, ','.join(self.get_labels()),
+                 os.linesep))
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        f.write('\014' + os.linesep)
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        f.write(os.linesep + '\037')
+
+    def _install_message(self, message):
+        """Write message contents and return (start, stop)."""
+        start = self._file.tell()
+        if isinstance(message, BabylMessage):
+            special_labels = []
+            labels = []
+            for label in message.get_labels():
+                if label in self._special_labels:
+                    special_labels.append(label)
+                else:
+                    labels.append(label)
+            self._file.write('1')
+            for label in special_labels:
+                self._file.write(', ' + label)
+            self._file.write(',,')
+            for label in labels:
+                self._file.write(' ' + label + ',')
+            self._file.write(os.linesep)
+        else:
+            self._file.write('1,,' + os.linesep)
+        if isinstance(message, email.message.Message):
+            orig_buffer = StringIO.StringIO()
+            orig_generator = email.generator.Generator(orig_buffer, False, 0)
+            orig_generator.flatten(message)
+            orig_buffer.seek(0)
+            while True:
+                line = orig_buffer.readline()
+                self._file.write(line.replace('\n', os.linesep))
+                if line == '\n' or line == '':
+                    break
+            self._file.write('*** EOOH ***' + os.linesep)
+            if isinstance(message, BabylMessage):
+                vis_buffer = StringIO.StringIO()
+                vis_generator = email.generator.Generator(vis_buffer, False, 0)
+                vis_generator.flatten(message.get_visible())
+                while True:
+                    line = vis_buffer.readline()
+                    self._file.write(line.replace('\n', os.linesep))
+                    if line == '\n' or line == '':
+                        break
+            else:
+                orig_buffer.seek(0)
+                while True:
+                    line = orig_buffer.readline()
+                    self._file.write(line.replace('\n', os.linesep))
+                    if line == '\n' or line == '':
+                        break
+            while True:
+                buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
+                if buffer == '':
+                    break
+                self._file.write(buffer.replace('\n', os.linesep))
+        elif isinstance(message, str):
+            body_start = message.find('\n\n') + 2
+            if body_start - 2 != -1:
+                self._file.write(message[:body_start].replace('\n',
+                                                              os.linesep))
+                self._file.write('*** EOOH ***' + os.linesep)
+                self._file.write(message[:body_start].replace('\n',
+                                                              os.linesep))
+                self._file.write(message[body_start:].replace('\n',
+                                                              os.linesep))
+            else:
+                self._file.write('*** EOOH ***' + os.linesep + os.linesep)
+                self._file.write(message.replace('\n', os.linesep))
+        elif hasattr(message, 'readline'):
+            original_pos = message.tell()
+            first_pass = True
+            while True:
+                line = message.readline()
+                self._file.write(line.replace('\n', os.linesep))
+                if line == '\n' or line == '':
+                    self._file.write('*** EOOH ***' + os.linesep)
+                    if first_pass:
+                        first_pass = False
+                        message.seek(original_pos)
+                    else:
+                        break
+            while True:
+                buffer = message.read(4096)     # Buffer size is arbitrary.
+                if buffer == '':
+                    break
+                self._file.write(buffer.replace('\n', os.linesep))
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+        stop = self._file.tell()
+        return (start, stop)
+
+
+class Message(email.message.Message):
+    """Message with mailbox-format-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize a Message instance."""
+        if isinstance(message, email.message.Message):
+            self._become_message(copy.deepcopy(message))
+            if isinstance(message, Message):
+                message._explain_to(self)
+        elif isinstance(message, str):
+            self._become_message(email.message_from_string(message))
+        elif hasattr(message, "read"):
+            self._become_message(email.message_from_file(message))
+        elif message is None:
+            email.message.Message.__init__(self)
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+
+    def _become_message(self, message):
+        """Assume the non-format-specific state of message."""
+        for name in ('_headers', '_unixfrom', '_payload', '_charset',
+                     'preamble', 'epilogue', 'defects', '_default_type'):
+            self.__dict__[name] = message.__dict__[name]
+
+    def _explain_to(self, message):
+        """Copy format-specific state to message insofar as possible."""
+        if isinstance(message, Message):
+            return  # There's nothing format-specific to explain.
+        else:
+            raise TypeError('Cannot convert to specified type')
+
+
+class MaildirMessage(Message):
+    """Message with Maildir-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize a MaildirMessage instance."""
+        self._subdir = 'new'
+        self._info = ''
+        self._date = time.time()
+        Message.__init__(self, message)
+
+    def get_subdir(self):
+        """Return 'new' or 'cur'."""
+        return self._subdir
+
+    def set_subdir(self, subdir):
+        """Set subdir to 'new' or 'cur'."""
+        if subdir == 'new' or subdir == 'cur':
+            self._subdir = subdir
+        else:
+            raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
+
+    def get_flags(self):
+        """Return as a string the flags that are set."""
+        if self._info.startswith('2,'):
+            return self._info[2:]
+        else:
+            return ''
+
+    def set_flags(self, flags):
+        """Set the given flags and unset all others."""
+        self._info = '2,' + ''.join(sorted(flags))
+
+    def add_flag(self, flag):
+        """Set the given flag(s) without changing others."""
+        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+    def remove_flag(self, flag):
+        """Unset the given string flag(s) without changing others."""
+        if self.get_flags() != '':
+            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+    def get_date(self):
+        """Return delivery date of message, in seconds since the epoch."""
+        return self._date
+
+    def set_date(self, date):
+        """Set delivery date of message, in seconds since the epoch."""
+        try:
+            self._date = float(date)
+        except ValueError:
+            raise TypeError("can't convert to float: %s" % date)
+
+    def get_info(self):
+        """Get the message's "info" as a string."""
+        return self._info
+
+    def set_info(self, info):
+        """Set the message's "info" string."""
+        if isinstance(info, str):
+            self._info = info
+        else:
+            raise TypeError('info must be a string: %s' % type(info))
+
+    def _explain_to(self, message):
+        """Copy Maildir-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            message.set_flags(self.get_flags())
+            message.set_subdir(self.get_subdir())
+            message.set_date(self.get_date())
+        elif isinstance(message, _mboxMMDFMessage):
+            flags = set(self.get_flags())
+            if 'S' in flags:
+                message.add_flag('R')
+            if self.get_subdir() == 'cur':
+                message.add_flag('O')
+            if 'T' in flags:
+                message.add_flag('D')
+            if 'F' in flags:
+                message.add_flag('F')
+            if 'R' in flags:
+                message.add_flag('A')
+            message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
+        elif isinstance(message, MHMessage):
+            flags = set(self.get_flags())
+            if 'S' not in flags:
+                message.add_sequence('unseen')
+            if 'R' in flags:
+                message.add_sequence('replied')
+            if 'F' in flags:
+                message.add_sequence('flagged')
+        elif isinstance(message, BabylMessage):
+            flags = set(self.get_flags())
+            if 'S' not in flags:
+                message.add_label('unseen')
+            if 'T' in flags:
+                message.add_label('deleted')
+            if 'R' in flags:
+                message.add_label('answered')
+            if 'P' in flags:
+                message.add_label('forwarded')
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class _mboxMMDFMessage(Message):
+    """Message with mbox- or MMDF-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an mboxMMDFMessage instance."""
+        self.set_from('MAILER-DAEMON', True)
+        if isinstance(message, email.message.Message):
+            unixfrom = message.get_unixfrom()
+            if unixfrom is not None and unixfrom.startswith('From '):
+                self.set_from(unixfrom[5:])
+        Message.__init__(self, message)
+
+    def get_from(self):
+        """Return contents of "From " line."""
+        return self._from
+
+    def set_from(self, from_, time_=None):
+        """Set "From " line, formatting and appending time_ if specified."""
+        if time_ is not None:
+            if time_ is True:
+                time_ = time.gmtime()
+            from_ += ' ' + time.asctime(time_)
+        self._from = from_
+
+    def get_flags(self):
+        """Return as a string the flags that are set."""
+        return self.get('Status', '') + self.get('X-Status', '')
+
+    def set_flags(self, flags):
+        """Set the given flags and unset all others."""
+        flags = set(flags)
+        status_flags, xstatus_flags = '', ''
+        for flag in ('R', 'O'):
+            if flag in flags:
+                status_flags += flag
+                flags.remove(flag)
+        for flag in ('D', 'F', 'A'):
+            if flag in flags:
+                xstatus_flags += flag
+                flags.remove(flag)
+        xstatus_flags += ''.join(sorted(flags))
+        try:
+            self.replace_header('Status', status_flags)
+        except KeyError:
+            self.add_header('Status', status_flags)
+        try:
+            self.replace_header('X-Status', xstatus_flags)
+        except KeyError:
+            self.add_header('X-Status', xstatus_flags)
+
+    def add_flag(self, flag):
+        """Set the given flag(s) without changing others."""
+        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+    def remove_flag(self, flag):
+        """Unset the given string flag(s) without changing others."""
+        if 'Status' in self or 'X-Status' in self:
+            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+    def _explain_to(self, message):
+        """Copy mbox- or MMDF-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            flags = set(self.get_flags())
+            if 'O' in flags:
+                message.set_subdir('cur')
+            if 'F' in flags:
+                message.add_flag('F')
+            if 'A' in flags:
+                message.add_flag('R')
+            if 'R' in flags:
+                message.add_flag('S')
+            if 'D' in flags:
+                message.add_flag('T')
+            del message['status']
+            del message['x-status']
+            maybe_date = ' '.join(self.get_from().split()[-5:])
+            try:
+                message.set_date(calendar.timegm(time.strptime(maybe_date,
+                                                      '%a %b %d %H:%M:%S %Y')))
+            except (ValueError, OverflowError):
+                pass
+        elif isinstance(message, _mboxMMDFMessage):
+            message.set_flags(self.get_flags())
+            message.set_from(self.get_from())
+        elif isinstance(message, MHMessage):
+            flags = set(self.get_flags())
+            if 'R' not in flags:
+                message.add_sequence('unseen')
+            if 'A' in flags:
+                message.add_sequence('replied')
+            if 'F' in flags:
+                message.add_sequence('flagged')
+            del message['status']
+            del message['x-status']
+        elif isinstance(message, BabylMessage):
+            flags = set(self.get_flags())
+            if 'R' not in flags:
+                message.add_label('unseen')
+            if 'D' in flags:
+                message.add_label('deleted')
+            if 'A' in flags:
+                message.add_label('answered')
+            del message['status']
+            del message['x-status']
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class mboxMessage(_mboxMMDFMessage):
+    """Message with mbox-specific properties."""
+
+
+class MHMessage(Message):
+    """Message with MH-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an MHMessage instance."""
+        self._sequences = []
+        Message.__init__(self, message)
+
+    def get_sequences(self):
+        """Return a list of sequences that include the message."""
+        return self._sequences[:]
+
+    def set_sequences(self, sequences):
+        """Set the list of sequences that include the message."""
+        self._sequences = list(sequences)
+
+    def add_sequence(self, sequence):
+        """Add sequence to list of sequences including the message."""
+        if isinstance(sequence, str):
+            if not sequence in self._sequences:
+                self._sequences.append(sequence)
+        else:
+            raise TypeError('sequence must be a string: %s' % type(sequence))
+
+    def remove_sequence(self, sequence):
+        """Remove sequence from the list of sequences including the message."""
+        try:
+            self._sequences.remove(sequence)
+        except ValueError:
+            pass
+
+    def _explain_to(self, message):
+        """Copy MH-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' in sequences:
+                message.set_subdir('cur')
+            else:
+                message.set_subdir('cur')
+                message.add_flag('S')
+            if 'flagged' in sequences:
+                message.add_flag('F')
+            if 'replied' in sequences:
+                message.add_flag('R')
+        elif isinstance(message, _mboxMMDFMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' not in sequences:
+                message.add_flag('RO')
+            else:
+                message.add_flag('O')
+            if 'flagged' in sequences:
+                message.add_flag('F')
+            if 'replied' in sequences:
+                message.add_flag('A')
+        elif isinstance(message, MHMessage):
+            for sequence in self.get_sequences():
+                message.add_sequence(sequence)
+        elif isinstance(message, BabylMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' in sequences:
+                message.add_label('unseen')
+            if 'replied' in sequences:
+                message.add_label('answered')
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class BabylMessage(Message):
+    """Message with Babyl-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an BabylMessage instance."""
+        self._labels = []
+        self._visible = Message()
+        Message.__init__(self, message)
+
+    def get_labels(self):
+        """Return a list of labels on the message."""
+        return self._labels[:]
+
+    def set_labels(self, labels):
+        """Set the list of labels on the message."""
+        self._labels = list(labels)
+
+    def add_label(self, label):
+        """Add label to list of labels on the message."""
+        if isinstance(label, str):
+            if label not in self._labels:
+                self._labels.append(label)
+        else:
+            raise TypeError('label must be a string: %s' % type(label))
+
+    def remove_label(self, label):
+        """Remove label from the list of labels on the message."""
+        try:
+            self._labels.remove(label)
+        except ValueError:
+            pass
+
+    def get_visible(self):
+        """Return a Message representation of visible headers."""
+        return Message(self._visible)
+
+    def set_visible(self, visible):
+        """Set the Message representation of visible headers."""
+        self._visible = Message(visible)
+
+    def update_visible(self):
+        """Update and/or sensibly generate a set of visible headers."""
+        for header in self._visible.keys():
+            if header in self:
+                self._visible.replace_header(header, self[header])
+            else:
+                del self._visible[header]
+        for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
+            if header in self and header not in self._visible:
+                self._visible[header] = self[header]
+
+    def _explain_to(self, message):
+        """Copy Babyl-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            labels = set(self.get_labels())
+            if 'unseen' in labels:
+                message.set_subdir('cur')
+            else:
+                message.set_subdir('cur')
+                message.add_flag('S')
+            if 'forwarded' in labels or 'resent' in labels:
+                message.add_flag('P')
+            if 'answered' in labels:
+                message.add_flag('R')
+            if 'deleted' in labels:
+                message.add_flag('T')
+        elif isinstance(message, _mboxMMDFMessage):
+            labels = set(self.get_labels())
+            if 'unseen' not in labels:
+                message.add_flag('RO')
+            else:
+                message.add_flag('O')
+            if 'deleted' in labels:
+                message.add_flag('D')
+            if 'answered' in labels:
+                message.add_flag('A')
+        elif isinstance(message, MHMessage):
+            labels = set(self.get_labels())
+            if 'unseen' in labels:
+                message.add_sequence('unseen')
+            if 'answered' in labels:
+                message.add_sequence('replied')
+        elif isinstance(message, BabylMessage):
+            message.set_visible(self.get_visible())
+            for label in self.get_labels():
+                message.add_label(label)
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class MMDFMessage(_mboxMMDFMessage):
+    """Message with MMDF-specific properties."""
+
+
+class _ProxyFile:
+    """A read-only wrapper of a file."""
+
+    def __init__(self, f, pos=None):
+        """Initialize a _ProxyFile."""
+        self._file = f
+        if pos is None:
+            self._pos = f.tell()
+        else:
+            self._pos = pos
+
+    def read(self, size=None):
+        """Read bytes."""
+        return self._read(size, self._file.read)
+
+    def readline(self, size=None):
+        """Read a line."""
+        return self._read(size, self._file.readline)
+
+    def readlines(self, sizehint=None):
+        """Read multiple lines."""
+        result = []
+        for line in self:
+            result.append(line)
+            if sizehint is not None:
+                sizehint -= len(line)
+                if sizehint <= 0:
+                    break
+        return result
+
+    def __iter__(self):
+        """Iterate over lines."""
+        return iter(self.readline, "")
+
+    def tell(self):
+        """Return the position."""
+        return self._pos
+
+    def seek(self, offset, whence=0):
+        """Change position."""
+        if whence == 1:
+            self._file.seek(self._pos)
+        self._file.seek(offset, whence)
+        self._pos = self._file.tell()
+
+    def close(self):
+        """Close the file."""
+        del self._file
+
+    def _read(self, size, read_method):
+        """Read size bytes using read_method."""
+        if size is None:
+            size = -1
+        self._file.seek(self._pos)
+        result = read_method(size)
+        self._pos = self._file.tell()
+        return result
+
+
+class _PartialFile(_ProxyFile):
+    """A read-only wrapper of part of a file."""
+
+    def __init__(self, f, start=None, stop=None):
+        """Initialize a _PartialFile."""
+        _ProxyFile.__init__(self, f, start)
+        self._start = start
+        self._stop = stop
+
+    def tell(self):
+        """Return the position with respect to start."""
+        return _ProxyFile.tell(self) - self._start
+
+    def seek(self, offset, whence=0):
+        """Change position, possibly with respect to start or stop."""
+        if whence == 0:
+            self._pos = self._start
+            whence = 1
+        elif whence == 2:
+            self._pos = self._stop
+            whence = 1
+        _ProxyFile.seek(self, offset, whence)
+
+    def _read(self, size, read_method):
+        """Read size bytes using read_method, honoring start and stop."""
+        remaining = self._stop - self._pos
+        if remaining <= 0:
+            return ''
+        if size is None or size < 0 or size > remaining:
+            size = remaining
+        return _ProxyFile._read(self, size, read_method)
+
+
+def _lock_file(f, dotlock=True):
+    """Lock file f using lockf and dot locking."""
+    dotlock_done = False
+    try:
+        if fcntl:
+            try:
+                fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except IOError, e:
+                if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
+                    raise ExternalClashError('lockf: lock unavailable: %s' %
+                                             f.name)
+                else:
+                    raise
+        if dotlock:
+            try:
+                pre_lock = _create_temporary(f.name + '.lock')
+                pre_lock.close()
+            except IOError, e:
+                if e.errno in (errno.EACCES, errno.EROFS):
+                    return  # Without write access, just skip dotlocking.
+                else:
+                    raise
+            try:
+                if hasattr(os, 'link'):
+                    os.link(pre_lock.name, f.name + '.lock')
+                    dotlock_done = True
+                    os.unlink(pre_lock.name)
+                else:
+                    os.rename(pre_lock.name, f.name + '.lock')
+                    dotlock_done = True
+            except OSError, e:
+                if e.errno == errno.EEXIST or \
+                  (os.name == 'os2' and e.errno == errno.EACCES):
+                    os.remove(pre_lock.name)
+                    raise ExternalClashError('dot lock unavailable: %s' %
+                                             f.name)
+                else:
+                    raise
+    except:
+        if fcntl:
+            fcntl.lockf(f, fcntl.LOCK_UN)
+        if dotlock_done:
+            os.remove(f.name + '.lock')
+        raise
+
+def _unlock_file(f):
+    """Unlock file f using lockf and dot locking."""
+    if fcntl:
+        fcntl.lockf(f, fcntl.LOCK_UN)
+    if os.path.exists(f.name + '.lock'):
+        os.remove(f.name + '.lock')
+
+def _create_carefully(path):
+    """Create a file if it doesn't exist and open for reading and writing."""
+    fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
+    try:
+        return open(path, 'rb+')
+    finally:
+        os.close(fd)
+
+def _create_temporary(path):
+    """Create a temp file based on path and open for reading and writing."""
+    return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
+                                              socket.gethostname(),
+                                              os.getpid()))
+
+def _sync_flush(f):
+    """Ensure changes to file f are physically on disk."""
+    f.flush()
+    if hasattr(os, 'fsync'):
+        os.fsync(f.fileno())
+
+def _sync_close(f):
+    """Close file f, ensuring all changes are physically on disk."""
+    _sync_flush(f)
+    f.close()
+
+## Start: classes from the original module (for backward compatibility).
+
+# Note that the Maildir class, whose name is unchanged, itself offers a next()
+# method for backward compatibility.
+
+class _Mailbox:
+
+    def __init__(self, fp, factory=rfc822.Message):
+        self.fp = fp
+        self.seekp = 0
+        self.factory = factory
+
+    def __iter__(self):
+        return iter(self.next, None)
+
+    def next(self):
+        while 1:
+            self.fp.seek(self.seekp)
+            try:
+                self._search_start()
+            except EOFError:
+                self.seekp = self.fp.tell()
+                return None
+            start = self.fp.tell()
+            self._search_end()
+            self.seekp = stop = self.fp.tell()
+            if start != stop:
+                break
+        return self.factory(_PartialFile(self.fp, start, stop))
+
+# Recommended to use PortableUnixMailbox instead!
+class UnixMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line[:5] == 'From ' and self._isrealfromline(line):
+                self.fp.seek(pos)
+                return
+
+    def _search_end(self):
+        self.fp.readline()      # Throw away header line
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line[:5] == 'From ' and self._isrealfromline(line):
+                self.fp.seek(pos)
+                return
+
+    # An overridable mechanism to test for From-line-ness.  You can either
+    # specify a different regular expression or define a whole new
+    # _isrealfromline() method.  Note that this only gets called for lines
+    # starting with the 5 characters "From ".
+    #
+    # BAW: According to
+    #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
+    # the only portable, reliable way to find message delimiters in a BSD (i.e
+    # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
+    # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
+    # like a good idea, in practice, there are too many variations for more
+    # strict parsing of the line to be completely accurate.
+    #
+    # _strict_isrealfromline() is the old version which tries to do stricter
+    # parsing of the From_ line.  _portable_isrealfromline() simply returns
+    # true, since it's never called if the line doesn't already start with
+    # "From ".
+    #
+    # This algorithm, and the way it interacts with _search_start() and
+    # _search_end() may not be completely correct, because it doesn't check
+    # that the two characters preceding "From " are \n\n or the beginning of
+    # the file.  Fixing this would require a more extensive rewrite than is
+    # necessary.  For convenience, we've added a PortableUnixMailbox class
+    # which does no checking of the format of the 'From' line.
+
+    _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
+                        r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
+                        r"[^\s]*\s*"
+                        "$")
+    _regexp = None
+
+    def _strict_isrealfromline(self, line):
+        if not self._regexp:
+            import re
+            self._regexp = re.compile(self._fromlinepattern)
+        return self._regexp.match(line)
+
+    def _portable_isrealfromline(self, line):
+        return True
+
+    _isrealfromline = _strict_isrealfromline
+
+
+class PortableUnixMailbox(UnixMailbox):
+    _isrealfromline = UnixMailbox._portable_isrealfromline
+
+
+class MmdfMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line[:5] == '\001\001\001\001\n':
+                return
+
+    def _search_end(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line == '\001\001\001\001\n':
+                self.fp.seek(pos)
+                return
+
+
+class MHMailbox:
+
+    def __init__(self, dirname, factory=rfc822.Message):
+        import re
+        pat = re.compile('^[1-9][0-9]*$')
+        self.dirname = dirname
+        # the three following lines could be combined into:
+        # list = map(long, filter(pat.match, os.listdir(self.dirname)))
+        list = os.listdir(self.dirname)
+        list = filter(pat.match, list)
+        list = map(long, list)
+        list.sort()
+        # This only works in Python 1.6 or later;
+        # before that str() added 'L':
+        self.boxes = map(str, list)
+        self.boxes.reverse()
+        self.factory = factory
+
+    def __iter__(self):
+        return iter(self.next, None)
+
+    def next(self):
+        if not self.boxes:
+            return None
+        fn = self.boxes.pop()
+        fp = open(os.path.join(self.dirname, fn))
+        msg = self.factory(fp)
+        try:
+            msg._mh_msgno = fn
+        except (AttributeError, TypeError):
+            pass
+        return msg
+
+
+class BabylMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line == '*** EOOH ***\n':
+                return
+
+    def _search_end(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line == '\037\014\n' or line == '\037':
+                self.fp.seek(pos)
+                return
+
+## End: classes from the original module (for backward compatibility).
+
+
+class Error(Exception):
+    """Raised for module-specific errors."""
+
+class NoSuchMailboxError(Error):
+    """The specified mailbox does not exist and won't be created."""
+
+class NotEmptyError(Error):
+    """The specified mailbox is not empty and deletion was requested."""
+
+class ExternalClashError(Error):
+    """Another process caused an action to fail."""
+
+class FormatError(Error):
+    """A file appears to have an invalid format."""
diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/2.7/test/test_mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_mailbox.py
@@ -0,0 +1,1970 @@
+import os
+import sys
+import time
+import stat
+import socket
+import email
+import email.message
+import re
+import StringIO
+from test import test_support
+import unittest
+import mailbox
+import glob
+try:
+    import fcntl
+except ImportError:
+    pass
+
+# Silence Py3k warning
+rfc822 = test_support.import_module('rfc822', deprecated=True)
+
+class TestBase(unittest.TestCase):
+
+    def _check_sample(self, msg):
+        # Inspect a mailbox.Message representation of the sample message
+        self.assertIsInstance(msg, email.message.Message)
+        self.assertIsInstance(msg, mailbox.Message)
+        for key, value in _sample_headers.iteritems():
+            self.assertIn(value, msg.get_all(key))
+        self.assertTrue(msg.is_multipart())
+        self.assertEqual(len(msg.get_payload()), len(_sample_payloads))
+        for i, payload in enumerate(_sample_payloads):
+            part = msg.get_payload(i)
+            self.assertIsInstance(part, email.message.Message)
+            self.assertNotIsInstance(part, mailbox.Message)
+            self.assertEqual(part.get_payload(), payload)
+
+    def _delete_recursively(self, target):
+        # Delete a file or delete a directory recursively
+        if os.path.isdir(target):
+            for path, dirs, files in os.walk(target, topdown=False):
+                for name in files:
+                    os.remove(os.path.join(path, name))
+                for name in dirs:
+                    os.rmdir(os.path.join(path, name))
+            os.rmdir(target)
+        elif os.path.exists(target):
+            os.remove(target)
+
+
+class TestMailbox(TestBase):
+
+    _factory = None     # Overridden by subclasses to reuse tests
+    _template = 'From: foo\n\n%s'
+
+    def setUp(self):
+        self._path = test_support.TESTFN
+        self._delete_recursively(self._path)
+        self._box = self._factory(self._path)
+
+    def tearDown(self):
+        self._box.close()
+        self._delete_recursively(self._path)
+
+    def test_add(self):
+        # Add copies of a sample message
+        keys = []
+        keys.append(self._box.add(self._template % 0))
+        self.assertEqual(len(self._box), 1)
+        keys.append(self._box.add(mailbox.Message(_sample_message)))
+        self.assertEqual(len(self._box), 2)
+        keys.append(self._box.add(email.message_from_string(_sample_message)))
+        self.assertEqual(len(self._box), 3)
+        keys.append(self._box.add(StringIO.StringIO(_sample_message)))
+        self.assertEqual(len(self._box), 4)
+        keys.append(self._box.add(_sample_message))
+        self.assertEqual(len(self._box), 5)
+        self.assertEqual(self._box.get_string(keys[0]), self._template % 0)
+        for i in (1, 2, 3, 4):
+            self._check_sample(self._box[keys[i]])
+
+    def test_remove(self):
+        # Remove messages using remove()
+        self._test_remove_or_delitem(self._box.remove)
+
+    def test_delitem(self):
+        # Remove messages using __delitem__()
+        self._test_remove_or_delitem(self._box.__delitem__)
+
+    def _test_remove_or_delitem(self, method):
+        # (Used by test_remove() and test_delitem().)
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(self._template % 1)
+        self.assertEqual(len(self._box), 2)
+        method(key0)
+        l = len(self._box)
+        self.assertEqual(l, 1)
+        self.assertRaises(KeyError, lambda: self._box[key0])
+        self.assertRaises(KeyError, lambda: method(key0))
+        self.assertEqual(self._box.get_string(key1), self._template % 1)
+        key2 = self._box.add(self._template % 2)
+        self.assertEqual(len(self._box), 2)
+        method(key2)
+        l = len(self._box)
+        self.assertEqual(l, 1)
+        self.assertRaises(KeyError, lambda: self._box[key2])
+        self.assertRaises(KeyError, lambda: method(key2))
+        self.assertEqual(self._box.get_string(key1), self._template % 1)
+        method(key1)
+        self.assertEqual(len(self._box), 0)
+        self.assertRaises(KeyError, lambda: self._box[key1])
+        self.assertRaises(KeyError, lambda: method(key1))
+
+    def test_discard(self, repetitions=10):
+        # Discard messages
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(self._template % 1)
+        self.assertEqual(len(self._box), 2)
+        self._box.discard(key0)
+        self.assertEqual(len(self._box), 1)
+        self.assertRaises(KeyError, lambda: self._box[key0])
+        self._box.discard(key0)
+        self.assertEqual(len(self._box), 1)
+        self.assertRaises(KeyError, lambda: self._box[key0])
+
+    def test_get(self):
+        # Retrieve messages using get()
+        key0 = self._box.add(self._template % 0)
+        msg = self._box.get(key0)
+        self.assertEqual(msg['from'], 'foo')
+        self.assertEqual(msg.get_payload(), '0')
+        self.assertIs(self._box.get('foo'), None)
+        self.assertFalse(self._box.get('foo', False))
+        self._box.close()
+        self._box = self._factory(self._path, factory=rfc822.Message)
+        key1 = self._box.add(self._template % 1)
+        msg = self._box.get(key1)
+        self.assertEqual(msg['from'], 'foo')
+        self.assertEqual(msg.fp.read(), '1')
+
+    def test_getitem(self):
+        # Retrieve message using __getitem__()
+        key0 = self._box.add(self._template % 0)
+        msg = self._box[key0]
+        self.assertEqual(msg['from'], 'foo')
+        self.assertEqual(msg.get_payload(), '0')
+        self.assertRaises(KeyError, lambda: self._box['foo'])
+        self._box.discard(key0)
+        self.assertRaises(KeyError, lambda: self._box[key0])
+
+    def test_get_message(self):
+        # Get Message representations of messages
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(_sample_message)
+        msg0 = self._box.get_message(key0)
+        self.assertIsInstance(msg0, mailbox.Message)
+        self.assertEqual(msg0['from'], 'foo')
+        self.assertEqual(msg0.get_payload(), '0')
+        self._check_sample(self._box.get_message(key1))
+
+    def test_get_string(self):
+        # Get string representations of messages
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(_sample_message)
+        self.assertEqual(self._box.get_string(key0), self._template % 0)
+        self.assertEqual(self._box.get_string(key1), _sample_message)
+
+    def test_get_file(self):
+        # Get file representations of messages
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(_sample_message)
+        self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'),
+                         self._template % 0)
+        self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'),
+                         _sample_message)
+
+    def test_iterkeys(self):
+        # Get keys using iterkeys()
+        self._check_iteration(self._box.iterkeys, do_keys=True, do_values=False)
+
+    def test_keys(self):
+        # Get keys using keys()
+        self._check_iteration(self._box.keys, do_keys=True, do_values=False)
+
+    def test_itervalues(self):
+        # Get values using itervalues()
+        self._check_iteration(self._box.itervalues, do_keys=False,
+                              do_values=True)
+
+    def test_iter(self):
+        # Get values using __iter__()
+        self._check_iteration(self._box.__iter__, do_keys=False,
+                              do_values=True)
+
+    def test_values(self):
+        # Get values using values()
+        self._check_iteration(self._box.values, do_keys=False, do_values=True)
+
+    def test_iteritems(self):
+        # Get keys and values using iteritems()
+        self._check_iteration(self._box.iteritems, do_keys=True,
+                              do_values=True)
+
+    def test_items(self):
+        # Get keys and values using items()
+        self._check_iteration(self._box.items, do_keys=True, do_values=True)
+
+    def _check_iteration(self, method, do_keys, do_values, repetitions=10):
+        for value in method():
+            self.fail("Not empty")
+        keys, values = [], []
+        for i in xrange(repetitions):
+            keys.append(self._box.add(self._template % i))
+            values.append(self._template % i)
+        if do_keys and not do_values:
+            returned_keys = list(method())
+        elif do_values and not do_keys:
+            returned_values = list(method())
+        else:
+            returned_keys, returned_values = [], []
+            for key, value in method():
+                returned_keys.append(key)
+                returned_values.append(value)
+        if do_keys:
+            self.assertEqual(len(keys), len(returned_keys))
+            self.assertEqual(set(keys), set(returned_keys))
+        if do_values:
+            count = 0
+            for value in returned_values:
+                self.assertEqual(value['from'], 'foo')
+                self.assertTrue(int(value.get_payload()) < repetitions,
+                                (value.get_payload(), repetitions))
+                count += 1
+            self.assertEqual(len(values), count)
+
+    def test_has_key(self):
+        # Check existence of keys using has_key()
+        self._test_has_key_or_contains(self._box.has_key)
+
+    def test_contains(self):
+        # Check existence of keys using __contains__()
+        self._test_has_key_or_contains(self._box.__contains__)
+
+    def _test_has_key_or_contains(self, method):
+        # (Used by test_has_key() and test_contains().)
+        self.assertFalse(method('foo'))
+        key0 = self._box.add(self._template % 0)
+        self.assertTrue(method(key0))
+        self.assertFalse(method('foo'))
+        key1 = self._box.add(self._template % 1)
+        self.assertTrue(method(key1))
+        self.assertTrue(method(key0))
+        self.assertFalse(method('foo'))
+        self._box.remove(key0)
+        self.assertFalse(method(key0))
+        self.assertTrue(method(key1))
+        self.assertFalse(method('foo'))
+        self._box.remove(key1)
+        self.assertFalse(method(key1))
+        self.assertFalse(method(key0))
+        self.assertFalse(method('foo'))
+
+    def test_len(self, repetitions=10):
+        # Get message count
+        keys = []
+        for i in xrange(repetitions):
+            self.assertEqual(len(self._box), i)
+            keys.append(self._box.add(self._template % i))
+            self.assertEqual(len(self._box),  i + 1)
+        for i in xrange(repetitions):
+            self.assertEqual(len(self._box), repetitions - i)
+            self._box.remove(keys[i])
+            self.assertEqual(len(self._box), repetitions - i - 1)
+
+    def test_set_item(self):
+        # Modify messages using __setitem__()
+        key0 = self._box.add(self._template % 'original 0')
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'original 0')
+        key1 = self._box.add(self._template % 'original 1')
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % 'original 1')
+        self._box[key0] = self._template % 'changed 0'
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'changed 0')
+        self._box[key1] = self._template % 'changed 1'
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % 'changed 1')
+        self._box[key0] = _sample_message
+        self._check_sample(self._box[key0])
+        self._box[key1] = self._box[key0]
+        self._check_sample(self._box[key1])
+        self._box[key0] = self._template % 'original 0'
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'original 0')
+        self._check_sample(self._box[key1])
+        self.assertRaises(KeyError,
+                          lambda: self._box.__setitem__('foo', 'bar'))
+        self.assertRaises(KeyError, lambda: self._box['foo'])
+        self.assertEqual(len(self._box), 2)
+
+    def test_clear(self, iterations=10):
+        # Remove all messages using clear()
+        keys = []
+        for i in xrange(iterations):
+            self._box.add(self._template % i)
+        for i, key in enumerate(keys):
+            self.assertEqual(self._box.get_string(key), self._template % i)
+        self._box.clear()
+        self.assertEqual(len(self._box), 0)
+        for i, key in enumerate(keys):
+            self.assertRaises(KeyError, lambda: self._box.get_string(key))
+
+    def test_pop(self):
+        # Get and remove a message using pop()
+        key0 = self._box.add(self._template % 0)
+        self.assertIn(key0, self._box)
+        key1 = self._box.add(self._template % 1)
+        self.assertIn(key1, self._box)
+        self.assertEqual(self._box.pop(key0).get_payload(), '0')
+        self.assertNotIn(key0, self._box)
+        self.assertIn(key1, self._box)
+        key2 = self._box.add(self._template % 2)
+        self.assertIn(key2, self._box)
+        self.assertEqual(self._box.pop(key2).get_payload(), '2')
+        self.assertNotIn(key2, self._box)
+        self.assertIn(key1, self._box)
+        self.assertEqual(self._box.pop(key1).get_payload(), '1')
+        self.assertNotIn(key1, self._box)
+        self.assertEqual(len(self._box), 0)
+
+    def test_popitem(self, iterations=10):
+        # Get and remove an arbitrary (key, message) using popitem()
+        keys = []
+        for i in xrange(10):
+            keys.append(self._box.add(self._template % i))
+        seen = []
+        for i in xrange(10):
+            key, msg = self._box.popitem()
+            self.assertIn(key, keys)
+            self.assertNotIn(key, seen)
+            seen.append(key)
+            self.assertEqual(int(msg.get_payload()), keys.index(key))
+        self.assertEqual(len(self._box), 0)
+        for key in keys:
+            self.assertRaises(KeyError, lambda: self._box[key])
+
+    def test_update(self):
+        # Modify multiple messages using update()
+        key0 = self._box.add(self._template % 'original 0')
+        key1 = self._box.add(self._template % 'original 1')
+        key2 = self._box.add(self._template % 'original 2')
+        self._box.update({key0: self._template % 'changed 0',
+                          key2: _sample_message})
+        self.assertEqual(len(self._box), 3)
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'changed 0')
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % 'original 1')
+        self._check_sample(self._box[key2])
+        self._box.update([(key2, self._template % 'changed 2'),
+                    (key1, self._template % 'changed 1'),
+                    (key0, self._template % 'original 0')])
+        self.assertEqual(len(self._box), 3)
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'original 0')
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % 'changed 1')
+        self.assertEqual(self._box.get_string(key2),
+                         self._template % 'changed 2')
+        self.assertRaises(KeyError,
+                          lambda: self._box.update({'foo': 'bar',
+                                          key0: self._template % "changed 0"}))
+        self.assertEqual(len(self._box), 3)
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % "changed 0")
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % "changed 1")
+        self.assertEqual(self._box.get_string(key2),
+                         self._template % "changed 2")
+
+    def test_flush(self):
+        # Write changes to disk
+        self._test_flush_or_close(self._box.flush, True)
+
+    def test_lock_unlock(self):
+        # Lock and unlock the mailbox
+        self.assertFalse(os.path.exists(self._get_lock_path()))
+        self._box.lock()
+        self.assertTrue(os.path.exists(self._get_lock_path()))
+        self._box.unlock()
+        self.assertFalse(os.path.exists(self._get_lock_path()))
+
+    def test_close(self):
+        # Close mailbox and flush changes to disk
+        self._test_flush_or_close(self._box.close, False)
+
+    def _test_flush_or_close(self, method, should_call_close):
+        contents = [self._template % i for i in xrange(3)]
+        self._box.add(contents[0])
+        self._box.add(contents[1])
+        self._box.add(contents[2])
+        method()
+        if should_call_close:
+            self._box.close()
+        self._box = self._factory(self._path)
+        keys = self._box.keys()
+        self.assertEqual(len(keys), 3)
+        for key in keys:
+            self.assertIn(self._box.get_string(key), contents)
+
+    def test_dump_message(self):
+        # Write message representations to disk
+        for input in (email.message_from_string(_sample_message),
+                      _sample_message, StringIO.StringIO(_sample_message)):
+            output = StringIO.StringIO()
+            self._box._dump_message(input, output)
+            self.assertEqual(output.getvalue(),
+                             _sample_message.replace('\n', os.linesep))
+        output = StringIO.StringIO()
+        self.assertRaises(TypeError,
+                          lambda: self._box._dump_message(None, output))
+
+    def _get_lock_path(self):
+        # Return the path of the dot lock file. May be overridden.
+        return self._path + '.lock'
+
+
+class TestMailboxSuperclass(TestBase):
+
+    def test_notimplemented(self):
+        # Test that all Mailbox methods raise NotImplementedException.
+        box = mailbox.Mailbox('path')
+        self.assertRaises(NotImplementedError, lambda: box.add(''))
+        self.assertRaises(NotImplementedError, lambda: box.remove(''))
+        self.assertRaises(NotImplementedError, lambda: box.__delitem__(''))
+        self.assertRaises(NotImplementedError, lambda: box.discard(''))
+        self.assertRaises(NotImplementedError, lambda: box.__setitem__('', ''))
+        self.assertRaises(NotImplementedError, lambda: box.iterkeys())
+        self.assertRaises(NotImplementedError, lambda: box.keys())
+        self.assertRaises(NotImplementedError, lambda: box.itervalues().next())
+        self.assertRaises(NotImplementedError, lambda: box.__iter__().next())
+        self.assertRaises(NotImplementedError, lambda: box.values())
+        self.assertRaises(NotImplementedError, lambda: box.iteritems().next())
+        self.assertRaises(NotImplementedError, lambda: box.items())
+        self.assertRaises(NotImplementedError, lambda: box.get(''))
+        self.assertRaises(NotImplementedError, lambda: box.__getitem__(''))
+        self.assertRaises(NotImplementedError, lambda: box.get_message(''))
+        self.assertRaises(NotImplementedError, lambda: box.get_string(''))
+        self.assertRaises(NotImplementedError, lambda: box.get_file(''))
+        self.assertRaises(NotImplementedError, lambda: box.has_key(''))
+        self.assertRaises(NotImplementedError, lambda: box.__contains__(''))
+        self.assertRaises(NotImplementedError, lambda: box.__len__())
+        self.assertRaises(NotImplementedError, lambda: box.clear())
+        self.assertRaises(NotImplementedError, lambda: box.pop(''))
+        self.assertRaises(NotImplementedError, lambda: box.popitem())
+        self.assertRaises(NotImplementedError, lambda: box.update((('', ''),)))
+        self.assertRaises(NotImplementedError, lambda: box.flush())
+        self.assertRaises(NotImplementedError, lambda: box.lock())
+        self.assertRaises(NotImplementedError, lambda: box.unlock())
+        self.assertRaises(NotImplementedError, lambda: box.close())
+
+
+class TestMaildir(TestMailbox):
+
+    _factory = lambda self, path, factory=None: mailbox.Maildir(path, factory)
+
+    def setUp(self):
+        TestMailbox.setUp(self)
+        if os.name in ('nt', 'os2') or sys.platform == 'cygwin':
+            self._box.colon = '!'
+
+    def test_add_MM(self):
+        # Add a MaildirMessage instance
+        msg = mailbox.MaildirMessage(self._template % 0)
+        msg.set_subdir('cur')
+        msg.set_info('foo')
+        key = self._box.add(msg)
+        self.assertTrue(os.path.exists(os.path.join(self._path, 'cur', '%s%sfoo' %
+                                                 (key, self._box.colon))))
+
+    def test_get_MM(self):
+        # Get a MaildirMessage instance
+        msg = mailbox.MaildirMessage(self._template % 0)
+        msg.set_subdir('cur')
+        msg.set_flags('RF')
+        key = self._box.add(msg)
+        msg_returned = self._box.get_message(key)
+        self.assertIsInstance(msg_returned, mailbox.MaildirMessage)
+        self.assertEqual(msg_returned.get_subdir(), 'cur')
+        self.assertEqual(msg_returned.get_flags(), 'FR')
+
+    def test_set_MM(self):
+        # Set with a MaildirMessage instance
+        msg0 = mailbox.MaildirMessage(self._template % 0)
+        msg0.set_flags('TP')
+        key = self._box.add(msg0)
+        msg_returned = self._box.get_message(key)
+        self.assertEqual(msg_returned.get_subdir(), 'new')
+        self.assertEqual(msg_returned.get_flags(), 'PT')
+        msg1 = mailbox.MaildirMessage(self._template % 1)
+        self._box[key] = msg1
+        msg_returned = self._box.get_message(key)
+        self.assertEqual(msg_returned.get_subdir(), 'new')
+        self.assertEqual(msg_returned.get_flags(), '')
+        self.assertEqual(msg_returned.get_payload(), '1')
+        msg2 = mailbox.MaildirMessage(self._template % 2)
+        msg2.set_info('2,S')
+        self._box[key] = msg2
+        self._box[key] = self._template % 3
+        msg_returned = self._box.get_message(key)
+        self.assertEqual(msg_returned.get_subdir(), 'new')
+        self.assertEqual(msg_returned.get_flags(), 'S')
+        self.assertEqual(msg_returned.get_payload(), '3')
+
+    def test_consistent_factory(self):
+        # Add a message.
+        msg = mailbox.MaildirMessage(self._template % 0)
+        msg.set_subdir('cur')
+        msg.set_flags('RF')
+        key = self._box.add(msg)
+
+        # Create new mailbox with
+        class FakeMessage(mailbox.MaildirMessage):
+            pass
+        box = mailbox.Maildir(self._path, factory=FakeMessage)
+        box.colon = self._box.colon
+        msg2 = box.get_message(key)
+        self.assertIsInstance(msg2, FakeMessage)
+
+    def test_initialize_new(self):
+        # Initialize a non-existent mailbox
+        self.tearDown()
+        self._box = mailbox.Maildir(self._path)
+        self._check_basics(factory=rfc822.Message)
+        self._delete_recursively(self._path)
+        self._box = self._factory(self._path, factory=None)
+        self._check_basics()
+
+    def test_initialize_existing(self):
+        # Initialize an existing mailbox
+        self.tearDown()
+        for subdir in '', 'tmp', 'new', 'cur':
+            os.mkdir(os.path.normpath(os.path.join(self._path, subdir)))
+        self._box = mailbox.Maildir(self._path)
+        self._check_basics(factory=rfc822.Message)
+        self._box = mailbox.Maildir(self._path, factory=None)
+        self._check_basics()
+
+    def _check_basics(self, factory=None):
+        # (Used by test_open_new() and test_open_existing().)
+        self.assertEqual(self._box._path, os.path.abspath(self._path))
+        self.assertEqual(self._box._factory, factory)
+        for subdir in '', 'tmp', 'new', 'cur':
+            path = os.path.join(self._path, subdir)
+            mode = os.stat(path)[stat.ST_MODE]
+            self.assertTrue(stat.S_ISDIR(mode), "Not a directory: '%s'" % path)
+
+    def test_list_folders(self):
+        # List folders
+        self._box.add_folder('one')
+        self._box.add_folder('two')
+        self._box.add_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 3)
+        self.assertEqual(set(self._box.list_folders()),
+                         set(('one', 'two', 'three')))
+
+    def test_get_folder(self):
+        # Open folders
+        self._box.add_folder('foo.bar')
+        folder0 = self._box.get_folder('foo.bar')
+        folder0.add(self._template % 'bar')
+        self.assertTrue(os.path.isdir(os.path.join(self._path, '.foo.bar')))
+        folder1 = self._box.get_folder('foo.bar')
+        self.assertEqual(folder1.get_string(folder1.keys()[0]),
+                         self._template % 'bar')
+
+    def test_add_and_remove_folders(self):
+        # Delete folders
+        self._box.add_folder('one')
+        self._box.add_folder('two')
+        self.assertEqual(len(self._box.list_folders()), 2)
+        self.assertEqual(set(self._box.list_folders()), set(('one', 'two')))
+        self._box.remove_folder('one')
+        self.assertEqual(len(self._box.list_folders()), 1)
+        self.assertEqual(set(self._box.list_folders()), set(('two',)))
+        self._box.add_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 2)
+        self.assertEqual(set(self._box.list_folders()), set(('two', 'three')))
+        self._box.remove_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 1)
+        self.assertEqual(set(self._box.list_folders()), set(('two',)))
+        self._box.remove_folder('two')
+        self.assertEqual(len(self._box.list_folders()), 0)
+        self.assertEqual(self._box.list_folders(), [])
+
+    def test_clean(self):
+        # Remove old files from 'tmp'
+        foo_path = os.path.join(self._path, 'tmp', 'foo')
+        bar_path = os.path.join(self._path, 'tmp', 'bar')
+        with open(foo_path, 'w') as f:
+            f.write("@")
+        with open(bar_path, 'w') as f:
+            f.write("@")
+        self._box.clean()
+        self.assertTrue(os.path.exists(foo_path))
+        self.assertTrue(os.path.exists(bar_path))
+        foo_stat = os.stat(foo_path)
+        os.utime(foo_path, (time.time() - 129600 - 2,
+                            foo_stat.st_mtime))
+        self._box.clean()
+        self.assertFalse(os.path.exists(foo_path))
+        self.assertTrue(os.path.exists(bar_path))
+
+    def test_create_tmp(self, repetitions=10):
+        # Create files in tmp directory
+        hostname = socket.gethostname()
+        if '/' in hostname:
+            hostname = hostname.replace('/', r'\057')
+        if ':' in hostname:
+            hostname = hostname.replace(':', r'\072')
+        pid = os.getpid()
+        pattern = re.compile(r"(?P<time>\d+)\.M(?P<M>\d{1,6})P(?P<P>\d+)"
+                             r"Q(?P<Q>\d+)\.(?P<host>[^:/]+)")
+        previous_groups = None
+        for x in xrange(repetitions):
+            tmp_file = self._box._create_tmp()
+            head, tail = os.path.split(tmp_file.name)
+            self.assertEqual(head, os.path.abspath(os.path.join(self._path,
+                                                                "tmp")),
+                             "File in wrong location: '%s'" % head)
+            match = pattern.match(tail)
+            self.assertTrue(match is not None, "Invalid file name: '%s'" % tail)
+            groups = match.groups()
+            if previous_groups is not None:
+                self.assertTrue(int(groups[0] >= previous_groups[0]),
+                             "Non-monotonic seconds: '%s' before '%s'" %
+                             (previous_groups[0], groups[0]))
+                self.assertTrue(int(groups[1] >= previous_groups[1]) or
+                             groups[0] != groups[1],
+                             "Non-monotonic milliseconds: '%s' before '%s'" %
+                             (previous_groups[1], groups[1]))
+                self.assertTrue(int(groups[2]) == pid,
+                             "Process ID mismatch: '%s' should be '%s'" %
+                             (groups[2], pid))
+                self.assertTrue(int(groups[3]) == int(previous_groups[3]) + 1,
+                             "Non-sequential counter: '%s' before '%s'" %
+                             (previous_groups[3], groups[3]))
+                self.assertTrue(groups[4] == hostname,
+                             "Host name mismatch: '%s' should be '%s'" %
+                             (groups[4], hostname))
+            previous_groups = groups
+            tmp_file.write(_sample_message)
+            tmp_file.seek(0)
+            self.assertTrue(tmp_file.read() == _sample_message)
+            tmp_file.close()
+        file_count = len(os.listdir(os.path.join(self._path, "tmp")))
+        self.assertTrue(file_count == repetitions,
+                     "Wrong file count: '%s' should be '%s'" %
+                     (file_count, repetitions))
+
+    def test_refresh(self):
+        # Update the table of contents
+        self.assertEqual(self._box._toc, {})
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(self._template % 1)
+        self.assertEqual(self._box._toc, {})
+        self._box._refresh()
+        self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+                                          key1: os.path.join('new', key1)})
+        key2 = self._box.add(self._template % 2)
+        self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+                                          key1: os.path.join('new', key1)})
+        self._box._refresh()
+        self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+                                          key1: os.path.join('new', key1),
+                                          key2: os.path.join('new', key2)})
+
+    def test_lookup(self):
+        # Look up message subpaths in the TOC
+        self.assertRaises(KeyError, lambda: self._box._lookup('foo'))
+        key0 = self._box.add(self._template % 0)
+        self.assertEqual(self._box._lookup(key0), os.path.join('new', key0))
+        os.remove(os.path.join(self._path, 'new', key0))
+        self.assertEqual(self._box._toc, {key0: os.path.join('new', key0)})
+        # Be sure that the TOC is read back from disk (see issue #6896
+        # about bad mtime behaviour on some systems).
+        self._box.flush()
+        self.assertRaises(KeyError, lambda: self._box._lookup(key0))
+        self.assertEqual(self._box._toc, {})
+
+    def test_lock_unlock(self):
+        # Lock and unlock the mailbox. For Maildir, this does nothing.
+        self._box.lock()
+        self._box.unlock()
+
+    def test_folder (self):
+        # Test for bug #1569790: verify that folders returned by .get_folder()
+        # use the same factory function.
+        def dummy_factory (s):
+            return None
+        box = self._factory(self._path, factory=dummy_factory)
+        folder = box.add_folder('folder1')
+        self.assertIs(folder._factory, dummy_factory)
+
+        folder1_alias = box.get_folder('folder1')
+        self.assertIs(folder1_alias._factory, dummy_factory)
+
+    def test_directory_in_folder (self):
+        # Test that mailboxes still work if there's a stray extra directory
+        # in a folder.
+        for i in range(10):
+            self._box.add(mailbox.Message(_sample_message))
+
+        # Create a stray directory
+        os.mkdir(os.path.join(self._path, 'cur', 'stray-dir'))
+
+        # Check that looping still works with the directory present.
+        for msg in self._box:
+            pass
+
+    def test_file_permissions(self):
+        # Verify that message files are created without execute permissions
+        if not hasattr(os, "stat") or not hasattr(os, "umask"):
+            return
+        msg = mailbox.MaildirMessage(self._template % 0)
+        orig_umask = os.umask(0)
+        try:
+            key = self._box.add(msg)
+        finally:
+            os.umask(orig_umask)
+        path = os.path.join(self._path, self._box._lookup(key))
+        mode = os.stat(path).st_mode
+        self.assertEqual(mode & 0111, 0)
+
+    def test_folder_file_perms(self):
+        # From bug #3228, we want to verify that the file created inside a Maildir
+        # subfolder isn't marked as executable.
+        if not hasattr(os, "stat") or not hasattr(os, "umask"):
+            return
+
+        orig_umask = os.umask(0)
+        try:
+            subfolder = self._box.add_folder('subfolder')
+        finally:
+            os.umask(orig_umask)
+
+        path = os.path.join(subfolder._path, 'maildirfolder')
+        st = os.stat(path)
+        perms = st.st_mode
+        self.assertFalse((perms & 0111)) # Execute bits should all be off.
+
+    def test_reread(self):
+
+        # Put the last modified times more than two seconds into the past
+        # (because mtime may have only a two second granularity).
+        for subdir in ('cur', 'new'):
+            os.utime(os.path.join(self._box._path, subdir),
+                     (time.time()-5,)*2)
+
+        # Because mtime has a two second granularity in worst case (FAT), a
+        # refresh is done unconditionally if called for within
+        # two-second-plus-a-bit of the last one, just in case the mbox has
+        # changed; so now we have to wait for that interval to expire.
+        time.sleep(2.01 + self._box._skewfactor)
+
+        # Re-reading causes the ._toc attribute to be assigned a new dictionary
+        # object, so we'll check that the ._toc attribute isn't a different
+        # object.
+        orig_toc = self._box._toc
+        def refreshed():
+            return self._box._toc is not orig_toc
+
+        self._box._refresh()
+        self.assertFalse(refreshed())
+
+        # Now, write something into cur and remove it.  This changes
+        # the mtime and should cause a re-read.
+        filename = os.path.join(self._path, 'cur', 'stray-file')
+        f = open(filename, 'w')
+        f.close()
+        os.unlink(filename)
+        self._box._refresh()
+        self.assertTrue(refreshed())
+
+class _TestMboxMMDF(TestMailbox):
+
+    def tearDown(self):
+        self._box.close()
+        self._delete_recursively(self._path)
+        for lock_remnant in glob.glob(self._path + '.*'):
+            test_support.unlink(lock_remnant)
+
+    def test_add_from_string(self):
+        # Add a string starting with 'From ' to the mailbox
+        key = self._box.add('From foo at bar blah\nFrom: foo\n\n0')
+        self.assertEqual(self._box[key].get_from(), 'foo at bar blah')
+        self.assertEqual(self._box[key].get_payload(), '0')
+
+    def test_add_mbox_or_mmdf_message(self):
+        # Add an mboxMessage or MMDFMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg = class_('From foo at bar blah\nFrom: foo\n\n0')
+            key = self._box.add(msg)
+
+    def test_open_close_open(self):
+        # Open and inspect previously-created mailbox
+        values = [self._template % i for i in xrange(3)]
+        for value in values:
+            self._box.add(value)
+        self._box.close()
+        mtime = os.path.getmtime(self._path)
+        self._box = self._factory(self._path)
+        self.assertEqual(len(self._box), 3)
+        for key in self._box.iterkeys():
+            self.assertIn(self._box.get_string(key), values)
+        self._box.close()
+        self.assertEqual(mtime, os.path.getmtime(self._path))
+
+    def test_add_and_close(self):
+        # Verifying that closing a mailbox doesn't change added items
+        self._box.add(_sample_message)
+        for i in xrange(3):
+            self._box.add(self._template % i)
+        self._box.add(_sample_message)
+        self._box._file.flush()
+        self._box._file.seek(0)
+        contents = self._box._file.read()
+        self._box.close()
+        with open(self._path, 'rb') as f:
+            self.assertEqual(contents, f.read())
+        self._box = self._factory(self._path)
+
+    def test_lock_conflict(self):
+        # Fork off a subprocess that will lock the file for 2 seconds,
+        # unlock it, and then exit.
+        if not hasattr(os, 'fork'):
+            return
+        pid = os.fork()
+        if pid == 0:
+            # In the child, lock the mailbox.
+            self._box.lock()
+            time.sleep(2)
+            self._box.unlock()
+            os._exit(0)
+
+        # In the parent, sleep a bit to give the child time to acquire
+        # the lock.
+        time.sleep(0.5)
+        try:
+            self.assertRaises(mailbox.ExternalClashError,
+                              self._box.lock)
+        finally:
+            # Wait for child to exit.  Locking should now succeed.
+            exited_pid, status = os.waitpid(pid, 0)
+
+        self._box.lock()
+        self._box.unlock()
+
+    def test_relock(self):
+        # Test case for bug #1575506: the mailbox class was locking the
+        # wrong file object in its flush() method.
+        msg = "Subject: sub\n\nbody\n"
+        key1 = self._box.add(msg)
+        self._box.flush()
+        self._box.close()
+
+        self._box = self._factory(self._path)
+        self._box.lock()
+        key2 = self._box.add(msg)
+        self._box.flush()
+        self.assertTrue(self._box._locked)
+        self._box.close()
+
+
+class TestMbox(_TestMboxMMDF):
+
+    _factory = lambda self, path, factory=None: mailbox.mbox(path, factory)
+
+    def test_file_perms(self):
+        # From bug #3228, we want to verify that the mailbox file isn't executable,
+        # even if the umask is set to something that would leave executable bits set.
+        # We only run this test on platforms that support umask.
+        if hasattr(os, 'umask') and hasattr(os, 'stat'):
+            try:
+                old_umask = os.umask(0077)
+                self._box.close()
+                os.unlink(self._path)
+                self._box = mailbox.mbox(self._path, create=True)
+                self._box.add('')
+                self._box.close()
+            finally:
+                os.umask(old_umask)
+
+            st = os.stat(self._path)
+            perms = st.st_mode
+            self.assertFalse((perms & 0111)) # Execute bits should all be off.
+
+class TestMMDF(_TestMboxMMDF):
+
+    _factory = lambda self, path, factory=None: mailbox.MMDF(path, factory)
+
+
+class TestMH(TestMailbox):
+
+    _factory = lambda self, path, factory=None: mailbox.MH(path, factory)
+
+    def test_list_folders(self):
+        # List folders
+        self._box.add_folder('one')
+        self._box.add_folder('two')
+        self._box.add_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 3)
+        self.assertEqual(set(self._box.list_folders()),
+                         set(('one', 'two', 'three')))
+
+    def test_get_folder(self):
+        # Open folders
+        def dummy_factory (s):
+            return None
+        self._box = self._factory(self._path, dummy_factory)
+
+        new_folder = self._box.add_folder('foo.bar')
+        folder0 = self._box.get_folder('foo.bar')
+        folder0.add(self._template % 'bar')
+        self.assertTrue(os.path.isdir(os.path.join(self._path, 'foo.bar')))
+        folder1 = self._box.get_folder('foo.bar')
+        self.assertEqual(folder1.get_string(folder1.keys()[0]),
+                         self._template % 'bar')
+
+        # Test for bug #1569790: verify that folders returned by .get_folder()
+        # use the same factory function.
+        self.assertIs(new_folder._factory, self._box._factory)
+        self.assertIs(folder0._factory, self._box._factory)
+
+    def test_add_and_remove_folders(self):
+        # Delete folders
+        self._box.add_folder('one')
+        self._box.add_folder('two')
+        self.assertEqual(len(self._box.list_folders()), 2)
+        self.assertEqual(set(self._box.list_folders()), set(('one', 'two')))
+        self._box.remove_folder('one')
+        self.assertEqual(len(self._box.list_folders()), 1)
+        self.assertEqual(set(self._box.list_folders()), set(('two', )))
+        self._box.add_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 2)
+        self.assertEqual(set(self._box.list_folders()), set(('two', 'three')))
+        self._box.remove_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 1)
+        self.assertEqual(set(self._box.list_folders()), set(('two', )))
+        self._box.remove_folder('two')
+        self.assertEqual(len(self._box.list_folders()), 0)
+        self.assertEqual(self._box.list_folders(), [])
+
+    def test_sequences(self):
+        # Get and set sequences
+        self.assertEqual(self._box.get_sequences(), {})
+        msg0 = mailbox.MHMessage(self._template % 0)
+        msg0.add_sequence('foo')
+        key0 = self._box.add(msg0)
+        self.assertEqual(self._box.get_sequences(), {'foo':[key0]})
+        msg1 = mailbox.MHMessage(self._template % 1)
+        msg1.set_sequences(['bar', 'replied', 'foo'])
+        key1 = self._box.add(msg1)
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[key0, key1], 'bar':[key1], 'replied':[key1]})
+        msg0.set_sequences(['flagged'])
+        self._box[key0] = msg0
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[key1], 'bar':[key1], 'replied':[key1],
+                          'flagged':[key0]})
+        self._box.remove(key1)
+        self.assertEqual(self._box.get_sequences(), {'flagged':[key0]})
+
+    def test_issue2625(self):
+        msg0 = mailbox.MHMessage(self._template % 0)
+        msg0.add_sequence('foo')
+        key0 = self._box.add(msg0)
+        refmsg0 = self._box.get_message(key0)
+
+    def test_issue7627(self):
+        msg0 = mailbox.MHMessage(self._template % 0)
+        key0 = self._box.add(msg0)
+        self._box.lock()
+        self._box.remove(key0)
+        self._box.unlock()
+
+    def test_pack(self):
+        # Pack the contents of the mailbox
+        msg0 = mailbox.MHMessage(self._template % 0)
+        msg1 = mailbox.MHMessage(self._template % 1)
+        msg2 = mailbox.MHMessage(self._template % 2)
+        msg3 = mailbox.MHMessage(self._template % 3)
+        msg0.set_sequences(['foo', 'unseen'])
+        msg1.set_sequences(['foo'])
+        msg2.set_sequences(['foo', 'flagged'])
+        msg3.set_sequences(['foo', 'bar', 'replied'])
+        key0 = self._box.add(msg0)
+        key1 = self._box.add(msg1)
+        key2 = self._box.add(msg2)
+        key3 = self._box.add(msg3)
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[key0,key1,key2,key3], 'unseen':[key0],
+                          'flagged':[key2], 'bar':[key3], 'replied':[key3]})
+        self._box.remove(key2)
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[key0,key1,key3], 'unseen':[key0], 'bar':[key3],
+                          'replied':[key3]})
+        self._box.pack()
+        self.assertEqual(self._box.keys(), [1, 2, 3])
+        key0 = key0
+        key1 = key0 + 1
+        key2 = key1 + 1
+        self.assertEqual(self._box.get_sequences(),
+                     {'foo':[1, 2, 3], 'unseen':[1], 'bar':[3], 'replied':[3]})
+
+        # Test case for packing while holding the mailbox locked.
+        key0 = self._box.add(msg1)
+        key1 = self._box.add(msg1)
+        key2 = self._box.add(msg1)
+        key3 = self._box.add(msg1)
+
+        self._box.remove(key0)
+        self._box.remove(key2)
+        self._box.lock()
+        self._box.pack()
+        self._box.unlock()
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[1, 2, 3, 4, 5],
+                          'unseen':[1], 'bar':[3], 'replied':[3]})
+
+    def _get_lock_path(self):
+        return os.path.join(self._path, '.mh_sequences.lock')
+
+
+class TestBabyl(TestMailbox):
+
+    _factory = lambda self, path, factory=None: mailbox.Babyl(path, factory)
+
+    def tearDown(self):
+        self._box.close()
+        self._delete_recursively(self._path)
+        for lock_remnant in glob.glob(self._path + '.*'):
+            test_support.unlink(lock_remnant)
+
+    def test_labels(self):
+        # Get labels from the mailbox
+        self.assertEqual(self._box.get_labels(), [])
+        msg0 = mailbox.BabylMessage(self._template % 0)
+        msg0.add_label('foo')
+        key0 = self._box.add(msg0)
+        self.assertEqual(self._box.get_labels(), ['foo'])
+        msg1 = mailbox.BabylMessage(self._template % 1)
+        msg1.set_labels(['bar', 'answered', 'foo'])
+        key1 = self._box.add(msg1)
+        self.assertEqual(set(self._box.get_labels()), set(['foo', 'bar']))
+        msg0.set_labels(['blah', 'filed'])
+        self._box[key0] = msg0
+        self.assertEqual(set(self._box.get_labels()),
+                         set(['foo', 'bar', 'blah']))
+        self._box.remove(key1)
+        self.assertEqual(set(self._box.get_labels()), set(['blah']))
+
+
+class TestMessage(TestBase):
+
+    _factory = mailbox.Message      # Overridden by subclasses to reuse tests
+
+    def setUp(self):
+        self._path = test_support.TESTFN
+
+    def tearDown(self):
+        self._delete_recursively(self._path)
+
+    def test_initialize_with_eMM(self):
+        # Initialize based on email.message.Message instance
+        eMM = email.message_from_string(_sample_message)
+        msg = self._factory(eMM)
+        self._post_initialize_hook(msg)
+        self._check_sample(msg)
+
+    def test_initialize_with_string(self):
+        # Initialize based on string
+        msg = self._factory(_sample_message)
+        self._post_initialize_hook(msg)
+        self._check_sample(msg)
+
+    def test_initialize_with_file(self):
+        # Initialize based on contents of file
+        with open(self._path, 'w+') as f:
+            f.write(_sample_message)
+            f.seek(0)
+            msg = self._factory(f)
+            self._post_initialize_hook(msg)
+            self._check_sample(msg)
+
+    def test_initialize_with_nothing(self):
+        # Initialize without arguments
+        msg = self._factory()
+        self._post_initialize_hook(msg)
+        self.assertIsInstance(msg, email.message.Message)
+        self.assertIsInstance(msg, mailbox.Message)
+        self.assertIsInstance(msg, self._factory)
+        self.assertEqual(msg.keys(), [])
+        self.assertFalse(msg.is_multipart())
+        self.assertEqual(msg.get_payload(), None)
+
+    def test_initialize_incorrectly(self):
+        # Initialize with invalid argument
+        self.assertRaises(TypeError, lambda: self._factory(object()))
+
+    def test_become_message(self):
+        # Take on the state of another message
+        eMM = email.message_from_string(_sample_message)
+        msg = self._factory()
+        msg._become_message(eMM)
+        self._check_sample(msg)
+
+    def test_explain_to(self):
+        # Copy self's format-specific data to other message formats.
+        # This test is superficial; better ones are in TestMessageConversion.
+        msg = self._factory()
+        for class_ in (mailbox.Message, mailbox.MaildirMessage,
+                       mailbox.mboxMessage, mailbox.MHMessage,
+                       mailbox.BabylMessage, mailbox.MMDFMessage):
+            other_msg = class_()
+            msg._explain_to(other_msg)
+        other_msg = email.message.Message()
+        self.assertRaises(TypeError, lambda: msg._explain_to(other_msg))
+
+    def _post_initialize_hook(self, msg):
+        # Overridden by subclasses to check extra things after initialization
+        pass
+
+
+class TestMaildirMessage(TestMessage):
+
+    _factory = mailbox.MaildirMessage
+
+    def _post_initialize_hook(self, msg):
+        self.assertEqual(msg._subdir, 'new')
+        self.assertEqual(msg._info,'')
+
+    def test_subdir(self):
+        # Use get_subdir() and set_subdir()
+        msg = mailbox.MaildirMessage(_sample_message)
+        self.assertEqual(msg.get_subdir(), 'new')
+        msg.set_subdir('cur')
+        self.assertEqual(msg.get_subdir(), 'cur')
+        msg.set_subdir('new')
+        self.assertEqual(msg.get_subdir(), 'new')
+        self.assertRaises(ValueError, lambda: msg.set_subdir('tmp'))
+        self.assertEqual(msg.get_subdir(), 'new')
+        msg.set_subdir('new')
+        self.assertEqual(msg.get_subdir(), 'new')
+        self._check_sample(msg)
+
+    def test_flags(self):
+        # Use get_flags(), set_flags(), add_flag(), remove_flag()
+        msg = mailbox.MaildirMessage(_sample_message)
+        self.assertEqual(msg.get_flags(), '')
+        self.assertEqual(msg.get_subdir(), 'new')
+        msg.set_flags('F')
+        self.assertEqual(msg.get_subdir(), 'new')
+        self.assertEqual(msg.get_flags(), 'F')
+        msg.set_flags('SDTP')
+        self.assertEqual(msg.get_flags(), 'DPST')
+        msg.add_flag('FT')
+        self.assertEqual(msg.get_flags(), 'DFPST')
+        msg.remove_flag('TDRP')
+        self.assertEqual(msg.get_flags(), 'FS')
+        self.assertEqual(msg.get_subdir(), 'new')
+        self._check_sample(msg)
+
+    def test_date(self):
+        # Use get_date() and set_date()
+        msg = mailbox.MaildirMessage(_sample_message)
+        diff = msg.get_date() - time.time()
+        self.assertTrue(abs(diff) < 60, diff)
+        msg.set_date(0.0)
+        self.assertEqual(msg.get_date(), 0.0)
+
+    def test_info(self):
+        # Use get_info() and set_info()
+        msg = mailbox.MaildirMessage(_sample_message)
+        self.assertEqual(msg.get_info(), '')
+        msg.set_info('1,foo=bar')
+        self.assertEqual(msg.get_info(), '1,foo=bar')
+        self.assertRaises(TypeError, lambda: msg.set_info(None))
+        self._check_sample(msg)
+
+    def test_info_and_flags(self):
+        # Test interaction of info and flag methods
+        msg = mailbox.MaildirMessage(_sample_message)
+        self.assertEqual(msg.get_info(), '')
+        msg.set_flags('SF')
+        self.assertEqual(msg.get_flags(), 'FS')
+        self.assertEqual(msg.get_info(), '2,FS')
+        msg.set_info('1,')
+        self.assertEqual(msg.get_flags(), '')
+        self.assertEqual(msg.get_info(), '1,')
+        msg.remove_flag('RPT')
+        self.assertEqual(msg.get_flags(), '')
+        self.assertEqual(msg.get_info(), '1,')
+        msg.add_flag('D')
+        self.assertEqual(msg.get_flags(), 'D')
+        self.assertEqual(msg.get_info(), '2,D')
+        self._check_sample(msg)
+
+
+class _TestMboxMMDFMessage(TestMessage):
+
+    _factory = mailbox._mboxMMDFMessage
+
+    def _post_initialize_hook(self, msg):
+        self._check_from(msg)
+
+    def test_initialize_with_unixfrom(self):
+        # Initialize with a message that already has a _unixfrom attribute
+        msg = mailbox.Message(_sample_message)
+        msg.set_unixfrom('From foo at bar blah')
+        msg = mailbox.mboxMessage(msg)
+        self.assertEqual(msg.get_from(), 'foo at bar blah')
+
+    def test_from(self):
+        # Get and set "From " line
+        msg = mailbox.mboxMessage(_sample_message)
+        self._check_from(msg)
+        msg.set_from('foo bar')
+        self.assertEqual(msg.get_from(), 'foo bar')
+        msg.set_from('foo at bar', True)
+        self._check_from(msg, 'foo at bar')
+        msg.set_from('blah at temp', time.localtime())
+        self._check_from(msg, 'blah at temp')
+
+    def test_flags(self):
+        # Use get_flags(), set_flags(), add_flag(), remove_flag()
+        msg = mailbox.mboxMessage(_sample_message)
+        self.assertEqual(msg.get_flags(), '')
+        msg.set_flags('F')
+        self.assertEqual(msg.get_flags(), 'F')
+        msg.set_flags('XODR')
+        self.assertEqual(msg.get_flags(), 'RODX')
+        msg.add_flag('FA')
+        self.assertEqual(msg.get_flags(), 'RODFAX')
+        msg.remove_flag('FDXA')
+        self.assertEqual(msg.get_flags(), 'RO')
+        self._check_sample(msg)
+
+    def _check_from(self, msg, sender=None):
+        # Check contents of "From " line
+        if sender is None:
+            sender = "MAILER-DAEMON"
+        self.assertTrue(re.match(sender + r" \w{3} \w{3} [\d ]\d [\d ]\d:\d{2}:"
+                                 r"\d{2} \d{4}", msg.get_from()))
+
+
+class TestMboxMessage(_TestMboxMMDFMessage):
+
+    _factory = mailbox.mboxMessage
+
+
+class TestMHMessage(TestMessage):
+
+    _factory = mailbox.MHMessage
+
+    def _post_initialize_hook(self, msg):
+        self.assertEqual(msg._sequences, [])
+
+    def test_sequences(self):
+        # Get, set, join, and leave sequences
+        msg = mailbox.MHMessage(_sample_message)
+        self.assertEqual(msg.get_sequences(), [])
+        msg.set_sequences(['foobar'])
+        self.assertEqual(msg.get_sequences(), ['foobar'])
+        msg.set_sequences([])
+        self.assertEqual(msg.get_sequences(), [])
+        msg.add_sequence('unseen')
+        self.assertEqual(msg.get_sequences(), ['unseen'])
+        msg.add_sequence('flagged')
+        self.assertEqual(msg.get_sequences(), ['unseen', 'flagged'])
+        msg.add_sequence('flagged')
+        self.assertEqual(msg.get_sequences(), ['unseen', 'flagged'])
+        msg.remove_sequence('unseen')
+        self.assertEqual(msg.get_sequences(), ['flagged'])
+        msg.add_sequence('foobar')
+        self.assertEqual(msg.get_sequences(), ['flagged', 'foobar'])
+        msg.remove_sequence('replied')
+        self.assertEqual(msg.get_sequences(), ['flagged', 'foobar'])
+        msg.set_sequences(['foobar', 'replied'])
+        self.assertEqual(msg.get_sequences(), ['foobar', 'replied'])
+
+
+class TestBabylMessage(TestMessage):
+
+    _factory = mailbox.BabylMessage
+
+    def _post_initialize_hook(self, msg):
+        self.assertEqual(msg._labels, [])
+
+    def test_labels(self):
+        # Get, set, join, and leave labels
+        msg = mailbox.BabylMessage(_sample_message)
+        self.assertEqual(msg.get_labels(), [])
+        msg.set_labels(['foobar'])
+        self.assertEqual(msg.get_labels(), ['foobar'])
+        msg.set_labels([])
+        self.assertEqual(msg.get_labels(), [])
+        msg.add_label('filed')
+        self.assertEqual(msg.get_labels(), ['filed'])
+        msg.add_label('resent')
+        self.assertEqual(msg.get_labels(), ['filed', 'resent'])
+        msg.add_label('resent')
+        self.assertEqual(msg.get_labels(), ['filed', 'resent'])
+        msg.remove_label('filed')
+        self.assertEqual(msg.get_labels(), ['resent'])
+        msg.add_label('foobar')
+        self.assertEqual(msg.get_labels(), ['resent', 'foobar'])
+        msg.remove_label('unseen')
+        self.assertEqual(msg.get_labels(), ['resent', 'foobar'])
+        msg.set_labels(['foobar', 'answered'])
+        self.assertEqual(msg.get_labels(), ['foobar', 'answered'])
+
+    def test_visible(self):
+        # Get, set, and update visible headers
+        msg = mailbox.BabylMessage(_sample_message)
+        visible = msg.get_visible()
+        self.assertEqual(visible.keys(), [])
+        self.assertIs(visible.get_payload(), None)
+        visible['User-Agent'] = 'FooBar 1.0'
+        visible['X-Whatever'] = 'Blah'
+        self.assertEqual(msg.get_visible().keys(), [])
+        msg.set_visible(visible)
+        visible = msg.get_visible()
+        self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever'])
+        self.assertEqual(visible['User-Agent'], 'FooBar 1.0')
+        self.assertEqual(visible['X-Whatever'], 'Blah')
+        self.assertIs(visible.get_payload(), None)
+        msg.update_visible()
+        self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever'])
+        self.assertIs(visible.get_payload(), None)
+        visible = msg.get_visible()
+        self.assertEqual(visible.keys(), ['User-Agent', 'Date', 'From', 'To',
+                                          'Subject'])
+        for header in ('User-Agent', 'Date', 'From', 'To', 'Subject'):
+            self.assertEqual(visible[header], msg[header])
+
+
+class TestMMDFMessage(_TestMboxMMDFMessage):
+
+    _factory = mailbox.MMDFMessage
+
+
+class TestMessageConversion(TestBase):
+
+    def test_plain_to_x(self):
+        # Convert Message to all formats
+        for class_ in (mailbox.Message, mailbox.MaildirMessage,
+                       mailbox.mboxMessage, mailbox.MHMessage,
+                       mailbox.BabylMessage, mailbox.MMDFMessage):
+            msg_plain = mailbox.Message(_sample_message)
+            msg = class_(msg_plain)
+            self._check_sample(msg)
+
+    def test_x_to_plain(self):
+        # Convert all formats to Message
+        for class_ in (mailbox.Message, mailbox.MaildirMessage,
+                       mailbox.mboxMessage, mailbox.MHMessage,
+                       mailbox.BabylMessage, mailbox.MMDFMessage):
+            msg = class_(_sample_message)
+            msg_plain = mailbox.Message(msg)
+            self._check_sample(msg_plain)
+
+    def test_x_to_invalid(self):
+        # Convert all formats to an invalid format
+        for class_ in (mailbox.Message, mailbox.MaildirMessage,
+                       mailbox.mboxMessage, mailbox.MHMessage,
+                       mailbox.BabylMessage, mailbox.MMDFMessage):
+            self.assertRaises(TypeError, lambda: class_(False))
+
+    def test_maildir_to_maildir(self):
+        # Convert MaildirMessage to MaildirMessage
+        msg_maildir = mailbox.MaildirMessage(_sample_message)
+        msg_maildir.set_flags('DFPRST')
+        msg_maildir.set_subdir('cur')
+        date = msg_maildir.get_date()
+        msg = mailbox.MaildirMessage(msg_maildir)
+        self._check_sample(msg)
+        self.assertEqual(msg.get_flags(), 'DFPRST')
+        self.assertEqual(msg.get_subdir(), 'cur')
+        self.assertEqual(msg.get_date(), date)
+
+    def test_maildir_to_mboxmmdf(self):
+        # Convert MaildirMessage to mboxmessage and MMDFMessage
+        pairs = (('D', ''), ('F', 'F'), ('P', ''), ('R', 'A'), ('S', 'R'),
+                 ('T', 'D'), ('DFPRST', 'RDFA'))
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg_maildir = mailbox.MaildirMessage(_sample_message)
+            msg_maildir.set_date(0.0)
+            for setting, result in pairs:
+                msg_maildir.set_flags(setting)
+                msg = class_(msg_maildir)
+                self.assertEqual(msg.get_flags(), result)
+                self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' %
+                                 time.asctime(time.gmtime(0.0)))
+            msg_maildir.set_subdir('cur')
+            self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA')
+
+    def test_maildir_to_mh(self):
+        # Convert MaildirMessage to MHMessage
+        msg_maildir = mailbox.MaildirMessage(_sample_message)
+        pairs = (('D', ['unseen']), ('F', ['unseen', 'flagged']),
+                 ('P', ['unseen']), ('R', ['unseen', 'replied']), ('S', []),
+                 ('T', ['unseen']), ('DFPRST', ['replied', 'flagged']))
+        for setting, result in pairs:
+            msg_maildir.set_flags(setting)
+            self.assertEqual(mailbox.MHMessage(msg_maildir).get_sequences(),
+                             result)
+
+    def test_maildir_to_babyl(self):
+        # Convert MaildirMessage to Babyl
+        msg_maildir = mailbox.MaildirMessage(_sample_message)
+        pairs = (('D', ['unseen']), ('F', ['unseen']),
+                 ('P', ['unseen', 'forwarded']), ('R', ['unseen', 'answered']),
+                 ('S', []), ('T', ['unseen', 'deleted']),
+                 ('DFPRST', ['deleted', 'answered', 'forwarded']))
+        for setting, result in pairs:
+            msg_maildir.set_flags(setting)
+            self.assertEqual(mailbox.BabylMessage(msg_maildir).get_labels(),
+                             result)
+
+    def test_mboxmmdf_to_maildir(self):
+        # Convert mboxMessage and MMDFMessage to MaildirMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg_mboxMMDF = class_(_sample_message)
+            msg_mboxMMDF.set_from('foo at bar', time.gmtime(0.0))
+            pairs = (('R', 'S'), ('O', ''), ('D', 'T'), ('F', 'F'), ('A', 'R'),
+                     ('RODFA', 'FRST'))
+            for setting, result in pairs:
+                msg_mboxMMDF.set_flags(setting)
+                msg = mailbox.MaildirMessage(msg_mboxMMDF)
+                self.assertEqual(msg.get_flags(), result)
+                self.assertEqual(msg.get_date(), 0.0)
+            msg_mboxMMDF.set_flags('O')
+            self.assertEqual(mailbox.MaildirMessage(msg_mboxMMDF).get_subdir(),
+                             'cur')
+
+    def test_mboxmmdf_to_mboxmmdf(self):
+        # Convert mboxMessage and MMDFMessage to mboxMessage and MMDFMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg_mboxMMDF = class_(_sample_message)
+            msg_mboxMMDF.set_flags('RODFA')
+            msg_mboxMMDF.set_from('foo at bar')
+            for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+                msg2 = class2_(msg_mboxMMDF)
+                self.assertEqual(msg2.get_flags(), 'RODFA')
+                self.assertEqual(msg2.get_from(), 'foo at bar')
+
+    def test_mboxmmdf_to_mh(self):
+        # Convert mboxMessage and MMDFMessage to MHMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg_mboxMMDF = class_(_sample_message)
+            pairs = (('R', []), ('O', ['unseen']), ('D', ['unseen']),
+                     ('F', ['unseen', 'flagged']),
+                     ('A', ['unseen', 'replied']),
+                     ('RODFA', ['replied', 'flagged']))
+            for setting, result in pairs:
+                msg_mboxMMDF.set_flags(setting)
+                self.assertEqual(mailbox.MHMessage(msg_mboxMMDF).get_sequences(),
+                                 result)
+
+    def test_mboxmmdf_to_babyl(self):
+        # Convert mboxMessage and MMDFMessage to BabylMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg = class_(_sample_message)
+            pairs = (('R', []), ('O', ['unseen']),
+                     ('D', ['unseen', 'deleted']), ('F', ['unseen']),
+                     ('A', ['unseen', 'answered']),
+                     ('RODFA', ['deleted', 'answered']))
+            for setting, result in pairs:
+                msg.set_flags(setting)
+                self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result)
+
+    def test_mh_to_maildir(self):
+        # Convert MHMessage to MaildirMessage
+        pairs = (('unseen', ''), ('replied', 'RS'), ('flagged', 'FS'))
+        for setting, result in pairs:
+            msg = mailbox.MHMessage(_sample_message)
+            msg.add_sequence(setting)
+            self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result)
+            self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+        msg = mailbox.MHMessage(_sample_message)
+        msg.add_sequence('unseen')
+        msg.add_sequence('replied')
+        msg.add_sequence('flagged')
+        self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'FR')
+        self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+
+    def test_mh_to_mboxmmdf(self):
+        # Convert MHMessage to mboxMessage and MMDFMessage
+        pairs = (('unseen', 'O'), ('replied', 'ROA'), ('flagged', 'ROF'))
+        for setting, result in pairs:
+            msg = mailbox.MHMessage(_sample_message)
+            msg.add_sequence(setting)
+            for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+                self.assertEqual(class_(msg).get_flags(), result)
+        msg = mailbox.MHMessage(_sample_message)
+        msg.add_sequence('unseen')
+        msg.add_sequence('replied')
+        msg.add_sequence('flagged')
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            self.assertEqual(class_(msg).get_flags(), 'OFA')
+
+    def test_mh_to_mh(self):
+        # Convert MHMessage to MHMessage
+        msg = mailbox.MHMessage(_sample_message)
+        msg.add_sequence('unseen')
+        msg.add_sequence('replied')
+        msg.add_sequence('flagged')
+        self.assertEqual(mailbox.MHMessage(msg).get_sequences(),
+                         ['unseen', 'replied', 'flagged'])
+
+    def test_mh_to_babyl(self):
+        # Convert MHMessage to BabylMessage
+        pairs = (('unseen', ['unseen']), ('replied', ['answered']),
+                 ('flagged', []))
+        for setting, result in pairs:
+            msg = mailbox.MHMessage(_sample_message)
+            msg.add_sequence(setting)
+            self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result)
+        msg = mailbox.MHMessage(_sample_message)
+        msg.add_sequence('unseen')
+        msg.add_sequence('replied')
+        msg.add_sequence('flagged')
+        self.assertEqual(mailbox.BabylMessage(msg).get_labels(),
+                         ['unseen', 'answered'])
+
+    def test_babyl_to_maildir(self):
+        # Convert BabylMessage to MaildirMessage
+        pairs = (('unseen', ''), ('deleted', 'ST'), ('filed', 'S'),
+                 ('answered', 'RS'), ('forwarded', 'PS'), ('edited', 'S'),
+                 ('resent', 'PS'))
+        for setting, result in pairs:
+            msg = mailbox.BabylMessage(_sample_message)
+            msg.add_label(setting)
+            self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result)
+            self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+        msg = mailbox.BabylMessage(_sample_message)
+        for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+                      'edited', 'resent'):
+            msg.add_label(label)
+        self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'PRT')
+        self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+
+    def test_babyl_to_mboxmmdf(self):
+        # Convert BabylMessage to mboxMessage and MMDFMessage
+        pairs = (('unseen', 'O'), ('deleted', 'ROD'), ('filed', 'RO'),
+                 ('answered', 'ROA'), ('forwarded', 'RO'), ('edited', 'RO'),
+                 ('resent', 'RO'))
+        for setting, result in pairs:
+            for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+                msg = mailbox.BabylMessage(_sample_message)
+                msg.add_label(setting)
+                self.assertEqual(class_(msg).get_flags(), result)
+        msg = mailbox.BabylMessage(_sample_message)
+        for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+                      'edited', 'resent'):
+            msg.add_label(label)
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            self.assertEqual(class_(msg).get_flags(), 'ODA')
+
+    def test_babyl_to_mh(self):
+        # Convert BabylMessage to MHMessage
+        pairs = (('unseen', ['unseen']), ('deleted', []), ('filed', []),
+                 ('answered', ['replied']), ('forwarded', []), ('edited', []),
+                 ('resent', []))
+        for setting, result in pairs:
+            msg = mailbox.BabylMessage(_sample_message)
+            msg.add_label(setting)
+            self.assertEqual(mailbox.MHMessage(msg).get_sequences(), result)
+        msg = mailbox.BabylMessage(_sample_message)
+        for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+                      'edited', 'resent'):
+            msg.add_label(label)
+        self.assertEqual(mailbox.MHMessage(msg).get_sequences(),
+                         ['unseen', 'replied'])
+
+    def test_babyl_to_babyl(self):
+        # Convert BabylMessage to BabylMessage
+        msg = mailbox.BabylMessage(_sample_message)
+        msg.update_visible()
+        for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+                      'edited', 'resent'):
+            msg.add_label(label)
+        msg2 = mailbox.BabylMessage(msg)
+        self.assertEqual(msg2.get_labels(), ['unseen', 'deleted', 'filed',
+                                             'answered', 'forwarded', 'edited',
+                                             'resent'])
+        self.assertEqual(msg.get_visible().keys(), msg2.get_visible().keys())
+        for key in msg.get_visible().keys():
+            self.assertEqual(msg.get_visible()[key], msg2.get_visible()[key])
+
+
+class TestProxyFileBase(TestBase):
+
+    def _test_read(self, proxy):
+        # Read by byte
+        proxy.seek(0)
+        self.assertEqual(proxy.read(), 'bar')
+        proxy.seek(1)
+        self.assertEqual(proxy.read(), 'ar')
+        proxy.seek(0)
+        self.assertEqual(proxy.read(2), 'ba')
+        proxy.seek(1)
+        self.assertEqual(proxy.read(-1), 'ar')
+        proxy.seek(2)
+        self.assertEqual(proxy.read(1000), 'r')
+
+    def _test_readline(self, proxy):
+        # Read by line
+        proxy.seek(0)
+        self.assertEqual(proxy.readline(), 'foo' + os.linesep)
+        self.assertEqual(proxy.readline(), 'bar' + os.linesep)
+        self.assertEqual(proxy.readline(), 'fred' + os.linesep)
+        self.assertEqual(proxy.readline(), 'bob')
+        proxy.seek(2)
+        self.assertEqual(proxy.readline(), 'o' + os.linesep)
+        proxy.seek(6 + 2 * len(os.linesep))
+        self.assertEqual(proxy.readline(), 'fred' + os.linesep)
+        proxy.seek(6 + 2 * len(os.linesep))
+        self.assertEqual(proxy.readline(2), 'fr')
+        self.assertEqual(proxy.readline(-10), 'ed' + os.linesep)
+
+    def _test_readlines(self, proxy):
+        # Read multiple lines
+        proxy.seek(0)
+        self.assertEqual(proxy.readlines(), ['foo' + os.linesep,
+                                            'bar' + os.linesep,
+                                            'fred' + os.linesep, 'bob'])
+        proxy.seek(0)
+        self.assertEqual(proxy.readlines(2), ['foo' + os.linesep])
+        proxy.seek(3 + len(os.linesep))
+        self.assertEqual(proxy.readlines(4 + len(os.linesep)),
+                         ['bar' + os.linesep, 'fred' + os.linesep])
+        proxy.seek(3)
+        self.assertEqual(proxy.readlines(1000), [os.linesep, 'bar' + os.linesep,
+                                                 'fred' + os.linesep, 'bob'])
+
+    def _test_iteration(self, proxy):
+        # Iterate by line
+        proxy.seek(0)
+        iterator = iter(proxy)
+        self.assertEqual(list(iterator),
+            ['foo' + os.linesep, 'bar' + os.linesep, 'fred' + os.linesep, 'bob'])
+
+    def _test_seek_and_tell(self, proxy):
+        # Seek and use tell to check position
+        proxy.seek(3)
+        self.assertEqual(proxy.tell(), 3)
+        self.assertEqual(proxy.read(len(os.linesep)), os.linesep)
+        proxy.seek(2, 1)
+        self.assertEqual(proxy.read(1 + len(os.linesep)), 'r' + os.linesep)
+        proxy.seek(-3 - len(os.linesep), 2)
+        self.assertEqual(proxy.read(3), 'bar')
+        proxy.seek(2, 0)
+        self.assertEqual(proxy.read(), 'o' + os.linesep + 'bar' + os.linesep)
+        proxy.seek(100)
+        self.assertEqual(proxy.read(), '')
+
+    def _test_close(self, proxy):
+        # Close a file
+        proxy.close()
+        self.assertRaises(AttributeError, lambda: proxy.close())
+
+
+class TestProxyFile(TestProxyFileBase):
+
+    def setUp(self):
+        self._path = test_support.TESTFN
+        self._file = open(self._path, 'wb+')
+
+    def tearDown(self):
+        self._file.close()
+        self._delete_recursively(self._path)
+
+    def test_initialize(self):
+        # Initialize and check position
+        self._file.write('foo')
+        pos = self._file.tell()
+        proxy0 = mailbox._ProxyFile(self._file)
+        self.assertEqual(proxy0.tell(), pos)
+        self.assertEqual(self._file.tell(), pos)
+        proxy1 = mailbox._ProxyFile(self._file, 0)
+        self.assertEqual(proxy1.tell(), 0)
+        self.assertEqual(self._file.tell(), pos)
+
+    def test_read(self):
+        self._file.write('bar')
+        self._test_read(mailbox._ProxyFile(self._file))
+
+    def test_readline(self):
+        self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+                                                  os.linesep))
+        self._test_readline(mailbox._ProxyFile(self._file))
+
+    def test_readlines(self):
+        self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+                                                  os.linesep))
+        self._test_readlines(mailbox._ProxyFile(self._file))
+
+    def test_iteration(self):
+        self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+                                                  os.linesep))
+        self._test_iteration(mailbox._ProxyFile(self._file))
+
+    def test_seek_and_tell(self):
+        self._file.write('foo%sbar%s' % (os.linesep, os.linesep))
+        self._test_seek_and_tell(mailbox._ProxyFile(self._file))
+
+    def test_close(self):
+        self._file.write('foo%sbar%s' % (os.linesep, os.linesep))
+        self._test_close(mailbox._ProxyFile(self._file))
+
+
+class TestPartialFile(TestProxyFileBase):
+
+    def setUp(self):
+        self._path = test_support.TESTFN
+        self._file = open(self._path, 'wb+')
+
+    def tearDown(self):
+        self._file.close()
+        self._delete_recursively(self._path)
+
+    def test_initialize(self):
+        # Initialize and check position
+        self._file.write('foo' + os.linesep + 'bar')
+        pos = self._file.tell()
+        proxy = mailbox._PartialFile(self._file, 2, 5)
+        self.assertEqual(proxy.tell(), 0)
+        self.assertEqual(self._file.tell(), pos)
+
+    def test_read(self):
+        self._file.write('***bar***')
+        self._test_read(mailbox._PartialFile(self._file, 3, 6))
+
+    def test_readline(self):
+        self._file.write('!!!!!foo%sbar%sfred%sbob!!!!!' %
+                         (os.linesep, os.linesep, os.linesep))
+        self._test_readline(mailbox._PartialFile(self._file, 5,
+                                                 18 + 3 * len(os.linesep)))
+
+    def test_readlines(self):
+        self._file.write('foo%sbar%sfred%sbob?????' %
+                         (os.linesep, os.linesep, os.linesep))
+        self._test_readlines(mailbox._PartialFile(self._file, 0,
+                                                  13 + 3 * len(os.linesep)))
+
+    def test_iteration(self):
+        self._file.write('____foo%sbar%sfred%sbob####' %
+                         (os.linesep, os.linesep, os.linesep))
+        self._test_iteration(mailbox._PartialFile(self._file, 4,
+                                                  17 + 3 * len(os.linesep)))
+
+    def test_seek_and_tell(self):
+        self._file.write('(((foo%sbar%s$$$' % (os.linesep, os.linesep))
+        self._test_seek_and_tell(mailbox._PartialFile(self._file, 3,
+                                                      9 + 2 * len(os.linesep)))
+
+    def test_close(self):
+        self._file.write('&foo%sbar%s^' % (os.linesep, os.linesep))
+        self._test_close(mailbox._PartialFile(self._file, 1,
+                                              6 + 3 * len(os.linesep)))
+
+
+## Start: tests from the original module (for backward compatibility).
+
+FROM_ = "From some.body at dummy.domain  Sat Jul 24 13:43:35 2004\n"
+DUMMY_MESSAGE = """\
+From: some.body at dummy.domain
+To: me at my.domain
+Subject: Simple Test
+
+This is a dummy message.
+"""
+
+class MaildirTestCase(unittest.TestCase):
+
+    def setUp(self):
+        # create a new maildir mailbox to work with:
+        self._dir = test_support.TESTFN
+        os.mkdir(self._dir)
+        os.mkdir(os.path.join(self._dir, "cur"))
+        os.mkdir(os.path.join(self._dir, "tmp"))
+        os.mkdir(os.path.join(self._dir, "new"))
+        self._counter = 1
+        self._msgfiles = []
+
+    def tearDown(self):
+        map(os.unlink, self._msgfiles)
+        os.rmdir(os.path.join(self._dir, "cur"))
+        os.rmdir(os.path.join(self._dir, "tmp"))
+        os.rmdir(os.path.join(self._dir, "new"))
+        os.rmdir(self._dir)
+
+    def createMessage(self, dir, mbox=False):
+        t = int(time.time() % 1000000)
+        pid = self._counter
+        self._counter += 1
+        filename = os.extsep.join((str(t), str(pid), "myhostname", "mydomain"))
+        tmpname = os.path.join(self._dir, "tmp", filename)
+        newname = os.path.join(self._dir, dir, filename)
+        with open(tmpname, "w") as fp:
+            self._msgfiles.append(tmpname)
+            if mbox:
+                fp.write(FROM_)
+            fp.write(DUMMY_MESSAGE)
+        if hasattr(os, "link"):
+            os.link(tmpname, newname)
+        else:
+            with open(newname, "w") as fp:
+                fp.write(DUMMY_MESSAGE)
+        self._msgfiles.append(newname)
+        return tmpname
+
+    def test_empty_maildir(self):
+        """Test an empty maildir mailbox"""
+        # Test for regression on bug #117490:
+        # Make sure the boxes attribute actually gets set.
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        #self.assertTrue(hasattr(self.mbox, "boxes"))
+        #self.assertTrue(len(self.mbox.boxes) == 0)
+        self.assertIs(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+
+    def test_nonempty_maildir_cur(self):
+        self.createMessage("cur")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        #self.assertTrue(len(self.mbox.boxes) == 1)
+        self.assertIsNot(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+
+    def test_nonempty_maildir_new(self):
+        self.createMessage("new")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        #self.assertTrue(len(self.mbox.boxes) == 1)
+        self.assertIsNot(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+
+    def test_nonempty_maildir_both(self):
+        self.createMessage("cur")
+        self.createMessage("new")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        #self.assertTrue(len(self.mbox.boxes) == 2)
+        self.assertIsNot(self.mbox.next(), None)
+        self.assertIsNot(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+
+    def test_unix_mbox(self):
+        ### should be better!
+        import email.parser
+        fname = self.createMessage("cur", True)
+        n = 0
+        for msg in mailbox.PortableUnixMailbox(open(fname),
+                                               email.parser.Parser().parse):
+            n += 1
+            self.assertEqual(msg["subject"], "Simple Test")
+            self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+        self.assertEqual(n, 1)
+
+## End: classes from the original module (for backward compatibility).
+
+
+_sample_message = """\
+Return-Path: <gkj at gregorykjohnson.com>
+X-Original-To: gkj+person at localhost
+Delivered-To: gkj+person at localhost
+Received: from localhost (localhost [127.0.0.1])
+        by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17
+        for <gkj+person at localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)
+Delivered-To: gkj at sundance.gregorykjohnson.com
+Received: from localhost [127.0.0.1]
+        by localhost with POP3 (fetchmail-6.2.5)
+        for gkj+person at localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)
+Received: from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228])
+        by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746
+        for <gkj at gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)
+Received: by andy.gregorykjohnson.com (Postfix, from userid 1000)
+        id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)
+Date: Wed, 13 Jul 2005 17:23:11 -0400
+From: "Gregory K. Johnson" <gkj at gregorykjohnson.com>
+To: gkj at gregorykjohnson.com
+Subject: Sample message
+Message-ID: <20050713212311.GC4701 at andy.gregorykjohnson.com>
+Mime-Version: 1.0
+Content-Type: multipart/mixed; boundary="NMuMz9nt05w80d4+"
+Content-Disposition: inline
+User-Agent: Mutt/1.5.9i
+
+
+--NMuMz9nt05w80d4+
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+This is a sample message.
+
+--
+Gregory K. Johnson
+
+--NMuMz9nt05w80d4+
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="text.gz"
+Content-Transfer-Encoding: base64
+
+H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs
+3FYlAAAA
+
+--NMuMz9nt05w80d4+--
+"""
+
+_sample_headers = {
+    "Return-Path":"<gkj at gregorykjohnson.com>",
+    "X-Original-To":"gkj+person at localhost",
+    "Delivered-To":"gkj+person at localhost",
+    "Received":"""from localhost (localhost [127.0.0.1])
+        by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17
+        for <gkj+person at localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)""",
+    "Delivered-To":"gkj at sundance.gregorykjohnson.com",
+    "Received":"""from localhost [127.0.0.1]
+        by localhost with POP3 (fetchmail-6.2.5)
+        for gkj+person at localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)""",
+    "Received":"""from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228])
+        by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746
+        for <gkj at gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)""",
+    "Received":"""by andy.gregorykjohnson.com (Postfix, from userid 1000)
+        id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)""",
+    "Date":"Wed, 13 Jul 2005 17:23:11 -0400",
+    "From":""""Gregory K. Johnson" <gkj at gregorykjohnson.com>""",
+    "To":"gkj at gregorykjohnson.com",
+    "Subject":"Sample message",
+    "Mime-Version":"1.0",
+    "Content-Type":"""multipart/mixed; boundary="NMuMz9nt05w80d4+\"""",
+    "Content-Disposition":"inline",
+    "User-Agent": "Mutt/1.5.9i" }
+
+_sample_payloads = ("""This is a sample message.
+
+--
+Gregory K. Johnson
+""",
+"""H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs
+3FYlAAAA
+""")
+
+
+def test_main():
+    tests = (TestMailboxSuperclass, TestMaildir, TestMbox, TestMMDF, TestMH,
+             TestBabyl, TestMessage, TestMaildirMessage, TestMboxMessage,
+             TestMHMessage, TestBabylMessage, TestMMDFMessage,
+             TestMessageConversion, TestProxyFile, TestPartialFile,
+             MaildirTestCase)
+    test_support.run_unittest(*tests)
+    test_support.reap_children()
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_old_mailbox.py
@@ -0,0 +1,152 @@
+# This set of tests exercises the backward-compatibility class
+# in mailbox.py (the ones without write support).
+
+import mailbox
+import os
+import time
+import unittest
+from test import test_support
+
+# cleanup earlier tests
+try:
+    os.unlink(test_support.TESTFN)
+except os.error:
+    pass
+
+FROM_ = "From some.body at dummy.domain  Sat Jul 24 13:43:35 2004\n"
+DUMMY_MESSAGE = """\
+From: some.body at dummy.domain
+To: me at my.domain
+Subject: Simple Test
+
+This is a dummy message.
+"""
+
+class MaildirTestCase(unittest.TestCase):
+
+    def setUp(self):
+        # create a new maildir mailbox to work with:
+        self._dir = test_support.TESTFN
+        os.mkdir(self._dir)
+        os.mkdir(os.path.join(self._dir, "cur"))
+        os.mkdir(os.path.join(self._dir, "tmp"))
+        os.mkdir(os.path.join(self._dir, "new"))
+        self._counter = 1
+        self._msgfiles = []
+
+    def tearDown(self):
+        map(os.unlink, self._msgfiles)
+        os.rmdir(os.path.join(self._dir, "cur"))
+        os.rmdir(os.path.join(self._dir, "tmp"))
+        os.rmdir(os.path.join(self._dir, "new"))
+        os.rmdir(self._dir)
+
+    def createMessage(self, dir, mbox=False):
+        t = int(time.time() % 1000000)
+        pid = self._counter
+        self._counter += 1
+        filename = os.extsep.join((str(t), str(pid), "myhostname", "mydomain"))
+        tmpname = os.path.join(self._dir, "tmp", filename)
+        newname = os.path.join(self._dir, dir, filename)
+        with open(tmpname, "w") as fp:
+            self._msgfiles.append(tmpname)
+            if mbox:
+                fp.write(FROM_)
+            fp.write(DUMMY_MESSAGE)
+        if hasattr(os, "link"):
+            os.link(tmpname, newname)
+        else:
+            with open(newname, "w") as fp:
+                fp.write(DUMMY_MESSAGE)
+        self._msgfiles.append(newname)
+        return tmpname
+
+    def test_empty_maildir(self):
+        """Test an empty maildir mailbox"""
+        # Test for regression on bug #117490:
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        self.assertTrue(len(self.mbox) == 0)
+        self.assertTrue(self.mbox.next() is None)
+        self.assertTrue(self.mbox.next() is None)
+
+    def test_nonempty_maildir_cur(self):
+        self.createMessage("cur")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        self.assertTrue(len(self.mbox) == 1)
+        self.assertTrue(self.mbox.next() is not None)
+        self.assertTrue(self.mbox.next() is None)
+        self.assertTrue(self.mbox.next() is None)
+
+    def test_nonempty_maildir_new(self):
+        self.createMessage("new")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        self.assertTrue(len(self.mbox) == 1)
+        self.assertTrue(self.mbox.next() is not None)
+        self.assertTrue(self.mbox.next() is None)
+        self.assertTrue(self.mbox.next() is None)
+
+    def test_nonempty_maildir_both(self):
+        self.createMessage("cur")
+        self.createMessage("new")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        self.assertTrue(len(self.mbox) == 2)
+        self.assertTrue(self.mbox.next() is not None)
+        self.assertTrue(self.mbox.next() is not None)
+        self.assertTrue(self.mbox.next() is None)
+        self.assertTrue(self.mbox.next() is None)
+
+    def test_unix_mbox(self):
+        ### should be better!
+        import email.parser
+        fname = self.createMessage("cur", True)
+        n = 0
+        with open(fname) as f:
+            for msg in mailbox.PortableUnixMailbox(f,
+                                               email.parser.Parser().parse):
+                n += 1
+                self.assertEqual(msg["subject"], "Simple Test")
+                self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+        self.assertEqual(n, 1)
+
+class MboxTestCase(unittest.TestCase):
+    def setUp(self):
+        # create a new maildir mailbox to work with:
+        self._path = test_support.TESTFN
+
+    def tearDown(self):
+        os.unlink(self._path)
+
+    def test_from_regex (self):
+        # Testing new regex from bug #1633678
+        with open(self._path, 'w') as f:
+            f.write("""From fred at example.com Mon May 31 13:24:50 2004 +0200
+Subject: message 1
+
+body1
+From fred at example.com Mon May 31 13:24:50 2004 -0200
+Subject: message 2
+
+body2
+From fred at example.com Mon May 31 13:24:50 2004
+Subject: message 3
+
+body3
+From fred at example.com Mon May 31 13:24:50 2004
+Subject: message 4
+
+body4
+""")
+        with open(self._path, 'r') as f:
+            box = mailbox.UnixMailbox(f)
+            self.assertTrue(len(list(iter(box))) == 4)
+
+
+    # XXX We still need more tests!
+
+
+def test_main():
+    test_support.run_unittest(MaildirTestCase, MboxTestCase)
+
+
+if __name__ == "__main__":
+    test_main()
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_os.py
@@ -0,0 +1,824 @@
+# As a test suite for the os module, this is woefully inadequate, but this
+# does add tests for a few functions which have been determined to be more
+# portable than they had been thought to be.
+
+import os
+import errno
+import unittest
+import warnings
+import sys
+import signal
+import subprocess
+import time
+from test import test_support
+import mmap
+import uuid
+
+warnings.filterwarnings("ignore", "tempnam", RuntimeWarning, __name__)
+warnings.filterwarnings("ignore", "tmpnam", RuntimeWarning, __name__)
+
+# Tests creating TESTFN
+class FileTests(unittest.TestCase):
+    def setUp(self):
+        if os.path.exists(test_support.TESTFN):
+            os.unlink(test_support.TESTFN)
+    tearDown = setUp
+
+    def test_access(self):
+        f = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
+        os.close(f)
+        self.assertTrue(os.access(test_support.TESTFN, os.W_OK))
+
+    def test_closerange(self):
+        first = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
+        # We must allocate two consecutive file descriptors, otherwise
+        # it will mess up other file descriptors (perhaps even the three
+        # standard ones).
+        second = os.dup(first)
+        try:
+            retries = 0
+            while second != first + 1:
+                os.close(first)
+                retries += 1
+                if retries > 10:
+                    # XXX test skipped
+                    self.skipTest("couldn't allocate two consecutive fds")
+                first, second = second, os.dup(second)
+        finally:
+            os.close(second)
+        # close a fd that is open, and one that isn't
+        os.closerange(first, first + 2)
+        self.assertRaises(OSError, os.write, first, "a")
+
+    @test_support.cpython_only
+    def test_rename(self):
+        path = unicode(test_support.TESTFN)
+        old = sys.getrefcount(path)
+        self.assertRaises(TypeError, os.rename, path, 0)
+        new = sys.getrefcount(path)
+        self.assertEqual(old, new)
+
+
+class TemporaryFileTests(unittest.TestCase):
+    def setUp(self):
+        self.files = []
+        os.mkdir(test_support.TESTFN)
+
+    def tearDown(self):
+        for name in self.files:
+            os.unlink(name)
+        os.rmdir(test_support.TESTFN)
+
+    def check_tempfile(self, name):
+        # make sure it doesn't already exist:
+        self.assertFalse(os.path.exists(name),
+                    "file already exists for temporary file")
+        # make sure we can create the file
+        open(name, "w")
+        self.files.append(name)
+
+    def test_tempnam(self):
+        if not hasattr(os, "tempnam"):
+            return
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "tempnam", RuntimeWarning,
+                                    r"test_os$")
+            warnings.filterwarnings("ignore", "tempnam", DeprecationWarning)
+            self.check_tempfile(os.tempnam())
+
+            name = os.tempnam(test_support.TESTFN)
+            self.check_tempfile(name)
+
+            name = os.tempnam(test_support.TESTFN, "pfx")
+            self.assertTrue(os.path.basename(name)[:3] == "pfx")
+            self.check_tempfile(name)
+
+    def test_tmpfile(self):
+        if not hasattr(os, "tmpfile"):
+            return
+        # As with test_tmpnam() below, the Windows implementation of tmpfile()
+        # attempts to create a file in the root directory of the current drive.
+        # On Vista and Server 2008, this test will always fail for normal users
+        # as writing to the root directory requires elevated privileges.  With
+        # XP and below, the semantics of tmpfile() are the same, but the user
+        # running the test is more likely to have administrative privileges on
+        # their account already.  If that's the case, then os.tmpfile() should
+        # work.  In order to make this test as useful as possible, rather than
+        # trying to detect Windows versions or whether or not the user has the
+        # right permissions, just try and create a file in the root directory
+        # and see if it raises a 'Permission denied' OSError.  If it does, then
+        # test that a subsequent call to os.tmpfile() raises the same error. If
+        # it doesn't, assume we're on XP or below and the user running the test
+        # has administrative privileges, and proceed with the test as normal.
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "tmpfile", DeprecationWarning)
+
+            if sys.platform == 'win32':
+                name = '\\python_test_os_test_tmpfile.txt'
+                if os.path.exists(name):
+                    os.remove(name)
+                try:
+                    fp = open(name, 'w')
+                except IOError, first:
+                    # open() failed, assert tmpfile() fails in the same way.
+                    # Although open() raises an IOError and os.tmpfile() raises an
+                    # OSError(), 'args' will be (13, 'Permission denied') in both
+                    # cases.
+                    try:
+                        fp = os.tmpfile()
+                    except OSError, second:
+                        self.assertEqual(first.args, second.args)
+                    else:
+                        self.fail("expected os.tmpfile() to raise OSError")
+                    return
+                else:
+                    # open() worked, therefore, tmpfile() should work.  Close our
+                    # dummy file and proceed with the test as normal.
+                    fp.close()
+                    os.remove(name)
+
+            fp = os.tmpfile()
+            fp.write("foobar")
+            fp.seek(0,0)
+            s = fp.read()
+            fp.close()
+            self.assertTrue(s == "foobar")
+
+    def test_tmpnam(self):
+        if not hasattr(os, "tmpnam"):
+            return
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "tmpnam", RuntimeWarning,
+                                    r"test_os$")
+            warnings.filterwarnings("ignore", "tmpnam", DeprecationWarning)
+
+            name = os.tmpnam()
+            if sys.platform in ("win32",):
+                # The Windows tmpnam() seems useless.  From the MS docs:
+                #
+                #     The character string that tmpnam creates consists of
+                #     the path prefix, defined by the entry P_tmpdir in the
+                #     file STDIO.H, followed by a sequence consisting of the
+                #     digit characters '0' through '9'; the numerical value
+                #     of this string is in the range 1 - 65,535.  Changing the
+                #     definitions of L_tmpnam or P_tmpdir in STDIO.H does not
+                #     change the operation of tmpnam.
+                #
+                # The really bizarre part is that, at least under MSVC6,
+                # P_tmpdir is "\\".  That is, the path returned refers to
+                # the root of the current drive.  That's a terrible place to
+                # put temp files, and, depending on privileges, the user
+                # may not even be able to open a file in the root directory.
+                self.assertFalse(os.path.exists(name),
+                            "file already exists for temporary file")
+            else:
+                self.check_tempfile(name)
+
+# Test attributes on return values from os.*stat* family.
+class StatAttributeTests(unittest.TestCase):
+    def setUp(self):
+        os.mkdir(test_support.TESTFN)
+        self.fname = os.path.join(test_support.TESTFN, "f1")
+        f = open(self.fname, 'wb')
+        f.write("ABC")
+        f.close()
+
+    def tearDown(self):
+        os.unlink(self.fname)
+        os.rmdir(test_support.TESTFN)
+
+    def test_stat_attributes(self):
+        if not hasattr(os, "stat"):
+            return
+
+        import stat
+        result = os.stat(self.fname)
+
+        # Make sure direct access works
+        self.assertEqual(result[stat.ST_SIZE], 3)
+        self.assertEqual(result.st_size, 3)
+
+        # Make sure all the attributes are there
+        members = dir(result)
+        for name in dir(stat):
+            if name[:3] == 'ST_':
+                attr = name.lower()
+                if name.endswith("TIME"):
+                    def trunc(x): return int(x)
+                else:
+                    def trunc(x): return x
+                self.assertEqual(trunc(getattr(result, attr)),
+                                 result[getattr(stat, name)])
+                self.assertIn(attr, members)
+
+        try:
+            result[200]
+            self.fail("No exception thrown")
+        except IndexError:
+            pass
+
+        # Make sure that assignment fails
+        try:
+            result.st_mode = 1
+            self.fail("No exception thrown")
+        except (AttributeError, TypeError):
+            pass
+
+        try:
+            result.st_rdev = 1
+            self.fail("No exception thrown")
+        except (AttributeError, TypeError):
+            pass
+
+        try:
+            result.parrot = 1
+            self.fail("No exception thrown")
+        except AttributeError:
+            pass
+
+        # Use the stat_result constructor with a too-short tuple.
+        try:
+            result2 = os.stat_result((10,))
+            self.fail("No exception thrown")
+        except TypeError:
+            pass
+
+        # Use the constructor with a too-long tuple.
+        try:
+            result2 = os.stat_result((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14))
+        except TypeError:
+            pass
+
+
+    def test_statvfs_attributes(self):
+        if not hasattr(os, "statvfs"):
+            return
+
+        try:
+            result = os.statvfs(self.fname)
+        except OSError, e:
+            # On AtheOS, glibc always returns ENOSYS
+            if e.errno == errno.ENOSYS:
+                return
+
+        # Make sure direct access works
+        self.assertEqual(result.f_bfree, result[3])
+
+        # Make sure all the attributes are there.
+        members = ('bsize', 'frsize', 'blocks', 'bfree', 'bavail', 'files',
+                    'ffree', 'favail', 'flag', 'namemax')
+        for value, member in enumerate(members):
+            self.assertEqual(getattr(result, 'f_' + member), result[value])
+
+        # Make sure that assignment really fails
+        try:
+            result.f_bfree = 1
+            self.fail("No exception thrown")
+        except TypeError:
+            pass
+
+        try:
+            result.parrot = 1
+            self.fail("No exception thrown")
+        except AttributeError:
+            pass
+
+        # Use the constructor with a too-short tuple.
+        try:
+            result2 = os.statvfs_result((10,))
+            self.fail("No exception thrown")
+        except TypeError:
+            pass
+
+        # Use the constructor with a too-long tuple.
+        try:
+            result2 = os.statvfs_result((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14))
+        except TypeError:
+            pass
+
+    def test_utime_dir(self):
+        delta = 1000000
+        st = os.stat(test_support.TESTFN)
+        # round to int, because some systems may support sub-second
+        # time stamps in stat, but not in utime.
+        os.utime(test_support.TESTFN, (st.st_atime, int(st.st_mtime-delta)))
+        st2 = os.stat(test_support.TESTFN)
+        self.assertEqual(st2.st_mtime, int(st.st_mtime-delta))
+
+    # Restrict test to Win32, since there is no guarantee other
+    # systems support centiseconds
+    if sys.platform == 'win32':
+        def get_file_system(path):
+            root = os.path.splitdrive(os.path.abspath(path))[0] + '\\'
+            import ctypes
+            kernel32 = ctypes.windll.kernel32
+            buf = ctypes.create_string_buffer("", 100)
+            if kernel32.GetVolumeInformationA(root, None, 0, None, None, None, buf, len(buf)):
+                return buf.value
+
+        if get_file_system(test_support.TESTFN) == "NTFS":
+            def test_1565150(self):
+                t1 = 1159195039.25
+                os.utime(self.fname, (t1, t1))
+                self.assertEqual(os.stat(self.fname).st_mtime, t1)
+
+            def test_large_time(self):
+                t1 = 5000000000 # some day in 2128
+                os.utime(self.fname, (t1, t1))
+                self.assertEqual(os.stat(self.fname).st_mtime, t1)
+
+        def test_1686475(self):
+            # Verify that an open file can be stat'ed
+            try:
+                os.stat(r"c:\pagefile.sys")
+            except WindowsError, e:
+                if e.errno == 2: # file does not exist; cannot run test
+                    return
+                self.fail("Could not stat pagefile.sys")
+
+from test import mapping_tests
+
+class EnvironTests(mapping_tests.BasicTestMappingProtocol):
+    """check that os.environ object conform to mapping protocol"""
+    type2test = None
+    def _reference(self):
+        return {"KEY1":"VALUE1", "KEY2":"VALUE2", "KEY3":"VALUE3"}
+    def _empty_mapping(self):
+        os.environ.clear()
+        return os.environ
+    def setUp(self):
+        self.__save = dict(os.environ)
+        os.environ.clear()
+    def tearDown(self):
+        os.environ.clear()
+        os.environ.update(self.__save)
+
+    # Bug 1110478
+    def test_update2(self):
+        if os.path.exists("/bin/sh"):
+            os.environ.update(HELLO="World")
+            with os.popen("/bin/sh -c 'echo $HELLO'") as popen:
+                value = popen.read().strip()
+                self.assertEqual(value, "World")
+
+class WalkTests(unittest.TestCase):
+    """Tests for os.walk()."""
+
+    def test_traversal(self):
+        import os
+        from os.path import join
+
+        # Build:
+        #     TESTFN/
+        #       TEST1/              a file kid and two directory kids
+        #         tmp1
+        #         SUB1/             a file kid and a directory kid
+        #           tmp2
+        #           SUB11/          no kids
+        #         SUB2/             a file kid and a dirsymlink kid
+        #           tmp3
+        #           link/           a symlink to TESTFN.2
+        #       TEST2/
+        #         tmp4              a lone file
+        walk_path = join(test_support.TESTFN, "TEST1")
+        sub1_path = join(walk_path, "SUB1")
+        sub11_path = join(sub1_path, "SUB11")
+        sub2_path = join(walk_path, "SUB2")
+        tmp1_path = join(walk_path, "tmp1")
+        tmp2_path = join(sub1_path, "tmp2")
+        tmp3_path = join(sub2_path, "tmp3")
+        link_path = join(sub2_path, "link")
+        t2_path = join(test_support.TESTFN, "TEST2")
+        tmp4_path = join(test_support.TESTFN, "TEST2", "tmp4")
+
+        # Create stuff.
+        os.makedirs(sub11_path)
+        os.makedirs(sub2_path)
+        os.makedirs(t2_path)
+        for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path:
+            f = file(path, "w")
+            f.write("I'm " + path + " and proud of it.  Blame test_os.\n")
+            f.close()
+        if hasattr(os, "symlink"):
+            os.symlink(os.path.abspath(t2_path), link_path)
+            sub2_tree = (sub2_path, ["link"], ["tmp3"])
+        else:
+            sub2_tree = (sub2_path, [], ["tmp3"])
+
+        # Walk top-down.
+        all = list(os.walk(walk_path))
+        self.assertEqual(len(all), 4)
+        # We can't know which order SUB1 and SUB2 will appear in.
+        # Not flipped:  TESTFN, SUB1, SUB11, SUB2
+        #     flipped:  TESTFN, SUB2, SUB1, SUB11
+        flipped = all[0][1][0] != "SUB1"
+        all[0][1].sort()
+        self.assertEqual(all[0], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
+        self.assertEqual(all[1 + flipped], (sub1_path, ["SUB11"], ["tmp2"]))
+        self.assertEqual(all[2 + flipped], (sub11_path, [], []))
+        self.assertEqual(all[3 - 2 * flipped], sub2_tree)
+
+        # Prune the search.
+        all = []
+        for root, dirs, files in os.walk(walk_path):
+            all.append((root, dirs, files))
+            # Don't descend into SUB1.
+            if 'SUB1' in dirs:
+                # Note that this also mutates the dirs we appended to all!
+                dirs.remove('SUB1')
+        self.assertEqual(len(all), 2)
+        self.assertEqual(all[0], (walk_path, ["SUB2"], ["tmp1"]))
+        self.assertEqual(all[1], sub2_tree)
+
+        # Walk bottom-up.
+        all = list(os.walk(walk_path, topdown=False))
+        self.assertEqual(len(all), 4)
+        # We can't know which order SUB1 and SUB2 will appear in.
+        # Not flipped:  SUB11, SUB1, SUB2, TESTFN
+        #     flipped:  SUB2, SUB11, SUB1, TESTFN
+        flipped = all[3][1][0] != "SUB1"
+        all[3][1].sort()
+        self.assertEqual(all[3], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
+        self.assertEqual(all[flipped], (sub11_path, [], []))
+        self.assertEqual(all[flipped + 1], (sub1_path, ["SUB11"], ["tmp2"]))
+        self.assertEqual(all[2 - 2 * flipped], sub2_tree)
+
+        if hasattr(os, "symlink"):
+            # Walk, following symlinks.
+            for root, dirs, files in os.walk(walk_path, followlinks=True):
+                if root == link_path:
+                    self.assertEqual(dirs, [])
+                    self.assertEqual(files, ["tmp4"])
+                    break
+            else:
+                self.fail("Didn't follow symlink with followlinks=True")
+
+    def tearDown(self):
+        # Tear everything down.  This is a decent use for bottom-up on
+        # Windows, which doesn't have a recursive delete command.  The
+        # (not so) subtlety is that rmdir will fail unless the dir's
+        # kids are removed first, so bottom up is essential.
+        for root, dirs, files in os.walk(test_support.TESTFN, topdown=False):
+            for name in files:
+                os.remove(os.path.join(root, name))
+            for name in dirs:
+                dirname = os.path.join(root, name)
+                if not os.path.islink(dirname):
+                    os.rmdir(dirname)
+                else:
+                    os.remove(dirname)
+        os.rmdir(test_support.TESTFN)
+
+class MakedirTests (unittest.TestCase):
+    def setUp(self):
+        os.mkdir(test_support.TESTFN)
+
+    def test_makedir(self):
+        base = test_support.TESTFN
+        path = os.path.join(base, 'dir1', 'dir2', 'dir3')
+        os.makedirs(path)             # Should work
+        path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4')
+        os.makedirs(path)
+
+        # Try paths with a '.' in them
+        self.assertRaises(OSError, os.makedirs, os.curdir)
+        path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', os.curdir)
+        os.makedirs(path)
+        path = os.path.join(base, 'dir1', os.curdir, 'dir2', 'dir3', 'dir4',
+                            'dir5', 'dir6')
+        os.makedirs(path)
+
+
+
+
+    def tearDown(self):
+        path = os.path.join(test_support.TESTFN, 'dir1', 'dir2', 'dir3',
+                            'dir4', 'dir5', 'dir6')
+        # If the tests failed, the bottom-most directory ('../dir6')
+        # may not have been created, so we look for the outermost directory
+        # that exists.
+        while not os.path.exists(path) and path != test_support.TESTFN:
+            path = os.path.dirname(path)
+
+        os.removedirs(path)
+
+class DevNullTests (unittest.TestCase):
+    def test_devnull(self):
+        f = file(os.devnull, 'w')
+        f.write('hello')
+        f.close()
+        f = file(os.devnull, 'r')
+        self.assertEqual(f.read(), '')
+        f.close()
+
+class URandomTests (unittest.TestCase):
+    def test_urandom(self):
+        try:
+            self.assertEqual(len(os.urandom(1)), 1)
+            self.assertEqual(len(os.urandom(10)), 10)
+            self.assertEqual(len(os.urandom(100)), 100)
+            self.assertEqual(len(os.urandom(1000)), 1000)
+            # see http://bugs.python.org/issue3708
+            self.assertRaises(TypeError, os.urandom, 0.9)
+            self.assertRaises(TypeError, os.urandom, 1.1)
+            self.assertRaises(TypeError, os.urandom, 2.0)
+        except NotImplementedError:
+            pass
+
+    def test_execvpe_with_bad_arglist(self):
+        self.assertRaises(ValueError, os.execvpe, 'notepad', [], None)
+
+class Win32ErrorTests(unittest.TestCase):
+    def test_rename(self):
+        self.assertRaises(WindowsError, os.rename, test_support.TESTFN, test_support.TESTFN+".bak")
+
+    def test_remove(self):
+        self.assertRaises(WindowsError, os.remove, test_support.TESTFN)
+
+    def test_chdir(self):
+        self.assertRaises(WindowsError, os.chdir, test_support.TESTFN)
+
+    def test_mkdir(self):
+        f = open(test_support.TESTFN, "w")
+        try:
+            self.assertRaises(WindowsError, os.mkdir, test_support.TESTFN)
+        finally:
+            f.close()
+            os.unlink(test_support.TESTFN)
+
+    def test_utime(self):
+        self.assertRaises(WindowsError, os.utime, test_support.TESTFN, None)
+
+    def test_chmod(self):
+        self.assertRaises(WindowsError, os.chmod, test_support.TESTFN, 0)
+
+class TestInvalidFD(unittest.TestCase):
+    singles = ["fchdir", "fdopen", "dup", "fdatasync", "fstat",
+               "fstatvfs", "fsync", "tcgetpgrp", "ttyname"]
+    #singles.append("close")
+    #We omit close because it doesn'r raise an exception on some platforms
+    def get_single(f):
+        def helper(self):
+            if  hasattr(os, f):
+                self.check(getattr(os, f))
+        return helper
+    for f in singles:
+        locals()["test_"+f] = get_single(f)
+
+    def check(self, f, *args):
+        try:
+            f(test_support.make_bad_fd(), *args)
+        except OSError as e:
+            self.assertEqual(e.errno, errno.EBADF)
+        else:
+            self.fail("%r didn't raise a OSError with a bad file descriptor"
+                      % f)
+
+    def test_isatty(self):
+        if hasattr(os, "isatty"):
+            self.assertEqual(os.isatty(test_support.make_bad_fd()), False)
+
+    def test_closerange(self):
+        if hasattr(os, "closerange"):
+            fd = test_support.make_bad_fd()
+            # Make sure none of the descriptors we are about to close are
+            # currently valid (issue 6542).
+            for i in range(10):
+                try: os.fstat(fd+i)
+                except OSError:
+                    pass
+                else:
+                    break
+            if i < 2:
+                raise unittest.SkipTest(
+                    "Unable to acquire a range of invalid file descriptors")
+            self.assertEqual(os.closerange(fd, fd + i-1), None)
+
+    def test_dup2(self):
+        if hasattr(os, "dup2"):
+            self.check(os.dup2, 20)
+
+    def test_fchmod(self):
+        if hasattr(os, "fchmod"):
+            self.check(os.fchmod, 0)
+
+    def test_fchown(self):
+        if hasattr(os, "fchown"):
+            self.check(os.fchown, -1, -1)
+
+    def test_fpathconf(self):
+        if hasattr(os, "fpathconf"):
+            self.check(os.fpathconf, "PC_NAME_MAX")
+
+    def test_ftruncate(self):
+        if hasattr(os, "ftruncate"):
+            self.check(os.ftruncate, 0)
+
+    def test_lseek(self):
+        if hasattr(os, "lseek"):
+            self.check(os.lseek, 0, 0)
+
+    def test_read(self):
+        if hasattr(os, "read"):
+            self.check(os.read, 1)
+
+    def test_tcsetpgrpt(self):
+        if hasattr(os, "tcsetpgrp"):
+            self.check(os.tcsetpgrp, 0)
+
+    def test_write(self):
+        if hasattr(os, "write"):
+            self.check(os.write, " ")
+
+if sys.platform != 'win32':
+    class Win32ErrorTests(unittest.TestCase):
+        pass
+
+    class PosixUidGidTests(unittest.TestCase):
+        if hasattr(os, 'setuid'):
+            def test_setuid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setuid, 0)
+                self.assertRaises(OverflowError, os.setuid, 1<<32)
+
+        if hasattr(os, 'setgid'):
+            def test_setgid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setgid, 0)
+                self.assertRaises(OverflowError, os.setgid, 1<<32)
+
+        if hasattr(os, 'seteuid'):
+            def test_seteuid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.seteuid, 0)
+                self.assertRaises(OverflowError, os.seteuid, 1<<32)
+
+        if hasattr(os, 'setegid'):
+            def test_setegid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setegid, 0)
+                self.assertRaises(OverflowError, os.setegid, 1<<32)
+
+        if hasattr(os, 'setreuid'):
+            def test_setreuid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setreuid, 0, 0)
+                self.assertRaises(OverflowError, os.setreuid, 1<<32, 0)
+                self.assertRaises(OverflowError, os.setreuid, 0, 1<<32)
+
+            def test_setreuid_neg1(self):
+                # Needs to accept -1.  We run this in a subprocess to avoid
+                # altering the test runner's process state (issue8045).
+                subprocess.check_call([
+                        sys.executable, '-c',
+                        'import os,sys;os.setreuid(-1,-1);sys.exit(0)'])
+
+        if hasattr(os, 'setregid'):
+            def test_setregid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setregid, 0, 0)
+                self.assertRaises(OverflowError, os.setregid, 1<<32, 0)
+                self.assertRaises(OverflowError, os.setregid, 0, 1<<32)
+
+            def test_setregid_neg1(self):
+                # Needs to accept -1.  We run this in a subprocess to avoid
+                # altering the test runner's process state (issue8045).
+                subprocess.check_call([
+                        sys.executable, '-c',
+                        'import os,sys;os.setregid(-1,-1);sys.exit(0)'])
+else:
+    class PosixUidGidTests(unittest.TestCase):
+        pass
+
+ at unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+class Win32KillTests(unittest.TestCase):
+    def _kill(self, sig):
+        # Start sys.executable as a subprocess and communicate from the
+        # subprocess to the parent that the interpreter is ready. When it
+        # becomes ready, send *sig* via os.kill to the subprocess and check
+        # that the return code is equal to *sig*.
+        import ctypes
+        from ctypes import wintypes
+        import msvcrt
+
+        # Since we can't access the contents of the process' stdout until the
+        # process has exited, use PeekNamedPipe to see what's inside stdout
+        # without waiting. This is done so we can tell that the interpreter
+        # is started and running at a point where it could handle a signal.
+        PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
+        PeekNamedPipe.restype = wintypes.BOOL
+        PeekNamedPipe.argtypes = (wintypes.HANDLE, # Pipe handle
+                                  ctypes.POINTER(ctypes.c_char), # stdout buf
+                                  wintypes.DWORD, # Buffer size
+                                  ctypes.POINTER(wintypes.DWORD), # bytes read
+                                  ctypes.POINTER(wintypes.DWORD), # bytes avail
+                                  ctypes.POINTER(wintypes.DWORD)) # bytes left
+        msg = "running"
+        proc = subprocess.Popen([sys.executable, "-c",
+                                 "import sys;"
+                                 "sys.stdout.write('{}');"
+                                 "sys.stdout.flush();"
+                                 "input()".format(msg)],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                stdin=subprocess.PIPE)
+        self.addCleanup(proc.stdout.close)
+        self.addCleanup(proc.stderr.close)
+        self.addCleanup(proc.stdin.close)
+
+        count, max = 0, 100
+        while count < max and proc.poll() is None:
+            # Create a string buffer to store the result of stdout from the pipe
+            buf = ctypes.create_string_buffer(len(msg))
+            # Obtain the text currently in proc.stdout
+            # Bytes read/avail/left are left as NULL and unused
+            rslt = PeekNamedPipe(msvcrt.get_osfhandle(proc.stdout.fileno()),
+                                 buf, ctypes.sizeof(buf), None, None, None)
+            self.assertNotEqual(rslt, 0, "PeekNamedPipe failed")
+            if buf.value:
+                self.assertEqual(msg, buf.value)
+                break
+            time.sleep(0.1)
+            count += 1
+        else:
+            self.fail("Did not receive communication from the subprocess")
+
+        os.kill(proc.pid, sig)
+        self.assertEqual(proc.wait(), sig)
+
+    def test_kill_sigterm(self):
+        # SIGTERM doesn't mean anything special, but make sure it works
+        self._kill(signal.SIGTERM)
+
+    def test_kill_int(self):
+        # os.kill on Windows can take an int which gets set as the exit code
+        self._kill(100)
+
+    def _kill_with_event(self, event, name):
+        tagname = "test_os_%s" % uuid.uuid1()
+        m = mmap.mmap(-1, 1, tagname)
+        m[0] = '0'
+        # Run a script which has console control handling enabled.
+        proc = subprocess.Popen([sys.executable,
+                   os.path.join(os.path.dirname(__file__),
+                                "win_console_handler.py"), tagname],
+                   creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
+        # Let the interpreter startup before we send signals. See #3137.
+        count, max = 0, 20
+        while count < max and proc.poll() is None:
+            if m[0] == '1':
+                break
+            time.sleep(0.5)
+            count += 1
+        else:
+            self.fail("Subprocess didn't finish initialization")
+        os.kill(proc.pid, event)
+        # proc.send_signal(event) could also be done here.
+        # Allow time for the signal to be passed and the process to exit.
+        time.sleep(0.5)
+        if not proc.poll():
+            # Forcefully kill the process if we weren't able to signal it.
+            os.kill(proc.pid, signal.SIGINT)
+            self.fail("subprocess did not stop on {}".format(name))
+
+    @unittest.skip("subprocesses aren't inheriting CTRL+C property")
+    def test_CTRL_C_EVENT(self):
+        from ctypes import wintypes
+        import ctypes
+
+        # Make a NULL value by creating a pointer with no argument.
+        NULL = ctypes.POINTER(ctypes.c_int)()
+        SetConsoleCtrlHandler = ctypes.windll.kernel32.SetConsoleCtrlHandler
+        SetConsoleCtrlHandler.argtypes = (ctypes.POINTER(ctypes.c_int),
+                                          wintypes.BOOL)
+        SetConsoleCtrlHandler.restype = wintypes.BOOL
+
+        # Calling this with NULL and FALSE causes the calling process to
+        # handle CTRL+C, rather than ignore it. This property is inherited
+        # by subprocesses.
+        SetConsoleCtrlHandler(NULL, 0)
+
+        self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT")
+
+    def test_CTRL_BREAK_EVENT(self):
+        self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT")
+
+
+def test_main():
+    test_support.run_unittest(
+        FileTests,
+        TemporaryFileTests,
+        StatAttributeTests,
+        EnvironTests,
+        WalkTests,
+        MakedirTests,
+        DevNullTests,
+        URandomTests,
+        Win32ErrorTests,
+        TestInvalidFD,
+        PosixUidGidTests,
+        Win32KillTests
+    )
+
+if __name__ == "__main__":
+    test_main()
diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py
--- a/pypy/rlib/clibffi.py
+++ b/pypy/rlib/clibffi.py
@@ -655,6 +655,13 @@
         return FuncPtr(name, argtypes, restype, dlsym(self.lib, name),
                        flags=flags, keepalive=self)
 
+    def getpointer_byordinal(self, name, argtypes, restype, 
+            flags=FUNCFLAG_CDECL):
+        # these arguments are already casted to proper ffi
+        # structures!
+        return FuncPtr(name, argtypes, restype, dlsym_byordinal(self.lib, name),
+                       flags=flags, keepalive=self)
+
     def getrawpointer(self, name, argtypes, restype, flags=FUNCFLAG_CDECL):
         # these arguments are already casted to proper ffi
         # structures!

From noreply at buildbot.pypy.org  Wed Apr 11 21:15:25 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 21:15:25 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: help find import library,
	silence compiler warnings
Message-ID: <20120411191525.9DDA582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54293:056d06128b67
Date: 2012-04-11 22:14 +0300
http://bitbucket.org/pypy/pypy/changeset/056d06128b67/

Log:	help find import library, silence compiler warnings

diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -21,7 +21,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC']
     res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
@@ -34,6 +34,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST'] # needed for VC10
     else:
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -16,7 +16,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
     res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],

From noreply at buildbot.pypy.org  Wed Apr 11 21:19:18 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 21:19:18 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: remove prematurely added function
Message-ID: <20120411191918.4163182F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54294:b4573bb1e0b4
Date: 2012-04-11 22:18 +0300
http://bitbucket.org/pypy/pypy/changeset/b4573bb1e0b4/

Log:	remove prematurely added function

diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py
--- a/pypy/rlib/clibffi.py
+++ b/pypy/rlib/clibffi.py
@@ -655,13 +655,6 @@
         return FuncPtr(name, argtypes, restype, dlsym(self.lib, name),
                        flags=flags, keepalive=self)
 
-    def getpointer_byordinal(self, name, argtypes, restype, 
-            flags=FUNCFLAG_CDECL):
-        # these arguments are already casted to proper ffi
-        # structures!
-        return FuncPtr(name, argtypes, restype, dlsym_byordinal(self.lib, name),
-                       flags=flags, keepalive=self)
-
     def getrawpointer(self, name, argtypes, restype, flags=FUNCFLAG_CDECL):
         # these arguments are already casted to proper ffi
         # structures!

From noreply at buildbot.pypy.org  Wed Apr 11 23:42:29 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 23:42:29 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: make tests applicicable to
 windows, fix check_signum for valid values
Message-ID: <20120411214229.C9FE782F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54295:a2db34cd9d65
Date: 2012-04-12 00:17 +0300
http://bitbucket.org/pypy/pypy/changeset/a2db34cd9d65/

Log:	make tests applicicable to windows, fix check_signum for valid
	values

diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -15,7 +15,8 @@
 
 def setup():
     for key, value in cpy_signal.__dict__.items():
-        if key.startswith('SIG') and is_valid_int(value):
+        if (key.startswith('SIG') or key.startswith('CTRL_')) and \
+                is_valid_int(value):
             globals()[key] = value
             yield key
 
@@ -242,9 +243,12 @@
     return space.w_None
 
 def check_signum(space, signum):
-    if signum < 1 or signum >= NSIG:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("signal number out of range"))
+    for sig in signal_names:
+        if signum == globals()[sig]:
+            return
+    raise OperationError(space.w_ValueError,
+                         space.wrap("invalid signal value"))
+
 
 @jit.dont_look_inside
 @unwrap_spec(signum=int)
diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -8,6 +8,8 @@
     def setup_class(cls):
         if not hasattr(os, 'kill') or not hasattr(os, 'getpid'):
             py.test.skip("requires os.kill() and os.getpid()")
+        if not hasattr(cpy_signal, 'SIGUSR1'):    
+            py.test.skip("requires SIGUSR1 in signal")
         cls.space = gettestobjspace(usemodules=['signal'])
 
     def test_checksignals(self):
@@ -45,64 +47,72 @@
     def test_exported_names(self):
         self.signal.__dict__   # crashes if the interpleveldefs are invalid
 
-    def test_usr1(self):
-        import types, posix
+    def test_basics(self):
+        import types, os
+        if not hasattr(os, 'kill') or not hasattr(os, 'getpid'):
+            skip("requires os.kill() and os.getpid()")
         signal = self.signal   # the signal module to test
+        try:
+            signum = signal.USR1
+        except:
+            signum = signal.CTRL_BREAK_EVENT
 
         received = []
         def myhandler(signum, frame):
             assert isinstance(frame, types.FrameType)
             received.append(signum)
-        signal.signal(signal.SIGUSR1, myhandler)
+        signal.signal(signum, myhandler)
 
-        posix.kill(posix.getpid(), signal.SIGUSR1)
+        print dir(os)
+
+        os.kill(os.getpid(), signum)
         # the signal should be delivered to the handler immediately
-        assert received == [signal.SIGUSR1]
+        assert received == [signum]
         del received[:]
 
-        posix.kill(posix.getpid(), signal.SIGUSR1)
+        os.kill(os.getpid(), signum)
         # the signal should be delivered to the handler immediately
-        assert received == [signal.SIGUSR1]
+        assert received == [signum]
         del received[:]
 
-        signal.signal(signal.SIGUSR1, signal.SIG_IGN)
+        signal.signal(signum, signal.SIG_IGN)
 
-        posix.kill(posix.getpid(), signal.SIGUSR1)
+        os.kill(os.getpid(), signum)
         for i in range(10000):
             # wait a bit - signal should not arrive
             if received:
                 break
         assert received == []
 
-        signal.signal(signal.SIGUSR1, signal.SIG_DFL)
+        signal.signal(signum, signal.SIG_DFL)
 
 
     def test_default_return(self):
         """
         Test that signal.signal returns SIG_DFL if that is the current handler.
         """
-        from signal import signal, SIGUSR1, SIG_DFL, SIG_IGN
+        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
 
         try:
             for handler in SIG_DFL, SIG_IGN, lambda *a: None:
-                signal(SIGUSR1, SIG_DFL)
-                assert signal(SIGUSR1, handler) == SIG_DFL
+                signal(SIGINT, SIG_DFL)
+                assert signal(SIGINT, handler) == SIG_DFL
         finally:
-            signal(SIGUSR1, SIG_DFL)
+            signal(SIGINT, SIG_DFL)
 
 
     def test_ignore_return(self):
         """
         Test that signal.signal returns SIG_IGN if that is the current handler.
         """
-        from signal import signal, SIGUSR1, SIG_DFL, SIG_IGN
+        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
 
         try:
             for handler in SIG_DFL, SIG_IGN, lambda *a: None:
-                signal(SIGUSR1, SIG_IGN)
-                assert signal(SIGUSR1, handler) == SIG_IGN
+                signal(SIGINT, SIG_IGN)
+                assert signal(SIGINT, handler) == SIG_IGN
         finally:
-            signal(SIGUSR1, SIG_DFL)
+            signal(SIGINT, SIG_DFL)
 
 
     def test_obj_return(self):
@@ -110,43 +120,47 @@
         Test that signal.signal returns a Python object if one is the current
         handler.
         """
-        from signal import signal, SIGUSR1, SIG_DFL, SIG_IGN
+        from signal import signal, SIGINT, SIG_DFL, SIG_IGN
         def installed(*a):
             pass
 
         try:
             for handler in SIG_DFL, SIG_IGN, lambda *a: None:
-                signal(SIGUSR1, installed)
-                assert signal(SIGUSR1, handler) is installed
+                signal(SIGINT, installed)
+                assert signal(SIGINT, handler) is installed
         finally:
-            signal(SIGUSR1, SIG_DFL)
+            signal(SIGINT, SIG_DFL)
 
 
     def test_getsignal(self):
         """
         Test that signal.getsignal returns the currently installed handler.
         """
-        from signal import getsignal, signal, SIGUSR1, SIG_DFL, SIG_IGN
+        from signal import getsignal, signal, SIGINT, SIG_DFL, SIG_IGN
 
         def handler(*a):
             pass
 
         try:
-            assert getsignal(SIGUSR1) == SIG_DFL
-            signal(SIGUSR1, SIG_DFL)
-            assert getsignal(SIGUSR1) == SIG_DFL
-            signal(SIGUSR1, SIG_IGN)
-            assert getsignal(SIGUSR1) == SIG_IGN
-            signal(SIGUSR1, handler)
-            assert getsignal(SIGUSR1) is handler
+            assert getsignal(SIGINT) == SIG_DFL
+            signal(SIGINT, SIG_DFL)
+            assert getsignal(SIGINT) == SIG_DFL
+            signal(SIGINT, SIG_IGN)
+            assert getsignal(SIGINT) == SIG_IGN
+            signal(SIGINT, handler)
+            assert getsignal(SIGINT) is handler
         finally:
-            signal(SIGUSR1, SIG_DFL)
+            signal(SIGINT, SIG_DFL)
 
         raises(ValueError, getsignal, 4444)
         raises(ValueError, signal, 4444, lambda *args: None)
+        raises(ValueError, signal, 7, lambda *args: None)
 
     def test_alarm(self):
-        from signal import alarm, signal, SIG_DFL, SIGALRM
+        try:
+            from signal import alarm, signal, SIG_DFL, SIGALRM
+        except:
+            skip('no alarm on this platform')
         import time
         l = []
         def handler(*a):
@@ -163,10 +177,13 @@
             signal(SIGALRM, SIG_DFL)
 
     def test_set_wakeup_fd(self):
-        import signal, posix, fcntl
+        try:
+            import signal, posix, fcntl
+        except ImportError:
+            skip('cannot import posix or fcntl')
         def myhandler(signum, frame):
             pass
-        signal.signal(signal.SIGUSR1, myhandler)
+        signal.signal(signal.SIGINT, myhandler)
         #
         def cannot_read():
             try:
@@ -187,17 +204,19 @@
         old_wakeup = signal.set_wakeup_fd(fd_write)
         try:
             cannot_read()
-            posix.kill(posix.getpid(), signal.SIGUSR1)
+            posix.kill(posix.getpid(), signal.SIGINT)
             res = posix.read(fd_read, 1)
             assert res == '\x00'
             cannot_read()
         finally:
             old_wakeup = signal.set_wakeup_fd(old_wakeup)
         #
-        signal.signal(signal.SIGUSR1, signal.SIG_DFL)
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
 
     def test_siginterrupt(self):
         import signal, os, time
+        if not hasattr(signal, 'siginterrupt'):
+            skip('non siginterrupt in signal')
         signum = signal.SIGUSR1
         def readpipe_is_not_interrupted():
             # from CPython's test_signal.readpipe_interrupted()

From noreply at buildbot.pypy.org  Wed Apr 11 23:42:31 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 11 Apr 2012 23:42:31 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: add tests,
	fix implementation of strftime('%f')
Message-ID: <20120411214231.1C10382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54296:30144e45f849
Date: 2012-04-12 00:41 +0300
http://bitbucket.org/pypy/pypy/changeset/30144e45f849/

Log:	add tests, fix implementation of strftime('%f')

diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -572,7 +572,7 @@
                 if i < length and format[i] == '#':
                     # not documented by python
                     i += 1
-                if i >= length or format[i] not in "aAbBcdfHIjmMpSUwWxXyYzZ%":
+                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
                     raise OperationError(space.w_ValueError,
                                          space.wrap("invalid format string"))
             i += 1
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -211,7 +211,7 @@
 
     def test_strftime(self):
         import time as rctime
-
+        import os
         t = rctime.time()
         tt = rctime.gmtime(t)
         for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I',
@@ -226,6 +226,14 @@
         exp = '2000 01 01 00 00 00 1 001'
         assert rctime.strftime("%Y %m %d %H %M %S %w %j", (0,)*9) == exp
 
+        # Guard against invalid/non-supported format string
+        # so that Python don't crash (Windows crashes when the format string
+        # input to [w]strftime is not kosher.
+        if os.name =='nt':
+            raises(ValueError, rctime.strftime, '%f')
+        else:
+            assert rctime.strftime('%f') == '%f'
+
     def test_strftime_ext(self):
         import time as rctime
 

From noreply at buildbot.pypy.org  Thu Apr 12 00:13:45 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 12 Apr 2012 00:13:45 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix for translation
Message-ID: <20120411221345.D1BC382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54297:b0a1bc6e4750
Date: 2012-04-12 01:13 +0300
http://bitbucket.org/pypy/pypy/changeset/b0a1bc6e4750/

Log:	fix for translation

diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -24,6 +24,7 @@
 SIG_DFL = cpy_signal.SIG_DFL
 SIG_IGN = cpy_signal.SIG_IGN
 signal_names = list(setup())
+signal_values = [globals()[key] for key in signal_names]
 
 includes = ['stdlib.h', 'src/signals.h']
 if sys.platform != 'win32':
@@ -243,8 +244,9 @@
     return space.w_None
 
 def check_signum(space, signum):
-    for sig in signal_names:
-        if signum == globals()[sig]:
+    xxx
+    for sig in signal_values:
+        if signum ==sig:
             return
     raise OperationError(space.w_ValueError,
                          space.wrap("invalid signal value"))
diff --git a/pypy/module/signal/test/test_interp_signal.py b/pypy/module/signal/test/test_interp_signal.py
--- a/pypy/module/signal/test/test_interp_signal.py
+++ b/pypy/module/signal/test/test_interp_signal.py
@@ -6,6 +6,8 @@
 def setup_module(mod):
     if not hasattr(os, 'kill') or not hasattr(os, 'getpid'):
         py.test.skip("requires os.kill() and os.getpid()")
+    if not hasattr(interp_signal, 'SIGUSR1'):
+        py.test.skip("requires SIGUSR1 in signal")
 
 
 def check(expected):

From noreply at buildbot.pypy.org  Thu Apr 12 07:24:32 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 12 Apr 2012 07:24:32 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: remove debug cruft
Message-ID: <20120412052432.27D8582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54298:eb926aa445bc
Date: 2012-04-12 08:24 +0300
http://bitbucket.org/pypy/pypy/changeset/eb926aa445bc/

Log:	remove debug cruft

diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -244,7 +244,6 @@
     return space.w_None
 
 def check_signum(space, signum):
-    xxx
     for sig in signal_values:
         if signum ==sig:
             return

From noreply at buildbot.pypy.org  Thu Apr 12 07:27:33 2012
From: noreply at buildbot.pypy.org (taavi_burns)
Date: Thu, 12 Apr 2012 07:27:33 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-ufuncs3: Replaced logaddexp(2) functions
 with the numpy implementations. Tests pass better now!
Message-ID: <20120412052733.A85CF82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Taavi Burns <taavi.burns at gmail.com>
Branch: numpy-ufuncs3
Changeset: r54299:c644bd3dfefe
Date: 2012-04-12 01:27 -0400
http://bitbucket.org/pypy/pypy/changeset/c644bd3dfefe/

Log:	Replaced logaddexp(2) functions with the numpy implementations.
	Tests pass better now!

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -764,6 +764,7 @@
     def test_logaddexp(self):
         import math
         import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -774,8 +775,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(sys.float_info.max, sys.float_info.max) == sys.float_info.max
-        assert logaddexp(sys.float_info.min, sys.float_info.min) == math.log(2)
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -789,6 +790,7 @@
     def test_logaddexp2(self):
         import math
         import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -800,8 +802,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(sys.float_info.max, sys.float_info.max) == sys.float_info.max
-        assert logaddexp2(sys.float_info.min, sys.float_info.min) == 1.0
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True

From noreply at buildbot.pypy.org  Thu Apr 12 10:20:52 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 12 Apr 2012 10:20:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: issue1126: document
Message-ID: <20120412082052.B680682F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54300:2a7fffa8e63e
Date: 2012-04-12 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/2a7fffa8e63e/

Log:	issue1126: document

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -158,6 +158,12 @@
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
+Note that this difference might show up indirectly in some cases.  For
+example, a generator left pending in the middle is --- again ---
+garbage-collected later in PyPy than in CPython.  You can see the
+difference if the ``yield`` keyword it is suspended at is itself
+enclosed in a ``try:`` or a ``with:`` block.
+
 Using the default GC called ``minimark``, the built-in function ``id()``
 works like it does in CPython.  With other GCs it returns numbers that
 are not real addresses (because an object can move around several times)

From noreply at buildbot.pypy.org  Thu Apr 12 11:07:11 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 12 Apr 2012 11:07:11 +0200 (CEST)
Subject: [pypy-commit] pypy vendor/stdlib: create branch for tracking the
	unchanged stdlib
Message-ID: <20120412090711.9A89582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: vendor/stdlib
Changeset: r54301:7406215587c6
Date: 2012-04-12 11:00 +0200
http://bitbucket.org/pypy/pypy/changeset/7406215587c6/

Log:	create branch for tracking the unchanged stdlib


From noreply at buildbot.pypy.org  Thu Apr 12 11:07:23 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 12 Apr 2012 11:07:23 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: copy all pypy mods into the
	stdlib folder
Message-ID: <20120412090723.5D83882F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54302:cb0772450c59
Date: 2012-04-12 11:05 +0200
http://bitbucket.org/pypy/pypy/changeset/cb0772450c59/

Log:	copy all pypy mods into the stdlib folder

diff too long, truncating to 10000 out of 186682 lines

diff --git a/lib-python/2.7/UserDict.py b/lib-python/2.7/UserDict.py
--- a/lib-python/2.7/UserDict.py
+++ b/lib-python/2.7/UserDict.py
@@ -1,5 +1,10 @@
 """A more or less complete user-defined wrapper around dictionary objects."""
 
+# XXX This is a bit of a hack (as usual :-))
+# the actual content of the file is not changed, but we put it here to make
+# virtualenv happy (because its internal logic expects at least one of the
+# REQUIRED_MODULES to be in modified-*)
+
 class UserDict:
     def __init__(self, dict=None, **kwargs):
         self.data = {}
@@ -80,8 +85,12 @@
     def __iter__(self):
         return iter(self.data)
 
-import _abcoll
-_abcoll.MutableMapping.register(IterableUserDict)
+try:
+    import _abcoll
+except ImportError:
+    pass    # e.g. no '_weakref' module on this pypy
+else:
+    _abcoll.MutableMapping.register(IterableUserDict)
 
 
 class DictMixin:
diff --git a/lib-python/2.7/_threading_local.py b/lib-python/2.7/_threading_local.py
--- a/lib-python/2.7/_threading_local.py
+++ b/lib-python/2.7/_threading_local.py
@@ -155,7 +155,7 @@
         object.__setattr__(self, '_local__args', (args, kw))
         object.__setattr__(self, '_local__lock', RLock())
 
-        if (args or kw) and (cls.__init__ is object.__init__):
+        if (args or kw) and (cls.__init__ == object.__init__):
             raise TypeError("Initialization arguments are not supported")
 
         # We need to create the thread dict in anticipation of
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -7,6 +7,7 @@
 
 __version__ = "1.1.0"
 
+import _ffi
 from _ctypes import Union, Structure, Array
 from _ctypes import _Pointer
 from _ctypes import CFuncPtr as _CFuncPtr
@@ -350,16 +351,17 @@
         self._FuncPtr = _FuncPtr
 
         if handle is None:
-            self._handle = _dlopen(self._name, mode)
+            self._handle = _ffi.CDLL(name, mode)
         else:
             self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %x at %x>" % \
+        return "<%s '%s', handle %r at %x>" % \
                (self.__class__.__name__, self._name,
-                (self._handle & (_sys.maxint*2 + 1)),
+                (self._handle),
                 id(self) & (_sys.maxint*2 + 1))
 
+
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
             raise AttributeError(name)
@@ -487,9 +489,12 @@
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
-    return _cast(obj, obj, typ)
+    try:
+        c_void_p.from_param(obj)
+    except TypeError, e:
+        raise ArgumentError(str(e))
+    return _cast_addr(obj, obj, typ)
 
 _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
diff --git a/lib-python/2.7/ctypes/test/__init__.py b/lib-python/2.7/ctypes/test/__init__.py
--- a/lib-python/2.7/ctypes/test/__init__.py
+++ b/lib-python/2.7/ctypes/test/__init__.py
@@ -206,3 +206,16 @@
         result = unittest.TestResult()
         test(result)
         return result
+
+def xfail(method):
+    """
+    Poor's man xfail: remove it when all the failures have been fixed
+    """
+    def new_method(self, *args, **kwds):
+        try:
+            method(self, *args, **kwds)
+        except:
+            pass
+        else:
+            self.assertTrue(False, "DID NOT RAISE")
+    return new_method
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -1,12 +1,23 @@
 import unittest
 from ctypes import *
+from test.test_support import impl_detail
 
 formats = "bBhHiIlLqQfd"
 
+# c_longdouble commented out for PyPy, look at the commend in test_longdouble
 formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \
-          c_long, c_ulonglong, c_float, c_double, c_longdouble
+          c_long, c_ulonglong, c_float, c_double #, c_longdouble
 
 class ArrayTestCase(unittest.TestCase):
+
+    @impl_detail('long double not supported by PyPy', pypy=False)
+    def test_longdouble(self):
+        """
+        This test is empty. It's just here to remind that we commented out
+        c_longdouble in "formats". If pypy will ever supports c_longdouble, we
+        should kill this test and uncomment c_longdouble inside formats.
+        """
+
     def test_simple(self):
         # create classes holding simple numeric types, and check
         # various properties.
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -115,17 +115,21 @@
     def test_nonint_types(self):
         # bit fields are not allowed on non-integer types.
         result = self.fail_fields(("a", c_char_p, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         result = self.fail_fields(("a", c_void_p, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         if c_int != c_long:
             result = self.fail_fields(("a", POINTER(c_int), 1))
-            self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int'))
+            self.assertEqual(result[0], TypeError)
+            self.assertIn('bit fields not allowed for type', result[1])
 
         result = self.fail_fields(("a", c_char, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         try:
             c_wchar
@@ -133,13 +137,15 @@
             pass
         else:
             result = self.fail_fields(("a", c_wchar, 1))
-            self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_wchar'))
+            self.assertEqual(result[0], TypeError)
+            self.assertIn('bit fields not allowed for type', result[1])
 
         class Dummy(Structure):
             _fields_ = []
 
         result = self.fail_fields(("a", Dummy, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
     def test_single_bitfield_size(self):
         for c_typ in int_types:
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -2,6 +2,7 @@
 from binascii import hexlify
 
 from ctypes import *
+from ctypes.test import xfail
 
 def bin(s):
     return hexlify(memoryview(s)).upper()
@@ -21,6 +22,7 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
+    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertTrue(c_short.__ctype_le__ is c_short)
@@ -48,6 +50,7 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
+    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertTrue(c_int.__ctype_le__ is c_int)
@@ -76,6 +79,7 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
+    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertTrue(c_longlong.__ctype_le__ is c_longlong)
@@ -104,6 +108,7 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
+    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertTrue(c_float.__ctype_le__ is c_float)
@@ -122,6 +127,7 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
+    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertTrue(c_double.__ctype_le__ is c_double)
@@ -149,6 +155,7 @@
         self.assertTrue(c_char.__ctype_le__ is c_char)
         self.assertTrue(c_char.__ctype_be__ is c_char)
 
+    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -198,6 +205,7 @@
             pass
         self.assertRaises(TypeError, setattr, S, "_fields_", [("s", T)])
 
+    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -221,6 +229,7 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
+    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import xfail
 import _ctypes_test
 
 class Callbacks(unittest.TestCase):
@@ -98,6 +99,7 @@
 ##        self.check_type(c_char_p, "abc")
 ##        self.check_type(c_char_p, "def")
 
+    @xfail
     def test_pyobject(self):
         o = ()
         from sys import getrefcount as grc
diff --git a/lib-python/2.7/ctypes/test/test_cfuncs.py b/lib-python/2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/2.7/ctypes/test/test_cfuncs.py
@@ -3,8 +3,8 @@
 
 import unittest
 from ctypes import *
-
 import _ctypes_test
+from test.test_support import impl_detail
 
 class CFunctions(unittest.TestCase):
     _dll = CDLL(_ctypes_test.__file__)
@@ -158,12 +158,14 @@
         self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
         self.assertEqual(self.S(), 42)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble(self):
         self._dll.tf_D.restype = c_longdouble
         self._dll.tf_D.argtypes = (c_longdouble,)
         self.assertEqual(self._dll.tf_D(42.), 14.)
         self.assertEqual(self.S(), 42)
-
+        
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble_plus(self):
         self._dll.tf_bD.restype = c_longdouble
         self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
diff --git a/lib-python/2.7/ctypes/test/test_delattr.py b/lib-python/2.7/ctypes/test/test_delattr.py
--- a/lib-python/2.7/ctypes/test/test_delattr.py
+++ b/lib-python/2.7/ctypes/test/test_delattr.py
@@ -6,15 +6,15 @@
 
 class TestCase(unittest.TestCase):
     def test_simple(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, c_int(42), "value")
 
     def test_chararray(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, (c_char * 5)(), "value")
 
     def test_struct(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, X(), "foo")
 
 if __name__ == "__main__":
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,6 +2,7 @@
 import array
 import gc
 import unittest
+from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -10,6 +11,7 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
+    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -35,6 +37,7 @@
         self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
+    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -43,6 +46,7 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
+    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -67,6 +71,7 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
+    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py
--- a/lib-python/2.7/ctypes/test/test_functions.py
+++ b/lib-python/2.7/ctypes/test/test_functions.py
@@ -7,6 +7,8 @@
 
 from ctypes import *
 import sys, unittest
+from ctypes.test import xfail
+from test.test_support import impl_detail
 
 try:
     WINFUNCTYPE
@@ -143,6 +145,7 @@
         self.assertEqual(result, -21)
         self.assertEqual(type(result), float)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdoubleresult(self):
         f = dll._testfunc_D_bhilfD
         f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
@@ -393,6 +396,7 @@
             self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
                                  (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
 
+    @xfail
     def test_sf1651235(self):
         # see http://www.python.org/sf/1651235
 
diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py
--- a/lib-python/2.7/ctypes/test/test_internals.py
+++ b/lib-python/2.7/ctypes/test/test_internals.py
@@ -33,7 +33,13 @@
         refcnt = grc(s)
         cs = c_char_p(s)
         self.assertEqual(refcnt + 1, grc(s))
-        self.assertSame(cs._objects, s)
+        try:
+            # Moving gcs need to allocate a nonmoving buffer
+            cs._objects._obj
+        except AttributeError:
+            self.assertSame(cs._objects, s)
+        else:
+            self.assertSame(cs._objects._obj, s)
 
     def test_simple_struct(self):
         class X(Structure):
diff --git a/lib-python/2.7/ctypes/test/test_libc.py b/lib-python/2.7/ctypes/test/test_libc.py
--- a/lib-python/2.7/ctypes/test/test_libc.py
+++ b/lib-python/2.7/ctypes/test/test_libc.py
@@ -25,5 +25,14 @@
         lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort))
         self.assertEqual(chars.raw, "   ,,aaaadmmmnpppsss\x00")
 
+    def SKIPPED_test_no_more_xfail(self):
+        # We decided to not explicitly support the whole ctypes-2.7
+        # and instead go for a case-by-case, demand-driven approach.
+        # So this test is skipped instead of failing.
+        import socket
+        import ctypes.test
+        self.assertTrue(not hasattr(ctypes.test, 'xfail'),
+                        "You should incrementally grep for '@xfail' and remove them, they are real failures")
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_loading.py b/lib-python/2.7/ctypes/test/test_loading.py
--- a/lib-python/2.7/ctypes/test/test_loading.py
+++ b/lib-python/2.7/ctypes/test/test_loading.py
@@ -2,7 +2,7 @@
 import sys, unittest
 import os
 from ctypes.util import find_library
-from ctypes.test import is_resource_enabled
+from ctypes.test import is_resource_enabled, xfail
 
 libc_name = None
 if os.name == "nt":
@@ -75,6 +75,7 @@
             self.assertRaises(AttributeError, dll.__getitem__, 1234)
 
     if os.name == "nt":
+        @xfail
         def test_1703286_A(self):
             from _ctypes import LoadLibrary, FreeLibrary
             # On winXP 64-bit, advapi32 loads at an address that does
@@ -85,6 +86,7 @@
             handle = LoadLibrary("advapi32")
             FreeLibrary(handle)
 
+        @xfail
         def test_1703286_B(self):
             # Since on winXP 64-bit advapi32 loads like described
             # above, the (arbitrarily selected) CloseEventLog function
diff --git a/lib-python/2.7/ctypes/test/test_macholib.py b/lib-python/2.7/ctypes/test/test_macholib.py
--- a/lib-python/2.7/ctypes/test/test_macholib.py
+++ b/lib-python/2.7/ctypes/test/test_macholib.py
@@ -52,7 +52,6 @@
                                  '/usr/lib/libSystem.B.dylib')
 
             result = find_lib('z')
-            self.assertTrue(result.startswith('/usr/lib/libz.1'))
             self.assertTrue(result.endswith('.dylib'))
 
             self.assertEqual(find_lib('IOKit'),
diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py
--- a/lib-python/2.7/ctypes/test/test_numbers.py
+++ b/lib-python/2.7/ctypes/test/test_numbers.py
@@ -1,6 +1,7 @@
 from ctypes import *
 import unittest
 import struct
+from ctypes.test import xfail
 
 def valid_ranges(*types):
     # given a sequence of numeric types, collect their _type_
@@ -89,12 +90,14 @@
 ##            self.assertRaises(ValueError, t, l-1)
 ##            self.assertRaises(ValueError, t, h+1)
 
+    @xfail
     def test_from_param(self):
         # the from_param class method attribute always
         # returns PyCArgObject instances
         for t in signed_types + unsigned_types + float_types:
             self.assertEqual(ArgType, type(t.from_param(0)))
 
+    @xfail
     def test_byref(self):
         # calling byref returns also a PyCArgObject instance
         for t in signed_types + unsigned_types + float_types + bool_types:
@@ -102,6 +105,7 @@
             self.assertEqual(ArgType, type(parm))
 
 
+    @xfail
     def test_floats(self):
         # c_float and c_double can be created from
         # Python int, long and float
@@ -115,6 +119,7 @@
             self.assertEqual(t(2L).value, 2.0)
             self.assertEqual(t(f).value, 2.0)
 
+    @xfail
     def test_integers(self):
         class FloatLike(object):
             def __float__(self):
diff --git a/lib-python/2.7/ctypes/test/test_objects.py b/lib-python/2.7/ctypes/test/test_objects.py
--- a/lib-python/2.7/ctypes/test/test_objects.py
+++ b/lib-python/2.7/ctypes/test/test_objects.py
@@ -22,7 +22,7 @@
 
 >>> array[4] = 'foo bar'
 >>> array._objects
-{'4': 'foo bar'}
+{'4': <CArgObject 'foo bar'>}
 >>> array[4]
 'foo bar'
 >>>
@@ -47,9 +47,9 @@
 
 >>> x.array[0] = 'spam spam spam'
 >>> x._objects
-{'0:2': 'spam spam spam'}
+{'0:2': <CArgObject 'spam spam spam'>}
 >>> x.array._b_base_._objects
-{'0:2': 'spam spam spam'}
+{'0:2': <CArgObject 'spam spam spam'>}
 >>>
 
 '''
diff --git a/lib-python/2.7/ctypes/test/test_parameters.py b/lib-python/2.7/ctypes/test/test_parameters.py
--- a/lib-python/2.7/ctypes/test/test_parameters.py
+++ b/lib-python/2.7/ctypes/test/test_parameters.py
@@ -1,5 +1,7 @@
 import unittest, sys
 
+from ctypes.test import xfail
+
 class SimpleTypesTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -49,6 +51,7 @@
         self.assertEqual(CWCHARP.from_param("abc"), "abcabcabc")
 
     # XXX Replace by c_char_p tests
+    @xfail
     def test_cstrings(self):
         from ctypes import c_char_p, byref
 
@@ -86,7 +89,10 @@
 
         pa = c_wchar_p.from_param(c_wchar_p(u"123"))
         self.assertEqual(type(pa), c_wchar_p)
+    if sys.platform == "win32":
+        test_cw_strings = xfail(test_cw_strings)
 
+    @xfail
     def test_int_pointers(self):
         from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer
         LPINT = POINTER(c_int)
diff --git a/lib-python/2.7/ctypes/test/test_pep3118.py b/lib-python/2.7/ctypes/test/test_pep3118.py
--- a/lib-python/2.7/ctypes/test/test_pep3118.py
+++ b/lib-python/2.7/ctypes/test/test_pep3118.py
@@ -1,6 +1,7 @@
 import unittest
 from ctypes import *
 import re, sys
+from ctypes.test import xfail
 
 if sys.byteorder == "little":
     THIS_ENDIAN = "<"
@@ -19,6 +20,7 @@
 
 class Test(unittest.TestCase):
 
+    @xfail
     def test_native_types(self):
         for tp, fmt, shape, itemtp in native_types:
             ob = tp()
@@ -46,6 +48,7 @@
                 print(tp)
                 raise
 
+    @xfail
     def test_endian_types(self):
         for tp, fmt, shape, itemtp in endian_types:
             ob = tp()
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -3,6 +3,7 @@
 from ctypes import *
 import _ctypes_test
 dll = CDLL(_ctypes_test.__file__)
+from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("a", c_int), ("b", c_double)]
@@ -21,6 +22,7 @@
     def loads(self, item):
         return pickle.loads(item)
 
+    @xfail
     def test_simple(self):
         for src in [
             c_int(42),
@@ -31,6 +33,7 @@
             self.assertEqual(memoryview(src).tobytes(),
                                  memoryview(dst).tobytes())
 
+    @xfail
     def test_struct(self):
         X.init_called = 0
 
@@ -49,6 +52,7 @@
         self.assertEqual(memoryview(y).tobytes(),
                              memoryview(x).tobytes())
 
+    @xfail
     def test_unpickable(self):
         # ctypes objects that are pointers or contain pointers are
         # unpickable.
@@ -66,6 +70,7 @@
             ]:
             self.assertRaises(ValueError, lambda: self.dumps(item))
 
+    @xfail
     def test_wchar(self):
         pickle.dumps(c_char("x"))
         # Issue 5049
diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py
--- a/lib-python/2.7/ctypes/test/test_python_api.py
+++ b/lib-python/2.7/ctypes/test/test_python_api.py
@@ -1,6 +1,6 @@
 from ctypes import *
 import unittest, sys
-from ctypes.test import is_resource_enabled
+from ctypes.test import is_resource_enabled, xfail
 
 ################################################################
 # This section should be moved into ctypes\__init__.py, when it's ready.
@@ -17,6 +17,7 @@
 
 class PythonAPITestCase(unittest.TestCase):
 
+    @xfail
     def test_PyString_FromStringAndSize(self):
         PyString_FromStringAndSize = pythonapi.PyString_FromStringAndSize
 
@@ -25,6 +26,7 @@
 
         self.assertEqual(PyString_FromStringAndSize("abcdefghi", 3), "abc")
 
+    @xfail
     def test_PyString_FromString(self):
         pythonapi.PyString_FromString.restype = py_object
         pythonapi.PyString_FromString.argtypes = (c_char_p,)
@@ -56,6 +58,7 @@
             del res
             self.assertEqual(grc(42), ref42)
 
+    @xfail
     def test_PyObj_FromPtr(self):
         s = "abc def ghi jkl"
         ref = grc(s)
@@ -81,6 +84,7 @@
         # not enough arguments
         self.assertRaises(TypeError, PyOS_snprintf, buf)
 
+    @xfail
     def test_pyobject_repr(self):
         self.assertEqual(repr(py_object()), "py_object(<NULL>)")
         self.assertEqual(repr(py_object(42)), "py_object(42)")
diff --git a/lib-python/2.7/ctypes/test/test_refcounts.py b/lib-python/2.7/ctypes/test/test_refcounts.py
--- a/lib-python/2.7/ctypes/test/test_refcounts.py
+++ b/lib-python/2.7/ctypes/test/test_refcounts.py
@@ -90,6 +90,7 @@
             return a * b * 2
         f = proto(func)
 
+        gc.collect()
         a = sys.getrefcount(ctypes.c_int)
         f(1, 2)
         self.assertEqual(sys.getrefcount(ctypes.c_int), a)
diff --git a/lib-python/2.7/ctypes/test/test_stringptr.py b/lib-python/2.7/ctypes/test/test_stringptr.py
--- a/lib-python/2.7/ctypes/test/test_stringptr.py
+++ b/lib-python/2.7/ctypes/test/test_stringptr.py
@@ -2,11 +2,13 @@
 from ctypes import *
 
 import _ctypes_test
+from ctypes.test import xfail
 
 lib = CDLL(_ctypes_test.__file__)
 
 class StringPtrTestCase(unittest.TestCase):
 
+    @xfail
     def test__POINTER_c_char(self):
         class X(Structure):
             _fields_ = [("str", POINTER(c_char))]
@@ -27,6 +29,7 @@
 
         self.assertRaises(TypeError, setattr, x, "str", "Hello, World")
 
+    @xfail
     def test__c_char_p(self):
         class X(Structure):
             _fields_ = [("str", c_char_p)]
diff --git a/lib-python/2.7/ctypes/test/test_strings.py b/lib-python/2.7/ctypes/test/test_strings.py
--- a/lib-python/2.7/ctypes/test/test_strings.py
+++ b/lib-python/2.7/ctypes/test/test_strings.py
@@ -31,8 +31,9 @@
         buf.value = "Hello, World"
         self.assertEqual(buf.value, "Hello, World")
 
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("Hello, World"))
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
+        if test_support.check_impl_detail():
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("Hello, World"))
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
         self.assertRaises(ValueError, setattr, buf, "raw", memoryview("x" * 100))
 
     def test_c_buffer_raw(self, memoryview=memoryview):
@@ -40,7 +41,8 @@
 
         buf.raw = memoryview("Hello, World")
         self.assertEqual(buf.value, "Hello, World")
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
+        if test_support.check_impl_detail():
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
         self.assertRaises(ValueError, setattr, buf, "raw", memoryview("x" * 100))
 
     def test_c_buffer_deprecated(self):
diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py
--- a/lib-python/2.7/ctypes/test/test_structures.py
+++ b/lib-python/2.7/ctypes/test/test_structures.py
@@ -194,8 +194,8 @@
         self.assertEqual(X.b.offset, min(8, longlong_align))
 
 
-        d = {"_fields_": [("a", "b"),
-                          ("b", "q")],
+        d = {"_fields_": [("a", c_byte),
+                          ("b", c_longlong)],
              "_pack_": -1}
         self.assertRaises(ValueError, type(Structure), "X", (Structure,), d)
 
diff --git a/lib-python/2.7/ctypes/test/test_varsize_struct.py b/lib-python/2.7/ctypes/test/test_varsize_struct.py
--- a/lib-python/2.7/ctypes/test/test_varsize_struct.py
+++ b/lib-python/2.7/ctypes/test/test_varsize_struct.py
@@ -1,7 +1,9 @@
 from ctypes import *
 import unittest
+from ctypes.test import xfail
 
 class VarSizeTest(unittest.TestCase):
+    @xfail
     def test_resize(self):
         class X(Structure):
             _fields_ = [("item", c_int),
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -72,8 +72,8 @@
         return name
 
 if os.name == "posix" and sys.platform == "darwin":
-    from ctypes.macholib.dyld import dyld_find as _dyld_find
     def find_library(name):
+        from ctypes.macholib.dyld import dyld_find as _dyld_find
         possible = ['lib%s.dylib' % name,
                     '%s.dylib' % name,
                     '%s.framework/%s' % (name, name)]
diff --git a/lib-python/2.7/distutils/command/bdist_wininst.py b/lib-python/2.7/distutils/command/bdist_wininst.py
--- a/lib-python/2.7/distutils/command/bdist_wininst.py
+++ b/lib-python/2.7/distutils/command/bdist_wininst.py
@@ -298,7 +298,8 @@
                              bitmaplen,        # number of bytes in bitmap
                              )
         file.write(header)
-        file.write(open(arcname, "rb").read())
+        with open(arcname, "rb") as arcfile:
+            file.write(arcfile.read())
 
     # create_exe()
 
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -184,7 +184,7 @@
             # the 'libs' directory is for binary installs - we assume that
             # must be the *native* platform.  But we don't really support
             # cross-compiling via a binary install anyway, so we let it go.
-            self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs'))
+            self.library_dirs.append(os.path.join(sys.exec_prefix, 'include'))
             if self.debug:
                 self.build_temp = os.path.join(self.build_temp, "Debug")
             else:
@@ -192,8 +192,13 @@
 
             # Append the source distribution include and library directories,
             # this allows distutils on windows to work in the source tree
-            self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
-            if MSVC_VERSION == 9:
+            if 0:
+                # pypy has no PC directory
+                self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
+            if 1:
+                # pypy has no PCBuild directory
+                pass
+            elif MSVC_VERSION == 9:
                 # Use the .lib files for the correct architecture
                 if self.plat_name == 'win32':
                     suffix = ''
@@ -695,24 +700,14 @@
         shared extension.  On most platforms, this is just 'ext.libraries';
         on Windows and OS/2, we add the Python library (eg. python20.dll).
         """
-        # The python library is always needed on Windows.  For MSVC, this
-        # is redundant, since the library is mentioned in a pragma in
-        # pyconfig.h that MSVC groks.  The other Windows compilers all seem
-        # to need it mentioned explicitly, though, so that's what we do.
-        # Append '_d' to the python import library on debug builds.
+        # The python library is always needed on Windows.
         if sys.platform == "win32":
-            from distutils.msvccompiler import MSVCCompiler
-            if not isinstance(self.compiler, MSVCCompiler):
-                template = "python%d%d"
-                if self.debug:
-                    template = template + '_d'
-                pythonlib = (template %
-                       (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
-                # don't extend ext.libraries, it may be shared with other
-                # extensions, it is a reference to the original list
-                return ext.libraries + [pythonlib]
-            else:
-                return ext.libraries
+            template = "python%d%d"
+            pythonlib = (template %
+                   (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
+            # don't extend ext.libraries, it may be shared with other
+            # extensions, it is a reference to the original list
+            return ext.libraries + [pythonlib]
         elif sys.platform == "os2emx":
             # EMX/GCC requires the python library explicitly, and I
             # believe VACPP does as well (though not confirmed) - AIM Apr01
diff --git a/lib-python/2.7/distutils/command/install.py b/lib-python/2.7/distutils/command/install.py
--- a/lib-python/2.7/distutils/command/install.py
+++ b/lib-python/2.7/distutils/command/install.py
@@ -83,6 +83,13 @@
         'scripts': '$userbase/bin',
         'data'   : '$userbase',
         },
+    'pypy': {
+        'purelib': '$base/site-packages',
+        'platlib': '$base/site-packages',
+        'headers': '$base/include',
+        'scripts': '$base/bin',
+        'data'   : '$base',
+        },
     }
 
 # The keys to an installation scheme; if any new types of files are to be
@@ -467,6 +474,8 @@
 
     def select_scheme (self, name):
         # it's the caller's problem if they supply a bad name!
+        if hasattr(sys, 'pypy_version_info'):
+            name = 'pypy'
         scheme = INSTALL_SCHEMES[name]
         for key in SCHEME_KEYS:
             attrname = 'install_' + key
diff --git a/lib-python/2.7/distutils/cygwinccompiler.py b/lib-python/2.7/distutils/cygwinccompiler.py
--- a/lib-python/2.7/distutils/cygwinccompiler.py
+++ b/lib-python/2.7/distutils/cygwinccompiler.py
@@ -75,6 +75,9 @@
         elif msc_ver == '1500':
             # VS2008 / MSVC 9.0
             return ['msvcr90']
+        elif msc_ver == '1600':
+            # VS2010 / MSVC 10.0
+            return ['msvcr100']
         else:
             raise ValueError("Unknown MS Compiler version %s " % msc_ver)
 
diff --git a/lib-python/2.7/distutils/msvc9compiler.py b/lib-python/2.7/distutils/msvc9compiler.py
--- a/lib-python/2.7/distutils/msvc9compiler.py
+++ b/lib-python/2.7/distutils/msvc9compiler.py
@@ -648,6 +648,7 @@
             temp_manifest = os.path.join(
                     build_temp,
                     os.path.basename(output_filename) + ".manifest")
+            ld_args.append('/MANIFEST')
             ld_args.append('/MANIFESTFILE:' + temp_manifest)
 
             if extra_preargs:
diff --git a/lib-python/2.7/distutils/spawn.py b/lib-python/2.7/distutils/spawn.py
--- a/lib-python/2.7/distutils/spawn.py
+++ b/lib-python/2.7/distutils/spawn.py
@@ -58,7 +58,6 @@
 
 def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0):
     executable = cmd[0]
-    cmd = _nt_quote_args(cmd)
     if search_path:
         # either we find one or it stays the same
         executable = find_executable(executable) or executable
@@ -66,7 +65,8 @@
     if not dry_run:
         # spawn for NT requires a full path to the .exe
         try:
-            rc = os.spawnv(os.P_WAIT, executable, cmd)
+            import subprocess
+            rc = subprocess.call(cmd)
         except OSError, exc:
             # this seems to happen when the command isn't found
             raise DistutilsExecError, \
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -9,563 +9,21 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
-import os
-import re
-import string
 import sys
 
-from distutils.errors import DistutilsPlatformError
 
-# These are needed in a couple of spots, so just compute them once.
-PREFIX = os.path.normpath(sys.prefix)
-EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+# The content of this file is redirected from
+# sysconfig_cpython or sysconfig_pypy.
 
-# Path to the base directory of the project. On Windows the binary may
-# live in project/PCBuild9.  If we're dealing with an x64 Windows build,
-# it'll live in project/PCbuild/amd64.
-project_base = os.path.dirname(os.path.abspath(sys.executable))
-if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
-# PC/VS7.1
-if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
-# PC/AMD64
-if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
+if '__pypy__' in sys.builtin_module_names:
+    from distutils.sysconfig_pypy import *
+    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
+    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+else:
+    from distutils.sysconfig_cpython import *
+    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
+    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
 
-# python_build: (Boolean) if true, we're either building Python or
-# building an extension with an un-installed Python, so we use
-# different (hard-wired) directories.
-# Setup.local is available for Makefile builds including VPATH builds,
-# Setup.dist is available on Windows
-def _python_build():
-    for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
-            return True
-    return False
-python_build = _python_build()
 
-
-def get_python_version():
-    """Return a string containing the major and minor Python version,
-    leaving off the patchlevel.  Sample return values could be '1.5'
-    or '2.2'.
-    """
-    return sys.version[:3]
-
-
-def get_python_inc(plat_specific=0, prefix=None):
-    """Return the directory containing installed Python header files.
-
-    If 'plat_specific' is false (the default), this is the path to the
-    non-platform-specific header files, i.e. Python.h and so on;
-    otherwise, this is the path to platform-specific header files
-    (namely pyconfig.h).
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        if python_build:
-            buildir = os.path.dirname(sys.executable)
-            if plat_specific:
-                # python.h is located in the buildir
-                inc_dir = buildir
-            else:
-                # the source dir is relative to the buildir
-                srcdir = os.path.abspath(os.path.join(buildir,
-                                         get_config_var('srcdir')))
-                # Include is located in the srcdir
-                inc_dir = os.path.join(srcdir, "Include")
-            return inc_dir
-        return os.path.join(prefix, "include", "python" + get_python_version())
-    elif os.name == "nt":
-        return os.path.join(prefix, "include")
-    elif os.name == "os2":
-        return os.path.join(prefix, "Include")
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its C header files "
-            "on platform '%s'" % os.name)
-
-
-def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
-    """Return the directory containing the Python library (standard or
-    site additions).
-
-    If 'plat_specific' is true, return the directory containing
-    platform-specific modules, i.e. any module from a non-pure-Python
-    module distribution; otherwise, return the platform-shared library
-    directory.  If 'standard_lib' is true, return the directory
-    containing standard Python library modules; otherwise, return the
-    directory for site-specific modules.
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
-        if standard_lib:
-            return libpython
-        else:
-            return os.path.join(libpython, "site-packages")
-
-    elif os.name == "nt":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            if get_python_version() < "2.2":
-                return prefix
-            else:
-                return os.path.join(prefix, "Lib", "site-packages")
-
-    elif os.name == "os2":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            return os.path.join(prefix, "Lib", "site-packages")
-
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its library "
-            "on platform '%s'" % os.name)
-
-
-def customize_compiler(compiler):
-    """Do any platform-specific customization of a CCompiler instance.
-
-    Mainly needed on Unix, so we can plug in the information that
-    varies across Unices and is stored in Python's Makefile.
-    """
-    if compiler.compiler_type == "unix":
-        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext) = \
-            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SO')
-
-        if 'CC' in os.environ:
-            cc = os.environ['CC']
-        if 'CXX' in os.environ:
-            cxx = os.environ['CXX']
-        if 'LDSHARED' in os.environ:
-            ldshared = os.environ['LDSHARED']
-        if 'CPP' in os.environ:
-            cpp = os.environ['CPP']
-        else:
-            cpp = cc + " -E"           # not always
-        if 'LDFLAGS' in os.environ:
-            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        if 'CPPFLAGS' in os.environ:
-            cpp = cpp + ' ' + os.environ['CPPFLAGS']
-            cflags = cflags + ' ' + os.environ['CPPFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-
-        cc_cmd = cc + ' ' + cflags
-        compiler.set_executables(
-            preprocessor=cpp,
-            compiler=cc_cmd,
-            compiler_so=cc_cmd + ' ' + ccshared,
-            compiler_cxx=cxx,
-            linker_so=ldshared,
-            linker_exe=cc)
-
-        compiler.shared_lib_extension = so_ext
-
-
-def get_config_h_filename():
-    """Return full pathname of installed pyconfig.h file."""
-    if python_build:
-        if os.name == "nt":
-            inc_dir = os.path.join(project_base, "PC")
-        else:
-            inc_dir = project_base
-    else:
-        inc_dir = get_python_inc(plat_specific=1)
-    if get_python_version() < '2.2':
-        config_h = 'config.h'
-    else:
-        # The name of the config.h file changed in 2.2
-        config_h = 'pyconfig.h'
-    return os.path.join(inc_dir, config_h)
-
-
-def get_makefile_filename():
-    """Return full pathname of installed Makefile from the Python build."""
-    if python_build:
-        return os.path.join(os.path.dirname(sys.executable), "Makefile")
-    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
-    return os.path.join(lib_dir, "config", "Makefile")
-
-
-def parse_config_h(fp, g=None):
-    """Parse a config.h-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    if g is None:
-        g = {}
-    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
-    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
-    #
-    while 1:
-        line = fp.readline()
-        if not line:
-            break
-        m = define_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            try: v = int(v)
-            except ValueError: pass
-            g[n] = v
-        else:
-            m = undef_rx.match(line)
-            if m:
-                g[m.group(1)] = 0
-    return g
-
-
-# Regexes needed for parsing Makefile (and similar syntaxes,
-# like old-style Setup files).
-_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-def parse_makefile(fn, g=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    from distutils.text_file import TextFile
-    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)
-
-    if g is None:
-        g = {}
-    done = {}
-    notdone = {}
-
-    while 1:
-        line = fp.readline()
-        if line is None:  # eof
-            break
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-
-    fp.close()
-
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    g.update(done)
-    return g
-
-
-def expand_makefile_vars(s, vars):
-    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
-    'string' according to 'vars' (a dictionary mapping variable names to
-    values).  Variables not present in 'vars' are silently expanded to the
-    empty string.  The variable values in 'vars' should not contain further
-    variable expansions; if 'vars' is the output of 'parse_makefile()',
-    you're fine.  Returns a variable-expanded version of 's'.
-    """
-
-    # This algorithm does multiple expansion, so if vars['foo'] contains
-    # "${bar}", it will expand ${foo} to ${bar}, and then expand
-    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
-    # 'parse_makefile()', which takes care of such expansions eagerly,
-    # according to make's variable expansion semantics.
-
-    while 1:
-        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
-        if m:
-            (beg, end) = m.span()
-            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
-        else:
-            break
-    return s
-
-
-_config_vars = None
-
-def _init_posix():
-    """Initialize the module as appropriate for POSIX systems."""
-    g = {}
-    # load the installed Makefile:
-    try:
-        filename = get_makefile_filename()
-        parse_makefile(filename, g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # load the installed pyconfig.h:
-    try:
-        filename = get_config_h_filename()
-        parse_config_h(file(filename), g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # On MacOSX we need to check the setting of the environment variable
-    # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
-    # it needs to be compatible.
-    # If it isn't set we set it to the configure-time value
-    if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g:
-        cfg_target = g['MACOSX_DEPLOYMENT_TARGET']
-        cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
-        if cur_target == '':
-            cur_target = cfg_target
-            os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target
-        elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
-            my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
-                % (cur_target, cfg_target))
-            raise DistutilsPlatformError(my_msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if python_build:
-        g['LDSHARED'] = g['BLDSHARED']
-
-    elif get_python_version() < '2.1':
-        # The following two branches are for 1.5.2 compatibility.
-        if sys.platform == 'aix4':          # what about AIX 3.x ?
-            # Linker script is in the config directory, not in Modules as the
-            # Makefile says.
-            python_lib = get_python_lib(standard_lib=1)
-            ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
-            python_exp = os.path.join(python_lib, 'config', 'python.exp')
-
-            g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
-
-        elif sys.platform == 'beos':
-            # Linker script is in the config directory.  In the Makefile it is
-            # relative to the srcdir, which after installation no longer makes
-            # sense.
-            python_lib = get_python_lib(standard_lib=1)
-            linkerscript_path = string.split(g['LDSHARED'])[0]
-            linkerscript_name = os.path.basename(linkerscript_path)
-            linkerscript = os.path.join(python_lib, 'config',
-                                        linkerscript_name)
-
-            # XXX this isn't the right place to do this: adding the Python
-            # library to the link, if needed, should be in the "build_ext"
-            # command.  (It's also needed for non-MS compilers on Windows, and
-            # it's taken care of for them by the 'build_ext.get_libraries()'
-            # method.)
-            g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
-                             (linkerscript, PREFIX, get_python_version()))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_nt():
-    """Initialize the module as appropriate for NT"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-    g['VERSION'] = get_python_version().replace(".", "")
-    g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_os2():
-    """Initialize the module as appropriate for OS/2"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-
-    global _config_vars
-    _config_vars = g
-
-
-def get_config_vars(*args):
-    """With no arguments, return a dictionary of all configuration
-    variables relevant for the current platform.  Generally this includes
-    everything needed to build extensions and install both pure modules and
-    extensions.  On Unix, this means every variable defined in Python's
-    installed Makefile; on Windows and Mac OS it's a much smaller set.
-
-    With arguments, return a list of values that result from looking up
-    each argument in the configuration variable dictionary.
-    """
-    global _config_vars
-    if _config_vars is None:
-        func = globals().get("_init_" + os.name)
-        if func:
-            func()
-        else:
-            _config_vars = {}
-
-        # Normalized versions of prefix and exec_prefix are handy to have;
-        # in fact, these are the standard versions used most places in the
-        # Distutils.
-        _config_vars['prefix'] = PREFIX
-        _config_vars['exec_prefix'] = EXEC_PREFIX
-
-        if sys.platform == 'darwin':
-            kernel_version = os.uname()[2] # Kernel version (8.4.3)
-            major_version = int(kernel_version.split('.')[0])
-
-            if major_version < 8:
-                # On Mac OS X before 10.4, check if -arch and -isysroot
-                # are in CFLAGS or LDFLAGS and remove them if they are.
-                # This is needed when building extensions on a 10.3 system
-                # using a universal build of python.
-                for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-                    flags = _config_vars[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
-                    _config_vars[key] = flags
-
-            else:
-
-                # Allow the user to override the architecture flags using
-                # an environment variable.
-                # NOTE: This name was introduced by Apple in OSX 10.5 and
-                # is used by several scripting languages distributed with
-                # that OS release.
-
-                if 'ARCHFLAGS' in os.environ:
-                    arch = os.environ['ARCHFLAGS']
-                    for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                        flags = _config_vars[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _config_vars[key] = flags
-
-                # If we're on OSX 10.5 or later and the user tries to
-                # compiles an extension using an SDK that is not present
-                # on the current machine it is better to not use an SDK
-                # than to fail.
-                #
-                # The major usecase for this is users using a Python.org
-                # binary installer  on OSX 10.6: that installer uses
-                # the 10.4u SDK, but that SDK is not installed by default
-                # when you install Xcode.
-                #
-                m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS'])
-                if m is not None:
-                    sdk = m.group(1)
-                    if not os.path.exists(sdk):
-                        for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                             # a number of derived variables. These need to be
-                             # patched up as well.
-                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                            flags = _config_vars[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
-                            _config_vars[key] = flags
-
-    if args:
-        vals = []
-        for name in args:
-            vals.append(_config_vars.get(name))
-        return vals
-    else:
-        return _config_vars
-
-def get_config_var(name):
-    """Return the value of a single variable using the dictionary
-    returned by 'get_config_vars()'.  Equivalent to
-    get_config_vars().get(name)
-    """
-    return get_config_vars().get(name)
diff --git a/lib-python/modified-2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
rename from lib-python/modified-2.7/distutils/sysconfig_cpython.py
rename to lib-python/2.7/distutils/sysconfig_cpython.py
diff --git a/lib-python/modified-2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
rename from lib-python/modified-2.7/distutils/sysconfig_pypy.py
rename to lib-python/2.7/distutils/sysconfig_pypy.py
diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py
--- a/lib-python/2.7/distutils/tests/test_build_ext.py
+++ b/lib-python/2.7/distutils/tests/test_build_ext.py
@@ -293,7 +293,7 @@
         finally:
             os.chdir(old_wd)
         self.assertTrue(os.path.exists(so_file))
-        self.assertEqual(os.path.splitext(so_file)[-1],
+        self.assertEqual(so_file[so_file.index(os.path.extsep):],
                          sysconfig.get_config_var('SO'))
         so_dir = os.path.dirname(so_file)
         self.assertEqual(so_dir, other_tmp_dir)
@@ -302,7 +302,7 @@
         cmd.run()
         so_file = cmd.get_outputs()[0]
         self.assertTrue(os.path.exists(so_file))
-        self.assertEqual(os.path.splitext(so_file)[-1],
+        self.assertEqual(so_file[so_file.index(os.path.extsep):],
                          sysconfig.get_config_var('SO'))
         so_dir = os.path.dirname(so_file)
         self.assertEqual(so_dir, cmd.build_lib)
diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py
--- a/lib-python/2.7/distutils/tests/test_install.py
+++ b/lib-python/2.7/distutils/tests/test_install.py
@@ -2,6 +2,7 @@
 
 import os
 import unittest
+from test import test_support
 
 from test.test_support import run_unittest
 
@@ -40,14 +41,15 @@
             expected = os.path.normpath(expected)
             self.assertEqual(got, expected)
 
-        libdir = os.path.join(destination, "lib", "python")
-        check_path(cmd.install_lib, libdir)
-        check_path(cmd.install_platlib, libdir)
-        check_path(cmd.install_purelib, libdir)
-        check_path(cmd.install_headers,
-                   os.path.join(destination, "include", "python", "foopkg"))
-        check_path(cmd.install_scripts, os.path.join(destination, "bin"))
-        check_path(cmd.install_data, destination)
+        if test_support.check_impl_detail():
+            libdir = os.path.join(destination, "lib", "python")
+            check_path(cmd.install_lib, libdir)
+            check_path(cmd.install_platlib, libdir)
+            check_path(cmd.install_purelib, libdir)
+            check_path(cmd.install_headers,
+                       os.path.join(destination, "include", "python", "foopkg"))
+            check_path(cmd.install_scripts, os.path.join(destination, "bin"))
+            check_path(cmd.install_data, destination)
 
 
 def test_suite():
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -125,7 +125,22 @@
                   }
 
     if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
         executables['ranlib'] = ["ranlib"]
+        executables['linker_so'] += ['-undefined', 'dynamic_lookup']
+
+        for k, v in executables.iteritems():
+            if v and v[0] == 'cc':
+                v += ['-arch', arch]
+
 
     # Needed for the filename generation methods provided by the base
     # class, CCompiler.  NB. whoever instantiates/uses a particular
@@ -309,7 +324,7 @@
             # On OSX users can specify an alternate SDK using
             # '-isysroot', calculate the SDK root if it is specified
             # (and use it further on)
-            cflags = sysconfig.get_config_var('CFLAGS')
+            cflags = sysconfig.get_config_var('CFLAGS') or ''
             m = re.search(r'-isysroot\s+(\S+)', cflags)
             if m is None:
                 sysroot = '/'
diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py
--- a/lib-python/2.7/heapq.py
+++ b/lib-python/2.7/heapq.py
@@ -193,6 +193,8 @@
 
     Equivalent to:  sorted(iterable, reverse=True)[:n]
     """
+    if n < 0: # for consistency with the c impl
+        return []
     it = iter(iterable)
     result = list(islice(it, n))
     if not result:
@@ -209,6 +211,8 @@
 
     Equivalent to:  sorted(iterable)[:n]
     """
+    if n < 0: # for consistency with the c impl
+        return []
     if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
         # For smaller values of n, the bisect method is faster than a minheap.
         # It is also memory efficient, consuming only n elements of space.
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -1024,7 +1024,11 @@
             kwds["buffering"] = True;
         response = self.response_class(*args, **kwds)
 
-        response.begin()
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
         assert response.will_close != _UNKNOWN
         self.__state = _CS_IDLE
 
diff --git a/lib-python/2.7/idlelib/Delegator.py b/lib-python/2.7/idlelib/Delegator.py
--- a/lib-python/2.7/idlelib/Delegator.py
+++ b/lib-python/2.7/idlelib/Delegator.py
@@ -12,6 +12,14 @@
         self.__cache[name] = attr
         return attr
 
+    def __nonzero__(self):
+        # this is needed for PyPy: else, if self.delegate is None, the
+        # __getattr__ above picks NoneType.__nonzero__, which returns
+        # False. Thus, bool(Delegator()) is False as well, but it's not what
+        # we want.  On CPython, bool(Delegator()) is True because NoneType
+        # does not have __nonzero__
+        return True
+
     def resetcache(self):
         for key in self.__cache.keys():
             try:
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -746,8 +746,15 @@
     'varargs' and 'varkw' are the names of the * and ** arguments or None."""
 
     if not iscode(co):
-        raise TypeError('{!r} is not a code object'.format(co))
+        if hasattr(len, 'func_code') and type(co) is type(len.func_code):
+            # PyPy extension: built-in function objects have a func_code too.
+            # There is no co_code on it, but co_argcount and co_varnames and
+            # co_flags are present.
+            pass
+        else:
+            raise TypeError('{!r} is not a code object'.format(co))
 
+    code = getattr(co, 'co_code', '')
     nargs = co.co_argcount
     names = co.co_varnames
     args = list(names[:nargs])
@@ -757,12 +764,12 @@
     for i in range(nargs):
         if args[i][:1] in ('', '.'):
             stack, remain, count = [], [], []
-            while step < len(co.co_code):
-                op = ord(co.co_code[step])
+            while step < len(code):
+                op = ord(code[step])
                 step = step + 1
                 if op >= dis.HAVE_ARGUMENT:
                     opname = dis.opname[op]
-                    value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
+                    value = ord(code[step]) + ord(code[step+1])*256
                     step = step + 2
                     if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
                         remain.append(value)
@@ -809,7 +816,9 @@
 
     if ismethod(func):
         func = func.im_func
-    if not isfunction(func):
+    if not (isfunction(func) or
+            isbuiltin(func) and hasattr(func, 'func_code')):
+            # PyPy extension: this works for built-in functions too
         raise TypeError('{!r} is not a Python function'.format(func))
     args, varargs, varkw = getargs(func.func_code)
     return ArgSpec(args, varargs, varkw, func.func_defaults)
@@ -949,7 +958,7 @@
                 raise TypeError('%s() takes exactly 0 arguments '
                                 '(%d given)' % (f_name, num_total))
         else:
-            raise TypeError('%s() takes no arguments (%d given)' %
+            raise TypeError('%s() takes no argument (%d given)' %
                             (f_name, num_total))
     for arg in args:
         if isinstance(arg, str) and arg in named:
diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py
--- a/lib-python/2.7/json/encoder.py
+++ b/lib-python/2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,23 +17,22 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
 FLOAT_REPR = repr
 
-def encode_basestring(s):
+def raw_encode_basestring(s):
     """Return a JSON representation of a Python string
 
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
+encode_basestring = lambda s: '"' + raw_encode_basestring(s) + '"'
 
-
-def py_encode_basestring_ascii(s):
+def raw_encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,21 +45,19 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+encode_basestring_ascii = lambda s: '"' + raw_encode_basestring_ascii(s) + '"'
 
 
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
-
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
 
@@ -147,6 +137,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = raw_encode_basestring_ascii
+        else:
+            self.encoder = raw_encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +185,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +320,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and self.indent is None and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +375,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +385,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +431,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +440,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +448,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +461,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +492,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/2.7/json/tests/test_unicode.py b/lib-python/2.7/json/tests/test_unicode.py
--- a/lib-python/2.7/json/tests/test_unicode.py
+++ b/lib-python/2.7/json/tests/test_unicode.py
@@ -80,6 +80,12 @@
         # Issue 10038.
         self.assertEqual(type(self.loads('"foo"')), unicode)
 
+    def test_encode_not_utf_8(self):
+        self.assertEqual(self.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(self.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
+
 
 class TestPyUnicode(TestUnicode, PyTest): pass
 class TestCUnicode(TestUnicode, CTest): pass
diff --git a/lib-python/2.7/multiprocessing/forking.py b/lib-python/2.7/multiprocessing/forking.py
--- a/lib-python/2.7/multiprocessing/forking.py
+++ b/lib-python/2.7/multiprocessing/forking.py
@@ -73,15 +73,12 @@
         return getattr, (m.im_self, m.im_func.func_name)
 ForkingPickler.register(type(ForkingPickler.save), _reduce_method)
 
-def _reduce_method_descriptor(m):
-    return getattr, (m.__objclass__, m.__name__)
-ForkingPickler.register(type(list.append), _reduce_method_descriptor)
-ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
-
-#def _reduce_builtin_function_or_method(m):
-#    return getattr, (m.__self__, m.__name__)
-#ForkingPickler.register(type(list().append), _reduce_builtin_function_or_method)
-#ForkingPickler.register(type(int().__add__), _reduce_builtin_function_or_method)
+if type(list.append) is not type(ForkingPickler.save):
+    # Some python implementations have unbound methods even for builtin types
+    def _reduce_method_descriptor(m):
+        return getattr, (m.__objclass__, m.__name__)
+    ForkingPickler.register(type(list.append), _reduce_method_descriptor)
+    ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
 
 try:
     from functools import partial
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -1,4 +1,3 @@
-
 """
 opcode module - potentially shared between dis and other modules which
 operate on bytecodes (e.g. peephole optimizers).
@@ -189,4 +188,10 @@
 def_op('SET_ADD', 146)
 def_op('MAP_ADD', 147)
 
+# pypy modification, experimental bytecode
+def_op('LOOKUP_METHOD', 201)          # Index in name list
+hasname.append(201)
+def_op('CALL_METHOD', 202)            # #args not including 'self'
+def_op('BUILD_LIST_FROM_ARG', 203)
+
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/pickle.py b/lib-python/2.7/pickle.py
--- a/lib-python/2.7/pickle.py
+++ b/lib-python/2.7/pickle.py
@@ -168,7 +168,7 @@
 
 # Pickling machinery
 
-class Pickler:
+class Pickler(object):
 
     def __init__(self, file, protocol=None):
         """This takes a file-like object for writing a pickle data stream.
@@ -638,6 +638,10 @@
             # else tmp is empty, and we're done
 
     def save_dict(self, obj):
+        modict_saver = self._pickle_moduledict(obj)
+        if modict_saver is not None:
+            return self.save_reduce(*modict_saver)
+
         write = self.write
 
         if self.bin:
@@ -687,6 +691,29 @@
                 write(SETITEM)
             # else tmp is empty, and we're done
 
+    def _pickle_moduledict(self, obj):
+        # save module dictionary as "getattr(module, '__dict__')"
+
+        # build index of module dictionaries
+        try:
+            modict = self.module_dict_ids
+        except AttributeError:
+            modict = {}
+            from sys import modules
+            for mod in modules.values():
+                if isinstance(mod, ModuleType):
+                    modict[id(mod.__dict__)] = mod
+            self.module_dict_ids = modict
+
+        thisid = id(obj)
+        try:
+            themodule = modict[thisid]
+        except KeyError:
+            return None
+        from __builtin__ import getattr
+        return getattr, (themodule, '__dict__')
+
+
     def save_inst(self, obj):
         cls = obj.__class__
 
@@ -727,6 +754,29 @@
 
     dispatch[InstanceType] = save_inst
 
+    def save_function(self, obj):
+        try:
+            return self.save_global(obj)
+        except PicklingError, e:
+            pass
+        # Check copy_reg.dispatch_table
+        reduce = dispatch_table.get(type(obj))
+        if reduce:
+            rv = reduce(obj)
+        else:
+            # Check for a __reduce_ex__ method, fall back to __reduce__
+            reduce = getattr(obj, "__reduce_ex__", None)
+            if reduce:
+                rv = reduce(self.proto)
+            else:
+                reduce = getattr(obj, "__reduce__", None)
+                if reduce:
+                    rv = reduce()
+                else:
+                    raise e
+        return self.save_reduce(obj=obj, *rv)
+    dispatch[FunctionType] = save_function
+
     def save_global(self, obj, name=None, pack=struct.pack):
         write = self.write
         memo = self.memo
@@ -768,7 +818,6 @@
         self.memoize(obj)
 
     dispatch[ClassType] = save_global
-    dispatch[FunctionType] = save_global
     dispatch[BuiltinFunctionType] = save_global
     dispatch[TypeType] = save_global
 
@@ -824,7 +873,7 @@
 
 # Unpickling machinery
 
-class Unpickler:
+class Unpickler(object):
 
     def __init__(self, file):
         """This takes a file-like object for reading a pickle data stream.
diff --git a/lib-python/2.7/pkgutil.py b/lib-python/2.7/pkgutil.py
--- a/lib-python/2.7/pkgutil.py
+++ b/lib-python/2.7/pkgutil.py
@@ -244,7 +244,8 @@
         return mod
 
     def get_data(self, pathname):
-        return open(pathname, "rb").read()
+        with open(pathname, "rb") as f:
+            return f.read()
 
     def _reopen(self):
         if self.file and self.file.closed:
diff --git a/lib-python/2.7/pprint.py b/lib-python/2.7/pprint.py
--- a/lib-python/2.7/pprint.py
+++ b/lib-python/2.7/pprint.py
@@ -144,7 +144,7 @@
             return
 
         r = getattr(typ, "__repr__", None)
-        if issubclass(typ, dict) and r is dict.__repr__:
+        if issubclass(typ, dict) and r == dict.__repr__:
             write('{')
             if self._indent_per_level > 1:
                 write((self._indent_per_level - 1) * ' ')
@@ -173,10 +173,10 @@
             write('}')
             return
 
-        if ((issubclass(typ, list) and r is list.__repr__) or
-            (issubclass(typ, tuple) and r is tuple.__repr__) or
-            (issubclass(typ, set) and r is set.__repr__) or
-            (issubclass(typ, frozenset) and r is frozenset.__repr__)
+        if ((issubclass(typ, list) and r == list.__repr__) or
+            (issubclass(typ, tuple) and r == tuple.__repr__) or
+            (issubclass(typ, set) and r == set.__repr__) or
+            (issubclass(typ, frozenset) and r == frozenset.__repr__)
            ):
             length = _len(object)
             if issubclass(typ, list):
@@ -266,7 +266,7 @@
         return ("%s%s%s" % (closure, sio.getvalue(), closure)), True, False
 
     r = getattr(typ, "__repr__", None)
-    if issubclass(typ, dict) and r is dict.__repr__:
+    if issubclass(typ, dict) and r == dict.__repr__:
         if not object:
             return "{}", True, False
         objid = _id(object)
@@ -291,8 +291,8 @@
         del context[objid]
         return "{%s}" % _commajoin(components), readable, recursive
 
-    if (issubclass(typ, list) and r is list.__repr__) or \
-       (issubclass(typ, tuple) and r is tuple.__repr__):
+    if (issubclass(typ, list) and r == list.__repr__) or \
+       (issubclass(typ, tuple) and r == tuple.__repr__):
         if issubclass(typ, list):
             if not object:
                 return "[]", True, False
diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py
--- a/lib-python/2.7/pydoc.py
+++ b/lib-python/2.7/pydoc.py
@@ -623,7 +623,9 @@
             head, '#ffffff', '#7799ee',
             '<a href=".">index</a><br>' + filelink + docloc)
 
-        modules = inspect.getmembers(object, inspect.ismodule)
+        def isnonbuiltinmodule(obj):
+            return inspect.ismodule(obj) and obj is not __builtin__
+        modules = inspect.getmembers(object, isnonbuiltinmodule)
 
         classes, cdict = [], {}
         for key, value in inspect.getmembers(object, inspect.isclass):
diff --git a/lib-python/2.7/random.py b/lib-python/2.7/random.py
--- a/lib-python/2.7/random.py
+++ b/lib-python/2.7/random.py
@@ -41,7 +41,6 @@
 
 from __future__ import division
 from warnings import warn as _warn
-from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethodType
 from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil
 from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin
 from os import urandom as _urandom
@@ -240,8 +239,7 @@
 
         return self.randrange(a, b+1)
 
-    def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L<<BPF,
-                   _Method=_MethodType, _BuiltinMethod=_BuiltinMethodType):
+    def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L<<BPF):
         """Return a random int in the range [0,n)
 
         Handles the case where n has more bits than returned
@@ -256,7 +254,8 @@
             # Only call self.getrandbits if the original random() builtin method
             # has not been overridden or if a new getrandbits() was supplied.
             # This assures that the two methods correspond.
-            if type(self.random) is _BuiltinMethod or type(getrandbits) is _Method:
+            if (self.random == super(Random, self).random or
+                getrandbits != super(Random, self).getrandbits):
                 k = int(1.00001 + _log(n-1, 2.0))   # 2**k > n-1 > 2**(k-2)
                 r = getrandbits(k)
                 while r >= n:
diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py
--- a/lib-python/2.7/site.py
+++ b/lib-python/2.7/site.py
@@ -75,7 +75,6 @@
 USER_SITE = None
 USER_BASE = None
 
-
 def makepath(*paths):
     dir = os.path.join(*paths)
     try:
@@ -91,7 +90,10 @@
         if hasattr(m, '__loader__'):
             continue   # don't mess with a PEP 302-supplied __file__
         try:
-            m.__file__ = os.path.abspath(m.__file__)
+            prev = m.__file__
+            new = os.path.abspath(m.__file__)
+            if prev != new:
+                m.__file__ = new
         except (AttributeError, OSError):
             pass
 
@@ -289,6 +291,7 @@
     will find its `site-packages` subdirectory depending on the system
     environment, and will return a list of full paths.
     """
+    is_pypy = '__pypy__' in sys.builtin_module_names
     sitepackages = []
     seen = set()
 
@@ -299,6 +302,10 @@
 
         if sys.platform in ('os2emx', 'riscos'):
             sitepackages.append(os.path.join(prefix, "Lib", "site-packages"))
+        elif is_pypy:
+            from distutils.sysconfig import get_python_lib
+            sitedir = get_python_lib(standard_lib=False, prefix=prefix)
+            sitepackages.append(sitedir)
         elif os.sep == '/':
             sitepackages.append(os.path.join(prefix, "lib",
                                         "python" + sys.version[:3],
@@ -435,22 +442,33 @@
                 if key == 'q':
                     break
 
+##def setcopyright():
+##    """Set 'copyright' and 'credits' in __builtin__"""
+##    __builtin__.copyright = _Printer("copyright", sys.copyright)
+##    if sys.platform[:4] == 'java':
+##        __builtin__.credits = _Printer(
+##            "credits",
+##            "Jython is maintained by the Jython developers (www.jython.org).")
+##    else:
+##        __builtin__.credits = _Printer("credits", """\
+##    Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
+##    for supporting Python development.  See www.python.org for more information.""")
+##    here = os.path.dirname(os.__file__)
+##    __builtin__.license = _Printer(
+##        "license", "See http://www.python.org/%.3s/license.html" % sys.version,
+##        ["LICENSE.txt", "LICENSE"],
+##        [os.path.join(here, os.pardir), here, os.curdir])
+
 def setcopyright():
-    """Set 'copyright' and 'credits' in __builtin__"""
+    # XXX this is the PyPy-specific version.  Should be unified with the above.
     __builtin__.copyright = _Printer("copyright", sys.copyright)
-    if sys.platform[:4] == 'java':
-        __builtin__.credits = _Printer(
-            "credits",
-            "Jython is maintained by the Jython developers (www.jython.org).")
-    else:
-        __builtin__.credits = _Printer("credits", """\
-    Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
-    for supporting Python development.  See www.python.org for more information.""")
-    here = os.path.dirname(os.__file__)
+    __builtin__.credits = _Printer(
+        "credits",
+        "PyPy is maintained by the PyPy developers: http://pypy.org/")
     __builtin__.license = _Printer(
-        "license", "See http://www.python.org/%.3s/license.html" % sys.version,
-        ["LICENSE.txt", "LICENSE"],
-        [os.path.join(here, os.pardir), here, os.curdir])
+        "license",
+        "See https://bitbucket.org/pypy/pypy/src/default/LICENSE")
+
 
 
 class _Helper(object):
@@ -476,7 +494,7 @@
     if sys.platform == 'win32':
         import locale, codecs
         enc = locale.getdefaultlocale()[1]
-        if enc.startswith('cp'):            # "cp***" ?
+        if enc is not None and enc.startswith('cp'):            # "cp***" ?
             try:
                 codecs.lookup(enc)
             except LookupError:
@@ -532,9 +550,18 @@
                 "'import usercustomize' failed; use -v for traceback"
 
 
+def import_builtin_stuff():
+    """PyPy specific: pre-import a few built-in modules, because
+    some programs actually rely on them to be in sys.modules :-("""
+    import exceptions
+    if 'zipimport' in sys.builtin_module_names:
+        import zipimport
+
+
 def main():
     global ENABLE_USER_SITE
 
+    import_builtin_stuff()
     abs__file__()
     known_paths = removeduppaths()
     if (os.name == "posix" and sys.path and
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -46,8 +46,6 @@
 
 import _socket
 from _socket import *
-from functools import partial
-from types import MethodType
 
 try:
     import _ssl
@@ -159,11 +157,6 @@
 if sys.platform == "riscos":
     _socketmethods = _socketmethods + ('sleeptaskw',)
 
-# All the method names that must be delegated to either the real socket
-# object or the _closedsocket object.
-_delegate_methods = ("recv", "recvfrom", "recv_into", "recvfrom_into",
-                     "send", "sendto")
-
 class _closedsocket(object):
     __slots__ = []
     def _dummy(*args):
@@ -180,22 +173,43 @@
 
     __doc__ = _realsocket.__doc__
 
-    __slots__ = ["_sock", "__weakref__"] + list(_delegate_methods)
-
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
         self._sock = _sock
-        for method in _delegate_methods:
-            setattr(self, method, getattr(_sock, method))
+        self._io_refs = 0
+        self._closed = False
 
-    def close(self, _closedsocket=_closedsocket,
-              _delegate_methods=_delegate_methods, setattr=setattr):
+    def send(self, data, flags=0):
+        return self._sock.send(data, flags=flags)
+    send.__doc__ = _realsocket.send.__doc__
+
+    def recv(self, buffersize, flags=0):
+        return self._sock.recv(buffersize, flags=flags)
+    recv.__doc__ = _realsocket.recv.__doc__
+
+    def recv_into(self, buffer, nbytes=0, flags=0):
+        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+    recv_into.__doc__ = _realsocket.recv_into.__doc__
+
+    def recvfrom(self, buffersize, flags=0):
+        return self._sock.recvfrom(buffersize, flags=flags)
+    recvfrom.__doc__ = _realsocket.recvfrom.__doc__
+
+    def recvfrom_into(self, buffer, nbytes=0, flags=0):
+        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+    recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
+
+    def sendto(self, data, param2, param3=None):
+        if param3 is None:
+            return self._sock.sendto(data, param2)
+        else:
+            return self._sock.sendto(data, param2, param3)
+    sendto.__doc__ = _realsocket.sendto.__doc__
+
+    def close(self):
         # This function should not reference any globals. See issue #808164.
         self._sock = _closedsocket()
-        dummy = self._sock._dummy
-        for method in _delegate_methods:
-            setattr(self, method, dummy)
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
@@ -214,21 +228,49 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        return _fileobject(self._sock, mode, bufsize)
+        self._io_refs += 1
+        return _fileobject(self, mode, bufsize)
+
+    def _decref_socketios(self):
+        if self._io_refs > 0:
+            self._io_refs -= 1
+        if self._closed:
+            self.close()
+
+    def _real_close(self):
+        # This function should not reference any globals. See issue #808164.
+        self._sock.close()
+
+    def close(self):
+        # This function should not reference any globals. See issue #808164.
+        self._closed = True
+        if self._io_refs <= 0:
+            self._real_close()
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
     proto = property(lambda self: self._sock.proto, doc="the socket protocol")
 
-def meth(name,self,*args):
-    return getattr(self._sock,name)(*args)
+    # Delegate many calls to the raw socket object.
+    _s = ("def %(name)s(self, %(args)s): return self._sock.%(name)s(%(args)s)\n\n"
+          "%(name)s.__doc__ = _realsocket.%(name)s.__doc__\n")
+    for _m in _socketmethods:
+        # yupi! we're on pypy, all code objects have this interface
+        argcount = getattr(_realsocket, _m).im_func.func_code.co_argcount - 1
+        exec _s % {'name': _m, 'args': ', '.join('arg%d' % i for i in range(argcount))}
+    del _m, _s, argcount
 
-for _m in _socketmethods:
-    p = partial(meth,_m)
-    p.__name__ = _m
-    p.__doc__ = getattr(_realsocket,_m).__doc__
-    m = MethodType(p,None,_socketobject)
-    setattr(_socketobject,_m,m)
+    # Delegation methods with default arguments, that the code above
+    # cannot handle correctly
+    def sendall(self, data, flags=0):
+        self._sock.sendall(data, flags)
+    sendall.__doc__ = _realsocket.sendall.__doc__
+
+    def getsockopt(self, level, optname, buflen=None):
+        if buflen is None:
+            return self._sock.getsockopt(level, optname)
+        return self._sock.getsockopt(level, optname, buflen)
+    getsockopt.__doc__ = _realsocket.getsockopt.__doc__
 
 socket = SocketType = _socketobject
 
@@ -278,8 +320,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._close:
-                self._sock.close()
+            if self._sock:
+                if self._close:
+                    self._sock.close()
+                else:
+                    self._sock._decref_socketios()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/dbapi.py: tests for DB-API compliance
 #
 # Copyright (C) 2004-2010 Gerhard H&#65533;ring <gh at ghaering.de>
@@ -332,6 +332,9 @@
             def __init__(self):
                 self.value = 5
 
+            def __iter__(self):
+                return self
+
             def next(self):
                 if self.value == 10:
                     raise StopIteration
@@ -826,7 +829,7 @@
         con = sqlite.connect(":memory:")
         con.close()
         try:
-            con()
+            con("select 1")
             self.fail("Should have raised a ProgrammingError")
         except sqlite.ProgrammingError:
             pass
diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -264,6 +264,28 @@
         """
         self.assertRaises(sqlite.Warning, self.con, 1)
 
+    def CheckUpdateDescriptionNone(self):
+        """
+        Call Cursor.update with an UPDATE query and check that it sets the
+        cursor's description to be None.
+        """
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        cur.execute("UPDATE foo SET id = 3 WHERE id = 1")
+        self.assertEqual(cur.description, None)
+
+    def CheckStatementCache(self):
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        values = [(i,) for i in xrange(5)]
+        cur.executemany("INSERT INTO foo (id) VALUES (?)", values)
+
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+        self.con.commit()
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+
 def suite():
     regression_suite = unittest.makeSuite(RegressionTests, "Check")
     return unittest.TestSuite((regression_suite,))
diff --git a/lib-python/2.7/sqlite3/test/userfunctions.py b/lib-python/2.7/sqlite3/test/userfunctions.py
--- a/lib-python/2.7/sqlite3/test/userfunctions.py
+++ b/lib-python/2.7/sqlite3/test/userfunctions.py
@@ -275,12 +275,14 @@
             pass
 
     def CheckAggrNoStep(self):
+        # XXX it's better to raise OperationalError in order to stop
+        # the query earlier.
         cur = self.con.cursor()
         try:
             cur.execute("select nostep(t) from test")
-            self.fail("should have raised an AttributeError")
-        except AttributeError, e:
-            self.assertEqual(e.args[0], "AggrNoStep instance has no attribute 'step'")
+            self.fail("should have raised an OperationalError")
+        except sqlite.OperationalError, e:
+            self.assertEqual(e.args[0], "user-defined aggregate's 'step' method raised error")
 
     def CheckAggrNoFinalize(self):
         cur = self.con.cursor()
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -86,7 +86,7 @@
 else:
     _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
-from socket import socket, _fileobject, _delegate_methods, error as socket_error
+from socket import socket, _fileobject, error as socket_error
 from socket import getnameinfo as _getnameinfo
 import base64        # for DER-to-PEM translation
 import errno
@@ -103,14 +103,6 @@
                  do_handshake_on_connect=True,
                  suppress_ragged_eofs=True, ciphers=None):
         socket.__init__(self, _sock=sock._sock)
-        # The initializer for socket overrides the methods send(), recv(), etc.
-        # in the instancce, which we don't need -- but we want to provide the
-        # methods defined in SSLSocket.
-        for attr in _delegate_methods:
-            try:
-                delattr(self, attr)
-            except AttributeError:
-                pass
 
         if certfile and not keyfile:
             keyfile = certfile
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -803,7 +803,7 @@
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
             elif stderr == STDOUT:
-                errwrite = c2pwrite
+                errwrite = c2pwrite.handle # pass id to not close it
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
@@ -818,9 +818,13 @@
 
         def _make_inheritable(self, handle):
             """Return a duplicate of handle, which is inheritable"""
-            return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
+            dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
+            # If the initial handle was obtained with CreatePipe, close it.
+            if not isinstance(handle, int):
+                handle.Close()
+            return dupl
 
 
         def _find_w9xpopen(self):
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -26,6 +26,16 @@
         'scripts': '{base}/bin',
         'data'   : '{base}',
         },
+    'pypy': {
+        'stdlib': '{base}/lib-python',
+        'platstdlib': '{base}/lib-python',
+        'purelib': '{base}/lib-python',
+        'platlib': '{base}/lib-python',
+        'include': '{base}/include',
+        'platinclude': '{base}/include',
+        'scripts': '{base}/bin',
+        'data'   : '{base}',
+        },
     'nt': {
         'stdlib': '{base}/Lib',
         'platstdlib': '{base}/Lib',
@@ -158,7 +168,9 @@
     return res
 
 def _get_default_scheme():
-    if os.name == 'posix':
+    if '__pypy__' in sys.builtin_module_names:
+        return 'pypy'
+    elif os.name == 'posix':
         # the default scheme for posix is posix_prefix
         return 'posix_prefix'
     return os.name
@@ -182,126 +194,9 @@
     return env_base if env_base else joinuser("~", ".local")
 
 
-def _parse_makefile(filename, vars=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    import re
-    # Regexes needed for parsing Makefile (and similar syntaxes,
-    # like old-style Setup files).
-    _variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-    _findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-    _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-    if vars is None:
-        vars = {}
-    done = {}
-    notdone = {}
-
-    with open(filename) as f:
-        lines = f.readlines()
-
-    for line in lines:
-        if line.startswith('#') or line.strip() == '':
-            continue
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    vars.update(done)
-    return vars
-
-
-def _get_makefile_filename():
-    if _PYTHON_BUILD:
-        return os.path.join(_PROJECT_BASE, "Makefile")
-    return os.path.join(get_path('platstdlib'), "config", "Makefile")
-
-
 def _init_posix(vars):
     """Initialize the module as appropriate for POSIX systems."""
-    # load the installed Makefile:
-    makefile = _get_makefile_filename()
-    try:
-        _parse_makefile(makefile, vars)
-    except IOError, e:
-        msg = "invalid Python installation: unable to open %s" % makefile
-        if hasattr(e, "strerror"):
-            msg = msg + " (%s)" % e.strerror
-        raise IOError(msg)
-
-    # load the installed pyconfig.h:
-    config_h = get_config_h_filename()
-    try:
-        with open(config_h) as f:
-            parse_config_h(f, vars)
-    except IOError, e:
-        msg = "invalid Python installation: unable to open %s" % config_h
-        if hasattr(e, "strerror"):
-            msg = msg + " (%s)" % e.strerror
-        raise IOError(msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if _PYTHON_BUILD:
-        vars['LDSHARED'] = vars['BLDSHARED']
+    return
 
 def _init_non_posix(vars):
     """Initialize the module as appropriate for NT"""
@@ -474,10 +369,11 @@
                         # patched up as well.
                         'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
 
-                        flags = _CONFIG_VARS[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _CONFIG_VARS[key] = flags
+                        if key in _CONFIG_VARS: 
+                            flags = _CONFIG_VARS[key]
+                            flags = re.sub('-arch\s+\w+\s', ' ', flags)
+                            flags = flags + ' ' + arch
+                            _CONFIG_VARS[key] = flags
 
                 # If we're on OSX 10.5 or later and the user tries to
                 # compiles an extension using an SDK that is not present
diff --git a/lib-python/2.7/tarfile.py b/lib-python/2.7/tarfile.py
--- a/lib-python/2.7/tarfile.py
+++ b/lib-python/2.7/tarfile.py
@@ -1716,9 +1716,6 @@
         except (ImportError, AttributeError):
             raise CompressionError("gzip module is not available")
 
-        if fileobj is None:
-            fileobj = bltn_open(name, mode + "b")
-
         try:
             t = cls.taropen(name, mode,
                 gzip.GzipFile(name, mode, compresslevel, fileobj),
diff --git a/lib-python/2.7/test/__init__.py b/lib-python/2.7/test/__init__.py
--- a/lib-python/2.7/test/__init__.py
+++ b/lib-python/2.7/test/__init__.py
@@ -1,1 +1,11 @@
-# Dummy file to make this directory a package.
+"""
+This package only contains the tests that we have modified for PyPy.
+It uses the 'official' hack to include the rest of the standard
+'test' package from CPython.
+
+This assumes that sys.path is configured to contain
+'lib-python/modified-2.7.0' before 'lib-python/2.7.0'.
+"""
+
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
diff --git a/lib-python/2.7/test/list_tests.py b/lib-python/2.7/test/list_tests.py
--- a/lib-python/2.7/test/list_tests.py
+++ b/lib-python/2.7/test/list_tests.py
@@ -45,8 +45,12 @@
         self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
         self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
 
+        if test_support.check_impl_detail():
+            depth = sys.getrecursionlimit() + 100
+        else:
+            depth = 1000 * 1000 # should be enough to exhaust the stack
         l0 = []
-        for i in xrange(sys.getrecursionlimit() + 100):
+        for i in xrange(depth):
             l0 = [l0]
         self.assertRaises(RuntimeError, repr, l0)
 
@@ -472,7 +476,11 @@
         u += "eggs"
         self.assertEqual(u, self.type2test("spameggs"))
 
-        self.assertRaises(TypeError, u.__iadd__, None)
+        def f_iadd(u, x):
+            u += x
+            return u
+
+        self.assertRaises(TypeError, f_iadd, u, None)
 
     def test_imul(self):
         u = self.type2test([0, 1])
diff --git a/lib-python/2.7/test/mapping_tests.py b/lib-python/2.7/test/mapping_tests.py
--- a/lib-python/2.7/test/mapping_tests.py
+++ b/lib-python/2.7/test/mapping_tests.py
@@ -531,7 +531,10 @@
                     self.assertEqual(va, int(ka))
                     kb, vb = tb = b.popitem()
                     self.assertEqual(vb, int(kb))
-                    self.assertTrue(not(copymode < 0 and ta != tb))
+                    if copymode < 0 and test_support.check_impl_detail():
+                        # popitem() is not guaranteed to be deterministic on
+                        # all implementations
+                        self.assertEqual(ta, tb)
                 self.assertTrue(not a)
                 self.assertTrue(not b)
 
diff --git a/lib-python/2.7/test/pickletester.py b/lib-python/2.7/test/pickletester.py
--- a/lib-python/2.7/test/pickletester.py
+++ b/lib-python/2.7/test/pickletester.py
@@ -6,7 +6,7 @@
 import pickletools
 import copy_reg
 
-from test.test_support import TestFailed, have_unicode, TESTFN
+from test.test_support import TestFailed, have_unicode, TESTFN, impl_detail
 
 # Tests that try a number of pickle protocols should have a
 #     for proto in protocols:
@@ -949,6 +949,7 @@
                              "Failed protocol %d: %r != %r"
                              % (proto, obj, loaded))
 
+    @impl_detail("pypy does not store attribute names", pypy=False)
     def test_attribute_name_interning(self):
         # Test that attribute names of pickled objects are interned when
         # unpickling.
@@ -1091,6 +1092,7 @@
         s = StringIO.StringIO("X''.")
         self.assertRaises(EOFError, self.module.load, s)
 
+    @impl_detail("no full restricted mode in pypy", pypy=False)
     def test_restricted(self):
         # issue7128: cPickle failed in restricted mode
         builtins = {self.module.__name__: self.module,
diff --git a/lib-python/2.7/test/regrtest.py b/lib-python/2.7/test/regrtest.py
--- a/lib-python/2.7/test/regrtest.py
+++ b/lib-python/2.7/test/regrtest.py
@@ -680,8 +680,13 @@
 
 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
     """Return a list of all applicable test modules."""
-    testdir = findtestdir(testdir)
-    names = os.listdir(testdir)
+    if testdir:
+        testdirs = [testdir]
+    else:
+        testdirs = findtestdirs()
+    names = {}
+    for testdir in testdirs:
+        names.update(dict.fromkeys(os.listdir(testdir)))
     tests = []
     others = set(stdtests) | nottests
     for name in names:
@@ -1080,8 +1085,19 @@
     # Collect cyclic trash.
     gc.collect()
 
-def findtestdir(path=None):
-    return path or os.path.dirname(__file__) or os.curdir
+def findtestdirs():
+    # XXX hacking: returns a list of both the '2.7.0/test' and the
+    # 'modified-2.7.0/test' directories, as full paths.
+    testdir = os.path.abspath(os.path.dirname(__file__) or os.curdir)
+    assert os.path.basename(testdir).lower() == 'test'
+    maindir = os.path.dirname(testdir)
+    libpythondir = os.path.dirname(maindir)
+    maindirname = os.path.basename(maindir).lower()
+    if maindirname.startswith('modified-'):
+        maindirname = maindirname[len('modified-'):]
+    testdir1 = os.path.join(libpythondir, maindirname, 'test')
+    testdir2 = os.path.join(libpythondir, 'modified-'+maindirname, 'test')
+    return [testdir1, testdir2]
 
 def removepy(names):
     if not names:
@@ -1388,7 +1404,26 @@
         test_zipimport
         test_zlib
         """,
-    'openbsd3':
+    'openbsd4':
+        """
+        test_ascii_formatd
+        test_bsddb
+        test_bsddb3
+        test_ctypes
+        test_dl
+        test_epoll
+        test_gdbm
+        test_locale
+        test_normalization
+        test_ossaudiodev
+        test_pep277
+        test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
+        test_multiprocessing
+        """,
+    'openbsd5':
         """
         test_ascii_formatd
         test_bsddb
@@ -1503,13 +1538,7 @@
         return self.expected
 
 if __name__ == '__main__':
-    # findtestdir() gets the dirname out of __file__, so we have to make it
-    # absolute before changing the working directory.
-    # For example __file__ may be relative when running trace or profile.
-    # See issue #9323.
-    __file__ = os.path.abspath(__file__)
-
-    # sanity check
+    # Simplification for findtestdir().
     assert __file__ == os.path.abspath(sys.argv[0])
 
     # When tests are run from the Python build directory, it is best practice
diff --git a/lib-python/2.7/test/seq_tests.py b/lib-python/2.7/test/seq_tests.py
--- a/lib-python/2.7/test/seq_tests.py
+++ b/lib-python/2.7/test/seq_tests.py
@@ -307,12 +307,18 @@
 
     def test_bigrepeat(self):
         import sys
-        if sys.maxint <= 2147483647:
-            x = self.type2test([0])
-            x *= 2**16
-            self.assertRaises(MemoryError, x.__mul__, 2**16)
-            if hasattr(x, '__imul__'):
-                self.assertRaises(MemoryError, x.__imul__, 2**16)
+        # we chose an N such as 2**16 * N does not fit into a cpu word
+        if sys.maxint == 2147483647:
+            # 32 bit system
+            N = 2**16
+        else:
+            # 64 bit system
+            N = 2**48
+        x = self.type2test([0])
+        x *= 2**16
+        self.assertRaises(MemoryError, x.__mul__, N)
+        if hasattr(x, '__imul__'):
+            self.assertRaises(MemoryError, x.__imul__, N)
 
     def test_subscript(self):
         a = self.type2test([10, 11])
diff --git a/lib-python/2.7/test/string_tests.py b/lib-python/2.7/test/string_tests.py
--- a/lib-python/2.7/test/string_tests.py
+++ b/lib-python/2.7/test/string_tests.py
@@ -1024,7 +1024,10 @@
         self.checkequal('abc', 'abc', '__mul__', 1)
         self.checkequal('abcabcabc', 'abc', '__mul__', 3)
         self.checkraises(TypeError, 'abc', '__mul__')
-        self.checkraises(TypeError, 'abc', '__mul__', '')
+        class Mul(object):
+            def mul(self, a, b):
+                return a * b
+        self.checkraises(TypeError, Mul(), 'mul', 'abc', '')
         # XXX: on a 64-bit system, this doesn't raise an overflow error,
         # but either raises a MemoryError, or succeeds (if you have 54TiB)
         #self.checkraises(OverflowError, 10000*'abc', '__mul__', 2000000000)
diff --git a/lib-python/2.7/test/test_abstract_numbers.py b/lib-python/2.7/test/test_abstract_numbers.py
--- a/lib-python/2.7/test/test_abstract_numbers.py
+++ b/lib-python/2.7/test/test_abstract_numbers.py
@@ -40,7 +40,8 @@
 
         c1, c2 = complex(3, 2), complex(4,1)
         # XXX: This is not ideal, but see the comment in math_trunc().
-        self.assertRaises(AttributeError, math.trunc, c1)
+        # Modified to suit PyPy, which gives TypeError in all cases
+        self.assertRaises((AttributeError, TypeError), math.trunc, c1)
         self.assertRaises(TypeError, float, c1)
         self.assertRaises(TypeError, int, c1)
 
diff --git a/lib-python/2.7/test/test_aifc.py b/lib-python/2.7/test/test_aifc.py
--- a/lib-python/2.7/test/test_aifc.py
+++ b/lib-python/2.7/test/test_aifc.py
@@ -1,4 +1,4 @@
-from test.test_support import findfile, run_unittest, TESTFN
+from test.test_support import findfile, run_unittest, TESTFN, impl_detail
 import unittest
 import os
 
@@ -68,6 +68,7 @@
         self.assertEqual(f.getparams(), fout.getparams())
         self.assertEqual(f.readframes(5), fout.readframes(5))
 
+    @impl_detail("PyPy has no audioop module yet", pypy=False)
     def test_compress(self):
         f = self.f = aifc.open(self.sndfilepath)
         fout = self.fout = aifc.open(TESTFN, 'wb')
diff --git a/lib-python/2.7/test/test_array.py b/lib-python/2.7/test/test_array.py
--- a/lib-python/2.7/test/test_array.py
+++ b/lib-python/2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, a.__add__, b)
-
-        self.assertRaises(TypeError, a.__add__, "bad")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, a.__add__, b)
-
-        self.assertRaises(TypeError, a.__iadd__, "bad")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, a.__mul__, "bad")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, a.__imul__, "bad")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
@@ -769,6 +773,7 @@
         p = proxy(s)
         self.assertEqual(p.tostring(), s.tostring())
         s = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, len, p)
 
     def test_bug_782369(self):
diff --git a/lib-python/2.7/test/test_ascii_formatd.py b/lib-python/2.7/test/test_ascii_formatd.py
--- a/lib-python/2.7/test/test_ascii_formatd.py
+++ b/lib-python/2.7/test/test_ascii_formatd.py
@@ -4,6 +4,10 @@
 
 import unittest
 from test.test_support import check_warnings, run_unittest, import_module
+from test.test_support import check_impl_detail
+
+if not check_impl_detail(cpython=True):
+    raise unittest.SkipTest("this test is only for CPython")
 
 # Skip tests if _ctypes module does not exist
 import_module('_ctypes')
diff --git a/lib-python/2.7/test/test_ast.py b/lib-python/2.7/test/test_ast.py
--- a/lib-python/2.7/test/test_ast.py
+++ b/lib-python/2.7/test/test_ast.py
@@ -20,10 +20,24 @@
 # These tests are compiled through "exec"
 # There should be atleast one test per statement
 exec_tests = [
+    # None
+    "None",
     # FunctionDef
     "def f(): pass",
+    # FunctionDef with arg
+    "def f(a): pass",
+    # FunctionDef with arg and default value
+    "def f(a=0): pass",
+    # FunctionDef with varargs
+    "def f(*args): pass",
+    # FunctionDef with kwargs
+    "def f(**kwargs): pass",
+    # FunctionDef with all kind of args
+    "def f(a, b=1, c=None, d=[], e={}, *args, **kwargs): pass",
     # ClassDef
     "class C:pass",
+    # ClassDef, new style class
+    "class C(object): pass",
     # Return
     "def f():return 1",
     # Delete
@@ -68,6 +82,27 @@
     "for a,b in c: pass",
     "[(a,b) for a,b in c]",
     "((a,b) for a,b in c)",
+    "((a,b) for (a,b) in c)",
+    # Multiline generator expression
+    """(
+    (
+    Aa
+    ,
+       Bb
+    )
+    for
+    Aa
+    ,
+    Bb in Cc
+    )""",
+    # dictcomp
+    "{a : b for w in x for m in p if g}",
+    # dictcomp with naked tuple
+    "{a : b for v,w in x}",
+    # setcomp
+    "{r for l in x if g}",
+    # setcomp with naked tuple
+    "{r for l,m in x}",
 ]
 
 # These are compiled through "single"
@@ -80,6 +115,8 @@
 # These are compiled through "eval"
 # It should test all expressions
 eval_tests = [
+  # None
+  "None",
   # BoolOp
   "a and b",
   # BinOp
@@ -90,6 +127,16 @@
   "lambda:None",
   # Dict
   "{ 1:2 }",
+  # Empty dict
+  "{}",
+  # Set
+  "{None,}",
+  # Multiline dict
+  """{
+      1
+        :
+          2
+     }""",
   # ListComp
   "[a for b in c if d]",
   # GeneratorExp
@@ -114,8 +161,14 @@
   "v",
   # List
   "[1,2,3]",
+  # Empty list
+  "[]",
   # Tuple
   "1,2,3",
+  # Tuple
+  "(1,2,3)",
+  # Empty tuple
+  "()",
   # Combination
   "a.b.c.d(a.b[1:2])",
 
@@ -141,6 +194,35 @@
             elif value is not None:
                 self._assertTrueorder(value, parent_pos)
 
+    def test_AST_objects(self):
+        if test_support.check_impl_detail():
+            # PyPy also provides a __dict__ to the ast.AST base class.
+
+            x = ast.AST()
+            try:
+                x.foobar = 21
+            except AttributeError, e:
+                self.assertEquals(e.args[0],
+                                  "'_ast.AST' object has no attribute 'foobar'")
+            else:
+                self.assert_(False)
+
+            try:
+                ast.AST(lineno=2)
+            except AttributeError, e:
+                self.assertEquals(e.args[0],
+                                  "'_ast.AST' object has no attribute 'lineno'")
+            else:
+                self.assert_(False)
+
+        try:
+            ast.AST(2)
+        except TypeError, e:
+            self.assertEquals(e.args[0],
+                              "_ast.AST constructor takes 0 positional arguments")
+        else:
+            self.assert_(False)
+
     def test_snippets(self):
         for input, output, kind in ((exec_tests, exec_results, "exec"),
                                     (single_tests, single_results, "single"),
@@ -169,6 +251,114 @@
         self.assertTrue(issubclass(ast.comprehension, ast.AST))
         self.assertTrue(issubclass(ast.Gt, ast.AST))
 
+    def test_field_attr_existence(self):
+        for name, item in ast.__dict__.iteritems():
+            if isinstance(item, type) and name != 'AST' and name[0].isupper(): # XXX: pypy does not allow abstract ast class instanciation
+                x = item()
+                if isinstance(x, ast.AST):
+                    self.assertEquals(type(x._fields), tuple)
+
+    def test_arguments(self):
+        x = ast.arguments()
+        self.assertEquals(x._fields, ('args', 'vararg', 'kwarg', 'defaults'))
+        try:
+            x.vararg
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'arguments' object has no attribute 'vararg'")
+        else:
+            self.assert_(False)
+        x = ast.arguments(1, 2, 3, 4)
+        self.assertEquals(x.vararg, 2)
+
+    def test_field_attr_writable(self):
+        x = ast.Num()
+        # We can assign to _fields
+        x._fields = 666
+        self.assertEquals(x._fields, 666)
+
+    def test_classattrs(self):
+        x = ast.Num()
+        self.assertEquals(x._fields, ('n',))
+        try:
+            x.n
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'n'")
+        else:
+            self.assert_(False)
+
+        x = ast.Num(42)
+        self.assertEquals(x.n, 42)
+        try:
+            x.lineno
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'lineno'")
+        else:
+            self.assert_(False)
+
+        y = ast.Num()
+        x.lineno = y
+        self.assertEquals(x.lineno, y)
+
+        try:
+            x.foobar
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'foobar'")
+        else:
+            self.assert_(False)
+
+        x = ast.Num(lineno=2)
+        self.assertEquals(x.lineno, 2)
+
+        x = ast.Num(42, lineno=0)
+        self.assertEquals(x.lineno, 0)
+        self.assertEquals(x._fields, ('n',))
+        self.assertEquals(x.n, 42)
+
+        self.assertRaises(TypeError, ast.Num, 1, 2)
+        self.assertRaises(TypeError, ast.Num, 1, 2, lineno=0)
+
+    def test_module(self):
+        body = [ast.Num(42)]
+        x = ast.Module(body)
+        self.assertEquals(x.body, body)
+
+    def test_nodeclass(self):
+        x = ast.BinOp()
+        self.assertEquals(x._fields, ('left', 'op', 'right'))
+
+        # Zero arguments constructor explicitely allowed
+        x = ast.BinOp()
+        # Random attribute allowed too
+        x.foobarbaz = 5
+        self.assertEquals(x.foobarbaz, 5)
+
+        n1 = ast.Num(1)
+        n3 = ast.Num(3)
+        addop = ast.Add()
+        x = ast.BinOp(n1, addop, n3)
+        self.assertEquals(x.left, n1)
+        self.assertEquals(x.op, addop)
+        self.assertEquals(x.right, n3)
+        
+        x = ast.BinOp(1, 2, 3)
+        self.assertEquals(x.left, 1)
+        self.assertEquals(x.op, 2)
+        self.assertEquals(x.right, 3)
+
+        x = ast.BinOp(1, 2, 3, lineno=0)
+        self.assertEquals(x.lineno, 0)
+
+    def test_nodeclasses(self):
+        x = ast.BinOp(1, 2, 3, lineno=0)
+        self.assertEquals(x.left, 1)
+        self.assertEquals(x.op, 2)
+        self.assertEquals(x.right, 3)
+        self.assertEquals(x.lineno, 0)
+
     def test_nodeclasses(self):
         x = ast.BinOp(1, 2, 3, lineno=0)
         self.assertEqual(x.left, 1)
@@ -178,6 +368,12 @@
 
         # node raises exception when not given enough arguments
         self.assertRaises(TypeError, ast.BinOp, 1, 2)
+        # node raises exception when given too many arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4)
+        # node raises exception when not given enough arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, lineno=0)
+        # node raises exception when given too many arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4, lineno=0)
 
         # can set attributes through kwargs too
         x = ast.BinOp(left=1, op=2, right=3, lineno=0)
@@ -186,8 +382,14 @@
         self.assertEqual(x.right, 3)
         self.assertEqual(x.lineno, 0)
 
+        # Random kwargs also allowed
+        x = ast.BinOp(1, 2, 3, foobarbaz=42)
+        self.assertEquals(x.foobarbaz, 42)
+
+    def test_no_fields(self):
         # this used to fail because Sub._fields was None
         x = ast.Sub()
+        self.assertEquals(x._fields, ())
 
     def test_pickling(self):
         import pickle
@@ -330,8 +532,15 @@
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
+('Module', [('Expr', (1, 0), ('Name', (1, 0), 'None', ('Load',)))]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',))], None, None, []), [('Pass', (1, 10))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',))], None, None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, []), [('Pass', (1, 14))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, 'kwargs', []), [('Pass', (1, 17))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',)), ('Name', (1, 9), 'b', ('Param',)), ('Name', (1, 14), 'c', ('Param',)), ('Name', (1, 22), 'd', ('Param',)), ('Name', (1, 28), 'e', ('Param',))], 'args', 'kwargs', [('Num', (1, 11), 1), ('Name', (1, 16), 'None', ('Load',)), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 52))], [])]),
 ('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [('Pass', (1, 17))], [])]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
 ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
 ('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
@@ -355,16 +564,26 @@
 ('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]),
 ('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]),
 ('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), []), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))])]))]),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))])]))]),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [])]))]),
 ]
 single_results = [
 ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
 ]
 eval_results = [
+('Expression', ('Name', (1, 0), 'None', ('Load',))),
 ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
 ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
 ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
 ('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
 ('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
+('Expression', ('Dict', (1, 0), [], [])),
+('Expression', ('Set', (1, 0), [('Name', (1, 1), 'None', ('Load',))])),
+('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
 ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
@@ -376,7 +595,10 @@
 ('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
 ('Expression', ('Name', (1, 0), 'v', ('Load',))),
 ('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('List', (1, 0), [], ('Load',))),
 ('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 1), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 0), [], ('Load',))),
 ('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
 ]
 main()
diff --git a/lib-python/2.7/test/test_builtin.py b/lib-python/2.7/test/test_builtin.py
--- a/lib-python/2.7/test/test_builtin.py
+++ b/lib-python/2.7/test/test_builtin.py
@@ -3,7 +3,8 @@
 import platform
 import unittest
 from test.test_support import fcmp, have_unicode, TESTFN, unlink, \
-                              run_unittest, check_py3k_warnings
+                              run_unittest, check_py3k_warnings, \
+                              check_impl_detail
 import warnings
 from operator import neg
 
@@ -247,12 +248,14 @@
         self.assertRaises(TypeError, compile)
         self.assertRaises(ValueError, compile, 'print 42\n', '<string>', 'badmode')
         self.assertRaises(ValueError, compile, 'print 42\n', '<string>', 'single', 0xff)
-        self.assertRaises(TypeError, compile, chr(0), 'f', 'exec')
+        if check_impl_detail(cpython=True):
+            self.assertRaises(TypeError, compile, chr(0), 'f', 'exec')
         self.assertRaises(TypeError, compile, 'pass', '?', 'exec',
                           mode='eval', source='0', filename='tmp')
         if have_unicode:
             compile(unicode('print u"\xc3\xa5"\n', 'utf8'), '', 'exec')
-            self.assertRaises(TypeError, compile, unichr(0), 'f', 'exec')
+            if check_impl_detail(cpython=True):
+                self.assertRaises(TypeError, compile, unichr(0), 'f', 'exec')
             self.assertRaises(ValueError, compile, unicode('a = 1'), 'f', 'bad')
 
 
@@ -395,12 +398,16 @@
         self.assertEqual(eval('dir()', g, m), list('xyz'))
         self.assertEqual(eval('globals()', g, m), g)
         self.assertEqual(eval('locals()', g, m), m)
-        self.assertRaises(TypeError, eval, 'a', m)
+        # on top of CPython, the first dictionary (the globals) has to
+        # be a real dict.  This is not the case on top of PyPy.
+        if check_impl_detail(pypy=False):
+            self.assertRaises(TypeError, eval, 'a', m)
+
         class A:
             "Non-mapping"
             pass
         m = A()
-        self.assertRaises(TypeError, eval, 'a', g, m)
+        self.assertRaises((TypeError, AttributeError), eval, 'a', g, m)
 
         # Verify that dict subclasses work as well
         class D(dict):
@@ -491,9 +498,10 @@
         execfile(TESTFN, globals, locals)
         self.assertEqual(locals['z'], 2)
 
+        self.assertRaises(TypeError, execfile, TESTFN, {}, ())
         unlink(TESTFN)
         self.assertRaises(TypeError, execfile)
-        self.assertRaises(TypeError, execfile, TESTFN, {}, ())
+        self.assertRaises((TypeError, IOError), execfile, TESTFN, {}, ())
         import os
         self.assertRaises(IOError, execfile, os.curdir)
         self.assertRaises(IOError, execfile, "I_dont_exist")
@@ -1108,7 +1116,8 @@
             def __cmp__(self, other):
                 raise RuntimeError
             __hash__ = None # Invalid cmp makes this unhashable
-        self.assertRaises(RuntimeError, range, a, a + 1, badzero(1))
+        if check_impl_detail(cpython=True):
+            self.assertRaises(RuntimeError, range, a, a + 1, badzero(1))
 
         # Reject floats.
         self.assertRaises(TypeError, range, 1., 1., 1.)
diff --git a/lib-python/2.7/test/test_bytes.py b/lib-python/2.7/test/test_bytes.py
--- a/lib-python/2.7/test/test_bytes.py
+++ b/lib-python/2.7/test/test_bytes.py
@@ -694,6 +694,7 @@
         self.assertEqual(b, b1)
         self.assertTrue(b is b1)
 
+    @test.test_support.impl_detail("undocumented bytes.__alloc__()")
     def test_alloc(self):
         b = bytearray()
         alloc = b.__alloc__()
@@ -821,6 +822,8 @@
         self.assertEqual(b, b"")
         self.assertEqual(c, b"")
 
+    @test.test_support.impl_detail(
+        "resizing semantics of CPython rely on refcounting")
     def test_resize_forbidden(self):
         # #4509: can't resize a bytearray when there are buffer exports, even
         # if it wouldn't reallocate the underlying buffer.
@@ -853,6 +856,26 @@
         self.assertRaises(BufferError, delslice)
         self.assertEqual(b, orig)
 
+    @test.test_support.impl_detail("resizing semantics", cpython=False)
+    def test_resize_forbidden_non_cpython(self):
+        # on non-CPython implementations, we cannot prevent changes to
+        # bytearrays just because there are buffers around.  Instead,
+        # we get (on PyPy) a buffer that follows the changes and resizes.
+        b = bytearray(range(10))
+        for v in [memoryview(b), buffer(b)]:
+            b[5] = 99
+            self.assertIn(v[5], (99, chr(99)))
+            b[5] = 100
+            b += b
+            b += b
+            b += b
+            self.assertEquals(len(v), 80)
+            self.assertIn(v[5], (100, chr(100)))
+            self.assertIn(v[79], (9, chr(9)))
+            del b[10:]
+            self.assertRaises(IndexError, lambda: v[10])
+            self.assertEquals(len(v), 10)
+
     def test_empty_bytearray(self):
         # Issue #7561: operations on empty bytearrays could crash in many
         # situations, due to a fragile implementation of the
diff --git a/lib-python/2.7/test/test_bz2.py b/lib-python/2.7/test/test_bz2.py
--- a/lib-python/2.7/test/test_bz2.py
+++ b/lib-python/2.7/test/test_bz2.py
@@ -50,6 +50,7 @@
         self.filename = TESTFN
 
     def tearDown(self):
+        test_support.gc_collect()
         if os.path.isfile(self.filename):
             os.unlink(self.filename)
 
@@ -246,6 +247,8 @@
         for i in xrange(10000):
             o = BZ2File(self.filename)
             del o
+            if i % 100 == 0:
+                test_support.gc_collect()
 
     def testOpenNonexistent(self):
         # "Test opening a nonexistent file"
@@ -310,6 +313,7 @@
             for t in threads:
                 t.join()
 
+    @test_support.impl_detail()
     def testMixedIterationReads(self):
         # Issue #8397: mixed iteration and reads should be forbidden.
         with bz2.BZ2File(self.filename, 'wb') as f:
diff --git a/lib-python/2.7/test/test_cmd_line_script.py b/lib-python/2.7/test/test_cmd_line_script.py
--- a/lib-python/2.7/test/test_cmd_line_script.py
+++ b/lib-python/2.7/test/test_cmd_line_script.py
@@ -112,6 +112,8 @@
             self._check_script(script_dir, script_name, script_dir, '')
 
     def test_directory_compiled(self):
+        if test.test_support.check_impl_detail(pypy=True):
+            raise unittest.SkipTest("pypy won't load lone .pyc files")
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             compiled_name = compile_script(script_name)
@@ -173,6 +175,8 @@
                                script_name, 'test_pkg')
 
     def test_package_compiled(self):
+        if test.test_support.check_impl_detail(pypy=True):
+            raise unittest.SkipTest("pypy won't load lone .pyc files")
         with temp_dir() as script_dir:
             pkg_dir = os.path.join(script_dir, 'test_pkg')
             make_pkg(pkg_dir)
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -82,7 +82,7 @@
 
 import unittest
 import weakref
-import _testcapi
+from test import test_support
 
 
 def consts(t):
@@ -104,7 +104,9 @@
 
 class CodeTest(unittest.TestCase):
 
+    @test_support.impl_detail("test for PyCode_NewEmpty")
     def test_newempty(self):
+        import _testcapi
         co = _testcapi.code_newempty("filename", "funcname", 15)
         self.assertEqual(co.co_filename, "filename")
         self.assertEqual(co.co_name, "funcname")
@@ -132,6 +134,7 @@
         coderef = weakref.ref(f.__code__, callback)
         self.assertTrue(bool(coderef()))
         del f
+        test_support.gc_collect()
         self.assertFalse(bool(coderef()))
         self.assertTrue(self.called)
 
diff --git a/lib-python/2.7/test/test_codeop.py b/lib-python/2.7/test/test_codeop.py
--- a/lib-python/2.7/test/test_codeop.py
+++ b/lib-python/2.7/test/test_codeop.py
@@ -3,7 +3,7 @@
    Nick Mathewson
 """
 import unittest
-from test.test_support import run_unittest, is_jython
+from test.test_support import run_unittest, is_jython, check_impl_detail
 
 from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
 
@@ -270,7 +270,9 @@
         ai("a = 'a\\\n")
 
         ai("a = 1","eval")
-        ai("a = (","eval")
+        if check_impl_detail():   # on PyPy it asks for more data, which is not
+            ai("a = (","eval")    # completely correct but hard to fix and
+                                  # really a detail (in my opinion <arigo>)
         ai("]","eval")
         ai("())","eval")
         ai("[}","eval")
diff --git a/lib-python/2.7/test/test_coercion.py b/lib-python/2.7/test/test_coercion.py
--- a/lib-python/2.7/test/test_coercion.py
+++ b/lib-python/2.7/test/test_coercion.py
@@ -1,6 +1,7 @@
 import copy
 import unittest
-from test.test_support import run_unittest, TestFailed, check_warnings
+from test.test_support import (
+    run_unittest, TestFailed, check_warnings, check_impl_detail)
 
 
 # Fake a number that implements numeric methods through __coerce__
@@ -306,12 +307,18 @@
         self.assertNotEqual(cmp(u'fish', evil_coercer), 0)
         self.assertNotEqual(cmp(slice(1), evil_coercer), 0)
         # ...but that this still works
-        class WackyComparer(object):
-            def __cmp__(slf, other):
-                self.assertTrue(other == 42, 'expected evil_coercer, got %r' % other)
-                return 0
-            __hash__ = None # Invalid cmp makes this unhashable
-        self.assertEqual(cmp(WackyComparer(), evil_coercer), 0)
+        if check_impl_detail():
+            # NB. I (arigo) would consider the following as implementation-
+            # specific.  For example, in CPython, if we replace 42 with 42.0
+            # both below and in CoerceTo() above, then the test fails.  This
+            # hints that the behavior is really dependent on some obscure
+            # internal details.
+            class WackyComparer(object):
+                def __cmp__(slf, other):
+                    self.assertTrue(other == 42, 'expected evil_coercer, got %r' % other)
+                    return 0
+                __hash__ = None # Invalid cmp makes this unhashable
+            self.assertEqual(cmp(WackyComparer(), evil_coercer), 0)
         # ...and classic classes too, since that code path is a little different
         class ClassicWackyComparer:
             def __cmp__(slf, other):
diff --git a/lib-python/2.7/test/test_compile.py b/lib-python/2.7/test/test_compile.py
--- a/lib-python/2.7/test/test_compile.py
+++ b/lib-python/2.7/test/test_compile.py
@@ -3,6 +3,7 @@
 import _ast
 from test import test_support
 import textwrap
+from test.test_support import check_impl_detail
 
 class TestSpecifics(unittest.TestCase):
 
@@ -90,12 +91,13 @@
         self.assertEqual(m.results, ('z', g))
         exec 'z = locals()' in g, m
         self.assertEqual(m.results, ('z', m))
-        try:
-            exec 'z = b' in m
-        except TypeError:
-            pass
-        else:
-            self.fail('Did not validate globals as a real dict')
+        if check_impl_detail():
+            try:
+                exec 'z = b' in m
+            except TypeError:
+                pass
+            else:
+                self.fail('Did not validate globals as a real dict')
 
         class A:
             "Non-mapping"
diff --git a/lib-python/2.7/test/test_copy.py b/lib-python/2.7/test/test_copy.py
--- a/lib-python/2.7/test/test_copy.py
+++ b/lib-python/2.7/test/test_copy.py
@@ -637,6 +637,7 @@
         self.assertEqual(v[c], d)
         self.assertEqual(len(v), 2)
         del c, d
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
         x, y = C(), C()
         # The underlying containers are decoupled
@@ -666,6 +667,7 @@
         self.assertEqual(v[a].i, b.i)
         self.assertEqual(v[c].i, d.i)
         del c
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
 
     def test_deepcopy_weakvaluedict(self):
@@ -689,6 +691,7 @@
         self.assertTrue(t is d)
         del x, y, z, t
         del d
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
 
     def test_deepcopy_bound_method(self):
diff --git a/lib-python/2.7/test/test_cpickle.py b/lib-python/2.7/test/test_cpickle.py
--- a/lib-python/2.7/test/test_cpickle.py
+++ b/lib-python/2.7/test/test_cpickle.py
@@ -61,27 +61,27 @@
     error = cPickle.BadPickleGet
 
     def test_recursive_list(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_list,
                           self)
 
     def test_recursive_tuple(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_tuple,
                           self)
 
     def test_recursive_inst(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_inst,
                           self)
 
     def test_recursive_dict(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_dict,
                           self)
 
     def test_recursive_multi(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_multi,
                           self)
 
diff --git a/lib-python/2.7/test/test_csv.py b/lib-python/2.7/test/test_csv.py
--- a/lib-python/2.7/test/test_csv.py
+++ b/lib-python/2.7/test/test_csv.py
@@ -54,8 +54,10 @@
         self.assertEqual(obj.dialect.skipinitialspace, False)
         self.assertEqual(obj.dialect.strict, False)
         # Try deleting or changing attributes (they are read-only)
-        self.assertRaises(TypeError, delattr, obj.dialect, 'delimiter')
-        self.assertRaises(TypeError, setattr, obj.dialect, 'delimiter', ':')
+        self.assertRaises((TypeError, AttributeError), delattr, obj.dialect,
+                          'delimiter')
+        self.assertRaises((TypeError, AttributeError), setattr, obj.dialect,
+                          'delimiter', ':')
         self.assertRaises(AttributeError, delattr, obj.dialect, 'quoting')
         self.assertRaises(AttributeError, setattr, obj.dialect,
                           'quoting', None)
diff --git a/lib-python/2.7/test/test_deque.py b/lib-python/2.7/test/test_deque.py
--- a/lib-python/2.7/test/test_deque.py
+++ b/lib-python/2.7/test/test_deque.py
@@ -109,7 +109,7 @@
         self.assertEqual(deque('abc', maxlen=4).maxlen, 4)
         self.assertEqual(deque('abc', maxlen=2).maxlen, 2)
         self.assertEqual(deque('abc', maxlen=0).maxlen, 0)
-        with self.assertRaises(AttributeError):
+        with self.assertRaises((AttributeError, TypeError)):
             d = deque('abc')
             d.maxlen = 10
 
@@ -352,7 +352,10 @@
         for match in (True, False):
             d = deque(['ab'])
             d.extend([MutateCmp(d, match), 'c'])
-            self.assertRaises(IndexError, d.remove, 'c')
+            # On CPython we get IndexError: deque mutated during remove().
+            # Why is it an IndexError during remove() only???
+            # On PyPy it is a RuntimeError, as in the other operations.
+            self.assertRaises((IndexError, RuntimeError), d.remove, 'c')
             self.assertEqual(d, deque())
 
     def test_repr(self):
@@ -514,7 +517,7 @@
                 container = reversed(deque([obj, 1]))
             obj.x = iter(container)
             del obj, container
-            gc.collect()
+            test_support.gc_collect()
             self.assertTrue(ref() is None, "Cycle was not collected")
 
 class TestVariousIteratorArgs(unittest.TestCase):
@@ -630,6 +633,7 @@
         p = weakref.proxy(d)
         self.assertEqual(str(p), str(d))
         d = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, str, p)
 
     def test_strange_subclass(self):
diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py
--- a/lib-python/2.7/test/test_descr.py
+++ b/lib-python/2.7/test/test_descr.py
@@ -2,6 +2,7 @@
 import sys
 import types
 import unittest
+import popen2     # trigger early the warning from popen2.py
 
 from copy import deepcopy
 from test import test_support
@@ -1128,7 +1129,7 @@
 
         # Test lookup leaks [SF bug 572567]
         import gc
-        if hasattr(gc, 'get_objects'):
+        if test_support.check_impl_detail():
             class G(object):
                 def __cmp__(self, other):
                     return 0
@@ -1741,6 +1742,10 @@
                 raise MyException
 
         for name, runner, meth_impl, ok, env in specials:
+            if name == '__length_hint__' or name == '__sizeof__':
+                if not test_support.check_impl_detail():
+                    continue
+
             class X(Checker):
                 pass
             for attr, obj in env.iteritems():
@@ -1980,7 +1985,9 @@
         except TypeError, msg:
             self.assertTrue(str(msg).find("weak reference") >= 0)
         else:
-            self.fail("weakref.ref(no) should be illegal")
+            if test_support.check_impl_detail(pypy=False):
+                self.fail("weakref.ref(no) should be illegal")
+            #else: pypy supports taking weakrefs to some more objects
         class Weak(object):
             __slots__ = ['foo', '__weakref__']
         yes = Weak()
@@ -3092,7 +3099,16 @@
         class R(J):
             __slots__ = ["__dict__", "__weakref__"]
 
-        for cls, cls2 in ((G, H), (G, I), (I, H), (Q, R), (R, Q)):
+        if test_support.check_impl_detail(pypy=False):
+            lst = ((G, H), (G, I), (I, H), (Q, R), (R, Q))
+        else:
+            # Not supported in pypy: changing the __class__ of an object
+            # to another __class__ that just happens to have the same slots.
+            # If needed, we can add the feature, but what we'll likely do
+            # then is to allow mostly any __class__ assignment, even if the
+            # classes have different __slots__, because we it's easier.
+            lst = ((Q, R), (R, Q))
+        for cls, cls2 in lst:
             x = cls()
             x.a = 1
             x.__class__ = cls2
@@ -3175,7 +3191,8 @@
             except TypeError:
                 pass
             else:
-                self.fail("%r's __dict__ can be modified" % cls)
+                if test_support.check_impl_detail(pypy=False):
+                    self.fail("%r's __dict__ can be modified" % cls)
 
         # Modules also disallow __dict__ assignment
         class Module1(types.ModuleType, Base):
@@ -4383,13 +4400,10 @@
         self.assertTrue(l.__add__ != [5].__add__)
         self.assertTrue(l.__add__ != l.__mul__)
         self.assertTrue(l.__add__.__name__ == '__add__')
-        if hasattr(l.__add__, '__self__'):
-            # CPython
-            self.assertTrue(l.__add__.__self__ is l)
+        self.assertTrue(l.__add__.__self__ is l)
+        if hasattr(l.__add__, '__objclass__'):   # CPython
             self.assertTrue(l.__add__.__objclass__ is list)
-        else:
-            # Python implementations where [].__add__ is a normal bound method
-            self.assertTrue(l.__add__.im_self is l)
+        else:                                    # PyPy
             self.assertTrue(l.__add__.im_class is list)
         self.assertEqual(l.__add__.__doc__, list.__add__.__doc__)
         try:
@@ -4578,8 +4592,12 @@
             str.split(fake_str)
 
         # call a slot wrapper descriptor
-        with self.assertRaises(TypeError):
-            str.__add__(fake_str, "abc")
+        try:
+            r = str.__add__(fake_str, "abc")
+        except TypeError:
+            pass
+        else:
+            self.assertEqual(r, NotImplemented)
 
 
 class DictProxyTests(unittest.TestCase):
diff --git a/lib-python/2.7/test/test_descrtut.py b/lib-python/2.7/test/test_descrtut.py
--- a/lib-python/2.7/test/test_descrtut.py
+++ b/lib-python/2.7/test/test_descrtut.py
@@ -172,46 +172,12 @@
     AttributeError: 'list' object has no attribute '__methods__'
     >>>
 
-Instead, you can get the same information from the list type:
+Instead, you can get the same information from the list type
+(the following example filters out the numerous method names
+starting with '_'):
 
-    >>> pprint.pprint(dir(list))    # like list.__dict__.keys(), but sorted
-    ['__add__',
-     '__class__',
-     '__contains__',
-     '__delattr__',
-     '__delitem__',
-     '__delslice__',
-     '__doc__',
-     '__eq__',
-     '__format__',
-     '__ge__',
-     '__getattribute__',
-     '__getitem__',
-     '__getslice__',
-     '__gt__',
-     '__hash__',
-     '__iadd__',
-     '__imul__',
-     '__init__',
-     '__iter__',
-     '__le__',
-     '__len__',
-     '__lt__',
-     '__mul__',
-     '__ne__',
-     '__new__',
-     '__reduce__',
-     '__reduce_ex__',
-     '__repr__',
-     '__reversed__',
-     '__rmul__',
-     '__setattr__',
-     '__setitem__',
-     '__setslice__',
-     '__sizeof__',
-     '__str__',
-     '__subclasshook__',
-     'append',
+    >>> pprint.pprint([name for name in dir(list) if not name.startswith('_')])
+    ['append',
      'count',
      'extend',
      'index',
diff --git a/lib-python/2.7/test/test_dict.py b/lib-python/2.7/test/test_dict.py
--- a/lib-python/2.7/test/test_dict.py
+++ b/lib-python/2.7/test/test_dict.py
@@ -319,7 +319,8 @@
                     self.assertEqual(va, int(ka))
                     kb, vb = tb = b.popitem()
                     self.assertEqual(vb, int(kb))
-                    self.assertFalse(copymode < 0 and ta != tb)
+                    if test_support.check_impl_detail():
+                        self.assertFalse(copymode < 0 and ta != tb)
                 self.assertFalse(a)
                 self.assertFalse(b)
 
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -56,8 +56,8 @@
  %-4d         0 LOAD_CONST               1 (0)
               3 POP_JUMP_IF_TRUE        38
               6 LOAD_GLOBAL              0 (AssertionError)
-              9 BUILD_LIST               0
-             12 LOAD_FAST                0 (x)
+              9 LOAD_FAST                0 (x)
+             12 BUILD_LIST_FROM_ARG      0
              15 GET_ITER
         >>   16 FOR_ITER                12 (to 31)
              19 STORE_FAST               1 (s)
diff --git a/lib-python/2.7/test/test_doctest.py b/lib-python/2.7/test/test_doctest.py
--- a/lib-python/2.7/test/test_doctest.py
+++ b/lib-python/2.7/test/test_doctest.py
@@ -782,7 +782,7 @@
     ...     >>> x = 12
     ...     >>> print x//0
     ...     Traceback (most recent call last):
-    ...     ZeroDivisionError: integer division or modulo by zero
+    ...     ZeroDivisionError: integer division by zero
     ...     '''
     >>> test = doctest.DocTestFinder().find(f)[0]
     >>> doctest.DocTestRunner(verbose=False).run(test)
@@ -799,7 +799,7 @@
     ...     >>> print 'pre-exception output', x//0
     ...     pre-exception output
     ...     Traceback (most recent call last):
-    ...     ZeroDivisionError: integer division or modulo by zero
+    ...     ZeroDivisionError: integer division by zero
     ...     '''
     >>> test = doctest.DocTestFinder().find(f)[0]
     >>> doctest.DocTestRunner(verbose=False).run(test)
@@ -810,7 +810,7 @@
         print 'pre-exception output', x//0
     Exception raised:
         ...
-        ZeroDivisionError: integer division or modulo by zero
+        ZeroDivisionError: integer division by zero
     TestResults(failed=1, attempted=2)
 
 Exception messages may contain newlines:
@@ -978,7 +978,7 @@
     Exception raised:
         Traceback (most recent call last):
         ...
-        ZeroDivisionError: integer division or modulo by zero
+        ZeroDivisionError: integer division by zero
     TestResults(failed=1, attempted=1)
 """
     def displayhook(): r"""
@@ -1924,7 +1924,7 @@
     > <doctest foo-b&#228;r at baz[1]>(1)<module>()
     -> calls_set_trace()
     (Pdb) print foo
-    *** NameError: name 'foo' is not defined
+    *** NameError: global name 'foo' is not defined
     (Pdb) continue
     TestResults(failed=0, attempted=2)
 """
@@ -2229,7 +2229,7 @@
           favorite_color
       Exception raised:
           ...
-          NameError: name 'favorite_color' is not defined
+          NameError: global name 'favorite_color' is not defined
       <BLANKLINE>
       <BLANKLINE>
 
@@ -2289,7 +2289,7 @@
         favorite_color
     Exception raised:
         ...
-        NameError: name 'favorite_color' is not defined
+        NameError: global name 'favorite_color' is not defined
     **********************************************************************
     1 items had failures:
        1 of   2 in test_doctest.txt
@@ -2382,7 +2382,7 @@
         favorite_color
     Exception raised:
         ...
-        NameError: name 'favorite_color' is not defined
+        NameError: global name 'favorite_color' is not defined
     TestResults(failed=1, attempted=2)
     >>> doctest.master = None  # Reset master.
 
diff --git a/lib-python/2.7/test/test_dumbdbm.py b/lib-python/2.7/test/test_dumbdbm.py
--- a/lib-python/2.7/test/test_dumbdbm.py
+++ b/lib-python/2.7/test/test_dumbdbm.py
@@ -107,9 +107,11 @@
         f.close()
 
         # Mangle the file by adding \r before each newline
-        data = open(_fname + '.dir').read()
+        with open(_fname + '.dir') as f:
+            data = f.read()
         data = data.replace('\n', '\r\n')
-        open(_fname + '.dir', 'wb').write(data)
+        with open(_fname + '.dir', 'wb') as f:
+            f.write(data)
 
         f = dumbdbm.open(_fname)
         self.assertEqual(f['1'], 'hello')
diff --git a/lib-python/2.7/test/test_extcall.py b/lib-python/2.7/test/test_extcall.py
--- a/lib-python/2.7/test/test_extcall.py
+++ b/lib-python/2.7/test/test_extcall.py
@@ -90,19 +90,19 @@
 
     >>> class Nothing: pass
     ...
-    >>> g(*Nothing())
+    >>> g(*Nothing())                     #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: g() argument after * must be a sequence, not instance
+    TypeError: ...argument after * must be a sequence, not instance
 
     >>> class Nothing:
     ...     def __len__(self): return 5
     ...
 
-    >>> g(*Nothing())
+    >>> g(*Nothing())                     #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: g() argument after * must be a sequence, not instance
+    TypeError: ...argument after * must be a sequence, not instance
 
     >>> class Nothing():
     ...     def __len__(self): return 5
@@ -154,52 +154,50 @@
       ...
     TypeError: g() got multiple values for keyword argument 'x'
 
-    >>> f(**{1:2})
+    >>> f(**{1:2})                             #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: f() keywords must be strings
+    TypeError: ...keywords must be strings
 
     >>> h(**{'e': 2})
     Traceback (most recent call last):
       ...
     TypeError: h() got an unexpected keyword argument 'e'
 
-    >>> h(*h)
+    >>> h(*h)                                  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after * must be a sequence, not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> dir(*h)
+    >>> dir(*h)                                #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() argument after * must be a sequence, not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> None(*h)
+    >>> None(*h)                               #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: NoneType object argument after * must be a sequence, \
-not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> h(**h)
+    >>> h(**h)                                 #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> dir(**h)
+    >>> dir(**h)                               #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> None(**h)
+    >>> None(**h)                              #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: NoneType object argument after ** must be a mapping, \
-not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> dir(b=1, **{'b': 1})
+    >>> dir(b=1, **{'b': 1})                   #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() got multiple values for keyword argument 'b'
+    TypeError: ...got multiple values for keyword argument 'b'
 
 Another helper function
 
@@ -247,10 +245,10 @@
     ...     False
     True
 
-    >>> id(1, **{'foo': 1})
+    >>> id(1, **{'foo': 1})                 #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: id() takes no keyword arguments
+    TypeError: id() ... keyword argument...
 
 A corner case of keyword dictionary items being deleted during
 the function call setup. See <http://bugs.python.org/issue2016>.
diff --git a/lib-python/2.7/test/test_fcntl.py b/lib-python/2.7/test/test_fcntl.py
--- a/lib-python/2.7/test/test_fcntl.py
+++ b/lib-python/2.7/test/test_fcntl.py
@@ -32,7 +32,7 @@
                         'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5',
                         'freebsd6', 'freebsd7', 'freebsd8',
                         'bsdos2', 'bsdos3', 'bsdos4',
-                        'openbsd', 'openbsd2', 'openbsd3', 'openbsd4'):
+                        'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', 'openbsd5'):
         if struct.calcsize('l') == 8:
             off_t = 'l'
             pid_t = 'i'
diff --git a/lib-python/2.7/test/test_file.py b/lib-python/2.7/test/test_file.py
--- a/lib-python/2.7/test/test_file.py
+++ b/lib-python/2.7/test/test_file.py
@@ -12,7 +12,7 @@
 import io
 import _pyio as pyio
 
-from test.test_support import TESTFN, run_unittest
+from test.test_support import TESTFN, run_unittest, gc_collect
 from UserList import UserList
 
 class AutoFileTests(unittest.TestCase):
@@ -33,6 +33,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testAttributes(self):
@@ -157,7 +158,12 @@
     def testStdin(self):
         # This causes the interpreter to exit on OSF1 v5.1.
         if sys.platform != 'osf1V5':
-            self.assertRaises((IOError, ValueError), sys.stdin.seek, -1)
+            if sys.stdin.isatty():
+                self.assertRaises((IOError, ValueError), sys.stdin.seek, -1)
+            else:
+                print((
+                    '  Skipping sys.stdin.seek(-1): stdin is not a tty.'
+                    ' Test manually.'), file=sys.__stdout__)
         else:
             print((
                 '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
diff --git a/lib-python/2.7/test/test_file2k.py b/lib-python/2.7/test/test_file2k.py
--- a/lib-python/2.7/test/test_file2k.py
+++ b/lib-python/2.7/test/test_file2k.py
@@ -11,7 +11,7 @@
     threading = None
 
 from test import test_support
-from test.test_support import TESTFN, run_unittest
+from test.test_support import TESTFN, run_unittest, gc_collect
 from UserList import UserList
 
 class AutoFileTests(unittest.TestCase):
@@ -32,6 +32,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testAttributes(self):
@@ -116,8 +117,12 @@
 
         for methodname in methods:
             method = getattr(self.f, methodname)
+            args = {'readinto': (bytearray(''),),
+                    'seek':     (0,),
+                    'write':    ('',),
+                    }.get(methodname, ())
             # should raise on closed file
-            self.assertRaises(ValueError, method)
+            self.assertRaises(ValueError, method, *args)
         with test_support.check_py3k_warnings():
             for methodname in deprecated_methods:
                 method = getattr(self.f, methodname)
@@ -216,7 +221,12 @@
     def testStdin(self):
         # This causes the interpreter to exit on OSF1 v5.1.
         if sys.platform != 'osf1V5':
-            self.assertRaises(IOError, sys.stdin.seek, -1)
+            if sys.stdin.isatty():
+                self.assertRaises(IOError, sys.stdin.seek, -1)
+            else:
+                print >>sys.__stdout__, (
+                    '  Skipping sys.stdin.seek(-1): stdin is not a tty.'
+                    ' Test manualy.')
         else:
             print >>sys.__stdout__, (
                 '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
@@ -336,8 +346,9 @@
                 except ValueError:
                     pass
                 else:
-                    self.fail("%s%r after next() didn't raise ValueError" %
-                                     (methodname, args))
+                    if test_support.check_impl_detail():
+                        self.fail("%s%r after next() didn't raise ValueError" %
+                                  (methodname, args))
                 f.close()
 
             # Test to see if harmless (by accident) mixing of read* and
@@ -388,6 +399,7 @@
             if lines != testlines:
                 self.fail("readlines() after next() with empty buffer "
                           "failed. Got %r, expected %r" % (line, testline))
+            f.close()
             # Reading after iteration hit EOF shouldn't hurt either
             f = open(TESTFN)
             try:
@@ -438,6 +450,9 @@
         self.close_count = 0
         self.close_success_count = 0
         self.use_buffering = False
+        # to prevent running out of file descriptors on PyPy,
+        # we only keep the 50 most recent files open
+        self.all_files = [None] * 50
 
     def tearDown(self):
         if self.f:
@@ -453,9 +468,14 @@
 
     def _create_file(self):
         if self.use_buffering:
-            self.f = open(self.filename, "w+", buffering=1024*16)
+            f = open(self.filename, "w+", buffering=1024*16)
         else:
-            self.f = open(self.filename, "w+")
+            f = open(self.filename, "w+")
+        self.f = f
+        self.all_files.append(f)
+        oldf = self.all_files.pop(0)
+        if oldf is not None:
+            oldf.close()
 
     def _close_file(self):
         with self._count_lock:
@@ -496,7 +516,6 @@
 
     def _test_close_open_io(self, io_func, nb_workers=5):
         def worker():
-            self._create_file()
             funcs = itertools.cycle((
                 lambda: io_func(),
                 lambda: self._close_and_reopen_file(),
@@ -508,7 +527,11 @@
                     f()
                 except (IOError, ValueError):
                     pass
+        self._create_file()
         self._run_workers(worker, nb_workers)
+        # make sure that all files can be closed now
+        del self.all_files
+        gc_collect()
         if test_support.verbose:
             # Useful verbose statistics when tuning this test to take
             # less time to run but still ensuring that its still useful.
diff --git a/lib-python/2.7/test/test_fileio.py b/lib-python/2.7/test/test_fileio.py
--- a/lib-python/2.7/test/test_fileio.py
+++ b/lib-python/2.7/test/test_fileio.py
@@ -12,6 +12,7 @@
 
 from test.test_support import TESTFN, check_warnings, run_unittest, make_bad_fd
 from test.test_support import py3k_bytes as bytes
+from test.test_support import gc_collect
 from test.script_helper import run_python
 
 from _io import FileIO as _FileIO
@@ -34,6 +35,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testSeekTell(self):
@@ -104,8 +106,8 @@
         self.assertTrue(f.closed)
 
     def testMethods(self):
-        methods = ['fileno', 'isatty', 'read', 'readinto',
-                   'seek', 'tell', 'truncate', 'write', 'seekable',
+        methods = ['fileno', 'isatty', 'read',
+                   'tell', 'truncate', 'seekable',
                    'readable', 'writable']
         if sys.platform.startswith('atheos'):
             methods.remove('truncate')
@@ -117,6 +119,10 @@
             method = getattr(self.f, methodname)
             # should raise on closed file
             self.assertRaises(ValueError, method)
+        # methods with one argument
+        self.assertRaises(ValueError, self.f.readinto, 0)
+        self.assertRaises(ValueError, self.f.write, 0)
+        self.assertRaises(ValueError, self.f.seek, 0)
 
     def testOpendir(self):
         # Issue 3703: opening a directory should fill the errno
diff --git a/lib-python/2.7/test/test_format.py b/lib-python/2.7/test/test_format.py
--- a/lib-python/2.7/test/test_format.py
+++ b/lib-python/2.7/test/test_format.py
@@ -242,7 +242,7 @@
             try:
                 testformat(formatstr, args)
             except exception, exc:
-                if str(exc) == excmsg:
+                if str(exc) == excmsg or not test_support.check_impl_detail():
                     if verbose:
                         print "yes"
                 else:
@@ -272,13 +272,16 @@
         test_exc(u'no format', u'1', TypeError,
                  "not all arguments converted during string formatting")
 
-        class Foobar(long):
-            def __oct__(self):
-                # Returning a non-string should not blow up.
-                return self + 1
-
-        test_exc('%o', Foobar(), TypeError,
-                 "expected string or Unicode object, long found")
+        if test_support.check_impl_detail():
+            # __oct__() is called if Foobar inherits from 'long', but
+            # not, say, 'object' or 'int' or 'str'.  This seems strange
+            # enough to consider it a complete implementation detail.
+            class Foobar(long):
+                def __oct__(self):
+                    # Returning a non-string should not blow up.
+                    return self + 1
+            test_exc('%o', Foobar(), TypeError,
+                     "expected string or Unicode object, long found")
 
         if maxsize == 2**31-1:
             # crashes 2.2.1 and earlier:
diff --git a/lib-python/2.7/test/test_funcattrs.py b/lib-python/2.7/test/test_funcattrs.py
--- a/lib-python/2.7/test/test_funcattrs.py
+++ b/lib-python/2.7/test/test_funcattrs.py
@@ -14,6 +14,8 @@
         self.b = b
 
     def cannot_set_attr(self, obj, name, value, exceptions):
+        if not test_support.check_impl_detail():
+            exceptions = (TypeError, AttributeError)
         # Helper method for other tests.
         try:
             setattr(obj, name, value)
@@ -286,13 +288,13 @@
     def test_delete_func_dict(self):
         try:
             del self.b.__dict__
-        except TypeError:
+        except (AttributeError, TypeError):
             pass
         else:
             self.fail("deleting function dictionary should raise TypeError")
         try:
             del self.b.func_dict
-        except TypeError:
+        except (AttributeError, TypeError):
             pass
         else:
             self.fail("deleting function dictionary should raise TypeError")
diff --git a/lib-python/2.7/test/test_functools.py b/lib-python/2.7/test/test_functools.py
--- a/lib-python/2.7/test/test_functools.py
+++ b/lib-python/2.7/test/test_functools.py
@@ -45,6 +45,8 @@
         # attributes should not be writable
         if not isinstance(self.thetype, type):
             return
+        if not test_support.check_impl_detail():
+            return
         self.assertRaises(TypeError, setattr, p, 'func', map)
         self.assertRaises(TypeError, setattr, p, 'args', (1, 2))
         self.assertRaises(TypeError, setattr, p, 'keywords', dict(a=1, b=2))
@@ -136,6 +138,7 @@
         p = proxy(f)
         self.assertEqual(f.func, p.func)
         f = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'func')
 
     def test_with_bound_and_unbound_methods(self):
@@ -172,7 +175,7 @@
                       updated=functools.WRAPPER_UPDATES):
         # Check attributes were assigned
         for name in assigned:
-            self.assertTrue(getattr(wrapper, name) is getattr(wrapped, name))
+            self.assertTrue(getattr(wrapper, name) == getattr(wrapped, name), name)
         # Check attributes were updated
         for name in updated:
             wrapper_attr = getattr(wrapper, name)
diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py
--- a/lib-python/2.7/test/test_generators.py
+++ b/lib-python/2.7/test/test_generators.py
@@ -190,7 +190,7 @@
       File "<stdin>", line 1, in ?
       File "<stdin>", line 2, in g
       File "<stdin>", line 2, in f
-    ZeroDivisionError: integer division or modulo by zero
+    ZeroDivisionError: integer division by zero
     >>> k.next()  # and the generator cannot be resumed
     Traceback (most recent call last):
       File "<stdin>", line 1, in ?
@@ -733,14 +733,16 @@
 ...     yield 1
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[0]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[0]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 >>> def f():
 ...     yield 1
 ...     return 22
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[1]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[1]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 "return None" is not the same as "return" in a generator:
 
@@ -749,7 +751,8 @@
 ...     return None
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[2]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[2]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 These are fine:
 
@@ -878,7 +881,9 @@
 ...     if 0:
 ...         yield 2             # because it's a generator (line 10)
 Traceback (most recent call last):
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[24]>, line 10)
+  ...
+  File "<doctest test.test_generators.__test__.syntax[24]>", line 10
+SyntaxError: 'return' with argument inside generator
 
 This one caused a crash (see SF bug 567538):
 
@@ -1496,6 +1501,10 @@
 """
 
 coroutine_tests = """\
+A helper function to call gc.collect() without printing
+>>> import gc
+>>> def gc_collect(): gc.collect()
+
 Sending a value into a started generator:
 
 >>> def f():
@@ -1570,13 +1579,14 @@
 >>> def f(): return lambda x=(yield): 1
 Traceback (most recent call last):
   ...
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.coroutine[22]>, line 1)
+  File "<doctest test.test_generators.__test__.coroutine[22]>", line 1
+SyntaxError: 'return' with argument inside generator
 
 >>> def f(): x = yield = y
 Traceback (most recent call last):
   ...
   File "<doctest test.test_generators.__test__.coroutine[23]>", line 1
-SyntaxError: assignment to yield expression not possible
+SyntaxError: can't assign to yield expression
 
 >>> def f(): (yield bar) = y
 Traceback (most recent call last):
@@ -1665,7 +1675,7 @@
 >>> f().throw("abc")     # throw on just-opened generator
 Traceback (most recent call last):
   ...
-TypeError: exceptions must be classes, or instances, not str
+TypeError: exceptions must be old-style classes or derived from BaseException, not str
 
 Now let's try closing a generator:
 
@@ -1697,7 +1707,7 @@
 
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 exiting
 
 >>> class context(object):
@@ -1708,7 +1718,7 @@
 ...          yield
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 exiting
 
 
@@ -1721,7 +1731,7 @@
 
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 finally
 
 
@@ -1747,6 +1757,7 @@
 >>> g = f()
 >>> g.next()
 >>> del g
+>>> gc_collect()
 >>> sys.stderr.getvalue().startswith(
 ...     "Exception RuntimeError: 'generator ignored GeneratorExit' in "
 ... )
@@ -1812,6 +1823,9 @@
 references. We add it to the standard suite so the routine refleak-tests
 would trigger if it starts being uncleanable again.
 
+>>> import gc
+>>> def gc_collect(): gc.collect()
+
 >>> import itertools
 >>> def leak():
 ...     class gen:
@@ -1863,9 +1877,10 @@
 ...
 ...     l = Leaker()
 ...     del l
+...     gc_collect()
 ...     err = sys.stderr.getvalue().strip()
 ...     err.startswith(
-...         "Exception RuntimeError: RuntimeError() in <"
+...         "Exception RuntimeError: RuntimeError() in "
 ...     )
 ...     err.endswith("> ignored")
 ...     len(err.splitlines())
diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py
--- a/lib-python/2.7/test/test_genexps.py
+++ b/lib-python/2.7/test/test_genexps.py
@@ -128,8 +128,9 @@
 
 Verify re-use of tuples (a side benefit of using genexps over listcomps)
 
+    >>> from test.test_support import check_impl_detail
     >>> tupleids = map(id, ((i,i) for i in xrange(10)))
-    >>> int(max(tupleids) - min(tupleids))
+    >>> int(max(tupleids) - min(tupleids)) if check_impl_detail() else 0
     0
 
 Verify that syntax error's are raised for genexps used as lvalues
@@ -198,13 +199,13 @@
     >>> g = (10 // i for i in (5, 0, 2))
     >>> g.next()
     2
-    >>> g.next()
+    >>> g.next()       # doctest: +ELLIPSIS
     Traceback (most recent call last):
       File "<pyshell#37>", line 1, in -toplevel-
         g.next()
       File "<pyshell#35>", line 1, in <generator expression>
         g = (10 // i for i in (5, 0, 2))
-    ZeroDivisionError: integer division or modulo by zero
+    ZeroDivisionError: integer division...by zero
     >>> g.next()
     Traceback (most recent call last):
       File "<pyshell#38>", line 1, in -toplevel-
diff --git a/lib-python/2.7/test/test_heapq.py b/lib-python/2.7/test/test_heapq.py
--- a/lib-python/2.7/test/test_heapq.py
+++ b/lib-python/2.7/test/test_heapq.py
@@ -215,6 +215,11 @@
 class TestHeapPython(TestHeap):
     module = py_heapq
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 @skipUnless(c_heapq, 'requires _heapq')
 class TestHeapC(TestHeap):
diff --git a/lib-python/2.7/test/test_import.py b/lib-python/2.7/test/test_import.py
--- a/lib-python/2.7/test/test_import.py
+++ b/lib-python/2.7/test/test_import.py
@@ -7,7 +7,8 @@
 import sys
 import unittest
 from test.test_support import (unlink, TESTFN, unload, run_unittest, rmtree,
-                               is_jython, check_warnings, EnvironmentVarGuard)
+                               is_jython, check_warnings, EnvironmentVarGuard,
+                               impl_detail, check_impl_detail)
 import textwrap
 from test import script_helper
 
@@ -69,7 +70,8 @@
                 self.assertEqual(mod.b, b,
                     "module loaded (%s) but contents invalid" % mod)
             finally:
-                unlink(source)
+                if check_impl_detail(pypy=False):
+                    unlink(source)
 
             try:
                 imp.reload(mod)
@@ -149,13 +151,16 @@
         # Compile & remove .py file, we only need .pyc (or .pyo).
         with open(filename, 'r') as f:
             py_compile.compile(filename)
-        unlink(filename)
+        if check_impl_detail(pypy=False):
+            # pypy refuses to import a .pyc if the .py does not exist
+            unlink(filename)
 
         # Need to be able to load from current dir.
         sys.path.append('')
 
         # This used to crash.
         exec 'import ' + module
+        reload(longlist)
 
         # Cleanup.
         del sys.path[-1]
@@ -326,6 +331,7 @@
         self.assertEqual(mod.code_filename, self.file_name)
         self.assertEqual(mod.func_filename, self.file_name)
 
+    @impl_detail("pypy refuses to import without a .py source", pypy=False)
     def test_module_without_source(self):
         target = "another_module.py"
         py_compile.compile(self.file_name, dfile=target)
diff --git a/lib-python/2.7/test/test_inspect.py b/lib-python/2.7/test/test_inspect.py
--- a/lib-python/2.7/test/test_inspect.py
+++ b/lib-python/2.7/test/test_inspect.py
@@ -4,11 +4,11 @@
 import unittest
 import inspect
 import linecache
-import datetime
 from UserList import UserList
 from UserDict import UserDict
 
 from test.test_support import run_unittest, check_py3k_warnings
+from test.test_support import check_impl_detail
 
 with check_py3k_warnings(
         ("tuple parameter unpacking has been removed", SyntaxWarning),
@@ -74,7 +74,8 @@
 
     def test_excluding_predicates(self):
         self.istest(inspect.isbuiltin, 'sys.exit')
-        self.istest(inspect.isbuiltin, '[].append')
+        if check_impl_detail():
+            self.istest(inspect.isbuiltin, '[].append')
         self.istest(inspect.iscode, 'mod.spam.func_code')
         self.istest(inspect.isframe, 'tb.tb_frame')
         self.istest(inspect.isfunction, 'mod.spam')
@@ -92,9 +93,9 @@
         else:
             self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals))
         if hasattr(types, 'MemberDescriptorType'):
-            self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days')
+            self.istest(inspect.ismemberdescriptor, 'type(lambda: None).func_globals')
         else:
-            self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days))
+            self.assertFalse(inspect.ismemberdescriptor(type(lambda: None).func_globals))
 
     def test_isroutine(self):
         self.assertTrue(inspect.isroutine(mod.spam))
@@ -567,7 +568,8 @@
         else:
             self.fail('Exception not raised')
         self.assertIs(type(ex1), type(ex2))
-        self.assertEqual(str(ex1), str(ex2))
+        if check_impl_detail():
+            self.assertEqual(str(ex1), str(ex2))
 
     def makeCallable(self, signature):
         """Create a function that returns its locals(), excluding the
diff --git a/lib-python/2.7/test/test_int.py b/lib-python/2.7/test/test_int.py
--- a/lib-python/2.7/test/test_int.py
+++ b/lib-python/2.7/test/test_int.py
@@ -1,7 +1,7 @@
 import sys
 
 import unittest
-from test.test_support import run_unittest, have_unicode
+from test.test_support import run_unittest, have_unicode, check_impl_detail
 import math
 
 L = [
@@ -392,9 +392,10 @@
                 try:
                     int(TruncReturnsNonIntegral())
                 except TypeError as e:
-                    self.assertEqual(str(e),
-                                      "__trunc__ returned non-Integral"
-                                      " (type NonIntegral)")
+                    if check_impl_detail(cpython=True):
+                        self.assertEqual(str(e),
+                                          "__trunc__ returned non-Integral"
+                                          " (type NonIntegral)")
                 else:
                     self.fail("Failed to raise TypeError with %s" %
                               ((base, trunc_result_base),))
diff --git a/lib-python/2.7/test/test_io.py b/lib-python/2.7/test/test_io.py
--- a/lib-python/2.7/test/test_io.py
+++ b/lib-python/2.7/test/test_io.py
@@ -2561,6 +2561,31 @@
         """Check that a partial write, when it gets interrupted, properly
         invokes the signal handler, and bubbles up the exception raised
         in the latter."""
+
+        # XXX This test has three flaws that appear when objects are
+        # XXX not reference counted.
+
+        # - if wio.write() happens to trigger a garbage collection,
+        #   the signal exception may be raised when some __del__
+        #   method is running; it will not reach the assertRaises()
+        #   call.
+
+        # - more subtle, if the wio object is not destroyed at once
+        #   and survives this function, the next opened file is likely
+        #   to have the same fileno (since the file descriptor was
+        #   actively closed).  When wio.__del__ is finally called, it
+        #   will close the other's test file...  To trigger this with
+        #   CPython, try adding "global wio" in this function.
+
+        # - This happens only for streams created by the _pyio module,
+        #   because a wio.close() that fails still consider that the
+        #   file needs to be closed again.  You can try adding an
+        #   "assert wio.closed" at the end of the function.
+
+        # Fortunately, a little gc.gollect() seems to be enough to
+        # work around all these issues.
+        support.gc_collect()
+
         read_results = []
         def _read():
             s = os.read(r, 1)
diff --git a/lib-python/2.7/test/test_isinstance.py b/lib-python/2.7/test/test_isinstance.py
--- a/lib-python/2.7/test/test_isinstance.py
+++ b/lib-python/2.7/test/test_isinstance.py
@@ -260,7 +260,18 @@
     # Make sure that calling isinstance with a deeply nested tuple for its
     # argument will raise RuntimeError eventually.
     tuple_arg = (compare_to,)
-    for cnt in xrange(sys.getrecursionlimit()+5):
+
+
+    if test_support.check_impl_detail(cpython=True):
+        RECURSION_LIMIT = sys.getrecursionlimit()
+    else:
+        # on non-CPython implementations, the maximum
+        # actual recursion limit might be higher, but
+        # probably not higher than 99999
+        #
+        RECURSION_LIMIT = 99999
+
+    for cnt in xrange(RECURSION_LIMIT+5):
         tuple_arg = (tuple_arg,)
         fxn(arg, tuple_arg)
 
diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py
--- a/lib-python/2.7/test/test_itertools.py
+++ b/lib-python/2.7/test/test_itertools.py
@@ -137,6 +137,8 @@
                 self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version
                 self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_combinations_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1)
@@ -207,7 +209,10 @@
                 self.assertEqual(result, list(cwr1(values, r)))         # matches first pure python version
                 self.assertEqual(result, list(cwr2(values, r)))         # matches second pure python version
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_combinations_with_replacement_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
+        cwr = combinations_with_replacement
         self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1)
 
@@ -271,6 +276,8 @@
                     self.assertEqual(result, list(permutations(values, None))) # test r as None
                     self.assertEqual(result, list(permutations(values)))       # test default r
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_permutations_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1)
@@ -526,6 +533,9 @@
         self.assertEqual(list(izip()), zip())
         self.assertRaises(TypeError, izip, 3)
         self.assertRaises(TypeError, izip, range(3), 3)
+
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_izip_tuple_reuse(self):
         # Check tuple re-use (implementation detail)
         self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')],
                          zip('abc', 'def'))
@@ -575,6 +585,8 @@
             else:
                 self.fail('Did not raise Type in:  ' + stmt)
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_iziplongest_tuple_reuse(self):
         # Check tuple re-use (implementation detail)
         self.assertEqual([tuple(list(pair)) for pair in izip_longest('abc', 'def')],
                          zip('abc', 'def'))
@@ -683,6 +695,8 @@
             args = map(iter, args)
             self.assertEqual(len(list(product(*args))), expected_len)
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_product_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, product('abc', 'def')))), 1)
         self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1)
@@ -771,11 +785,11 @@
         self.assertRaises(ValueError, islice, xrange(10), 1, -5, -1)
         self.assertRaises(ValueError, islice, xrange(10), 1, 10, -1)
         self.assertRaises(ValueError, islice, xrange(10), 1, 10, 0)
-        self.assertRaises(ValueError, islice, xrange(10), 'a')
-        self.assertRaises(ValueError, islice, xrange(10), 'a', 1)
-        self.assertRaises(ValueError, islice, xrange(10), 1, 'a')
-        self.assertRaises(ValueError, islice, xrange(10), 'a', 1, 1)
-        self.assertRaises(ValueError, islice, xrange(10), 1, 'a', 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a')
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a', 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 1, 'a')
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a', 1, 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 1, 'a', 1)
         self.assertEqual(len(list(islice(count(), 1, 10, maxsize))), 1)
 
         # Issue #10323:  Less islice in a predictable state
@@ -855,9 +869,17 @@
         self.assertRaises(TypeError, tee, [1,2], 3, 'x')
 
         # tee object should be instantiable
-        a, b = tee('abc')
-        c = type(a)('def')
-        self.assertEqual(list(c), list('def'))
+        if test_support.check_impl_detail():
+            # XXX I (arigo) would argue that 'type(a)(iterable)' has
+            # ill-defined semantics: it always return a fresh tee object,
+            # but depending on whether 'iterable' is itself a tee object
+            # or not, it is ok or not to continue using 'iterable' after
+            # the call.  I cannot imagine why 'type(a)(non_tee_object)'
+            # would be useful, as 'iter(non_tee_obect)' is equivalent
+            # as far as I can see.
+            a, b = tee('abc')
+            c = type(a)('def')
+            self.assertEqual(list(c), list('def'))
 
         # test long-lagged and multi-way split
         a, b, c = tee(xrange(2000), 3)
@@ -895,6 +917,7 @@
         p = proxy(a)
         self.assertEqual(getattr(p, '__class__'), type(b))
         del a
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, '__class__')
 
     def test_StopIteration(self):
@@ -1317,6 +1340,7 @@
 
 class LengthTransparency(unittest.TestCase):
 
+    @test_support.impl_detail("__length_hint__() API is undocumented")
     def test_repeat(self):
         from test.test_iterlen import len
         self.assertEqual(len(repeat(None, 50)), 50)
diff --git a/lib-python/2.7/test/test_linecache.py b/lib-python/2.7/test/test_linecache.py
--- a/lib-python/2.7/test/test_linecache.py
+++ b/lib-python/2.7/test/test_linecache.py
@@ -54,13 +54,13 @@
 
         # Check whether lines correspond to those from file iteration
         for entry in TESTS:
-            filename = os.path.join(TEST_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             for index, line in enumerate(open(filename)):
                 self.assertEqual(line, getline(filename, index + 1))
 
         # Check module loading
         for entry in MODULES:
-            filename = os.path.join(MODULE_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             for index, line in enumerate(open(filename)):
                 self.assertEqual(line, getline(filename, index + 1))
 
@@ -78,7 +78,7 @@
     def test_clearcache(self):
         cached = []
         for entry in TESTS:
-            filename = os.path.join(TEST_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             cached.append(filename)
             linecache.getline(filename, 1)
 
diff --git a/lib-python/2.7/test/test_list.py b/lib-python/2.7/test/test_list.py
--- a/lib-python/2.7/test/test_list.py
+++ b/lib-python/2.7/test/test_list.py
@@ -15,6 +15,10 @@
         self.assertEqual(list(''), [])
         self.assertEqual(list('spam'), ['s', 'p', 'a', 'm'])
 
+    # the following test also works with pypy, but eats all your address
+    # space's RAM before raising and takes too long.
+    @test_support.impl_detail("eats all your RAM before working", pypy=False)
+    def test_segfault_1(self):
         if sys.maxsize == 0x7fffffff:
             # This test can currently only work on 32-bit machines.
             # XXX If/when PySequence_Length() returns a ssize_t, it should be
@@ -32,6 +36,7 @@
             #     http://sources.redhat.com/ml/newlib/2002/msg00369.html
             self.assertRaises(MemoryError, list, xrange(sys.maxint // 2))
 
+    def test_segfault_2(self):
         # This code used to segfault in Py2.4a3
         x = []
         x.extend(-y for y in x)
diff --git a/lib-python/2.7/test/test_long.py b/lib-python/2.7/test/test_long.py
--- a/lib-python/2.7/test/test_long.py
+++ b/lib-python/2.7/test/test_long.py
@@ -530,9 +530,10 @@
                 try:
                     long(TruncReturnsNonIntegral())
                 except TypeError as e:
-                    self.assertEqual(str(e),
-                                     "__trunc__ returned non-Integral"
-                                     " (type NonIntegral)")
+                    if test_support.check_impl_detail(cpython=True):
+                        self.assertEqual(str(e),
+                                          "__trunc__ returned non-Integral"
+                                          " (type NonIntegral)")
                 else:
                     self.fail("Failed to raise TypeError with %s" %
                               ((base, trunc_result_base),))
diff --git a/lib-python/2.7/test/test_marshal.py b/lib-python/2.7/test/test_marshal.py
--- a/lib-python/2.7/test/test_marshal.py
+++ b/lib-python/2.7/test/test_marshal.py
@@ -7,20 +7,31 @@
 import unittest
 import os
 
-class IntTestCase(unittest.TestCase):
+class HelperMixin:
+    def helper(self, sample, *extra, **kwargs):
+        expected = kwargs.get('expected', sample)
+        new = marshal.loads(marshal.dumps(sample, *extra))
+        self.assertEqual(expected, new)
+        self.assertEqual(type(expected), type(new))
+        try:
+            with open(test_support.TESTFN, "wb") as f:
+                marshal.dump(sample, f, *extra)
+            with open(test_support.TESTFN, "rb") as f:
+                new = marshal.load(f)
+            self.assertEqual(expected, new)
+            self.assertEqual(type(expected), type(new))
+        finally:
+            test_support.unlink(test_support.TESTFN)
+
+
+class IntTestCase(unittest.TestCase, HelperMixin):
     def test_ints(self):
         # Test the full range of Python ints.
         n = sys.maxint
         while n:
             for expected in (-n, n):
-                s = marshal.dumps(expected)
-                got = marshal.loads(s)
-                self.assertEqual(expected, got)
-                marshal.dump(expected, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(expected, got)
+                self.helper(expected)
             n = n >> 1
-        os.unlink(test_support.TESTFN)
 
     def test_int64(self):
         # Simulate int marshaling on a 64-bit box.  This is most interesting if
@@ -48,28 +59,16 @@
 
     def test_bool(self):
         for b in (True, False):
-            new = marshal.loads(marshal.dumps(b))
-            self.assertEqual(b, new)
-            self.assertEqual(type(b), type(new))
-            marshal.dump(b, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(b, new)
-            self.assertEqual(type(b), type(new))
+            self.helper(b)
 
-class FloatTestCase(unittest.TestCase):
+class FloatTestCase(unittest.TestCase, HelperMixin):
     def test_floats(self):
         # Test a few floats
         small = 1e-25
         n = sys.maxint * 3.7e250
         while n > small:
             for expected in (-n, n):
-                f = float(expected)
-                s = marshal.dumps(f)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-                marshal.dump(f, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
+                self.helper(expected)
             n /= 123.4567
 
         f = 0.0
@@ -85,59 +84,25 @@
         while n < small:
             for expected in (-n, n):
                 f = float(expected)
+                self.helper(f)
+                self.helper(f, 1)
+            n *= 123.4567
 
-                s = marshal.dumps(f)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-
-                s = marshal.dumps(f, 1)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-
-                marshal.dump(f, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
-
-                marshal.dump(f, file(test_support.TESTFN, "wb"), 1)
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
-            n *= 123.4567
-        os.unlink(test_support.TESTFN)
-
-class StringTestCase(unittest.TestCase):
+class StringTestCase(unittest.TestCase, HelperMixin):
     def test_unicode(self):
         for s in [u"", u"Andr&#65533; Previn", u"abc", u" "*10000]:
-            new = marshal.loads(marshal.dumps(s))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-            marshal.dump(s, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-        os.unlink(test_support.TESTFN)
+            self.helper(s)
 
     def test_string(self):
         for s in ["", "Andr&#65533; Previn", "abc", " "*10000]:
-            new = marshal.loads(marshal.dumps(s))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-            marshal.dump(s, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-        os.unlink(test_support.TESTFN)
+            self.helper(s)
 
     def test_buffer(self):
         for s in ["", "Andr&#65533; Previn", "abc", " "*10000]:
             with test_support.check_py3k_warnings(("buffer.. not supported",
                                                      DeprecationWarning)):
                 b = buffer(s)
-            new = marshal.loads(marshal.dumps(b))
-            self.assertEqual(s, new)
-            marshal.dump(b, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-        os.unlink(test_support.TESTFN)
+            self.helper(b, expected=s)
 
 class ExceptionTestCase(unittest.TestCase):
     def test_exceptions(self):
@@ -150,7 +115,7 @@
         new = marshal.loads(marshal.dumps(co))
         self.assertEqual(co, new)
 
-class ContainerTestCase(unittest.TestCase):
+class ContainerTestCase(unittest.TestCase, HelperMixin):
     d = {'astring': 'foo at bar.baz.spam',
          'afloat': 7283.43,
          'anint': 2**20,
@@ -161,42 +126,20 @@
          'aunicode': u"Andr&#65533; Previn"
          }
     def test_dict(self):
-        new = marshal.loads(marshal.dumps(self.d))
-        self.assertEqual(self.d, new)
-        marshal.dump(self.d, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(self.d, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(self.d)
 
     def test_list(self):
         lst = self.d.items()
-        new = marshal.loads(marshal.dumps(lst))
-        self.assertEqual(lst, new)
-        marshal.dump(lst, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(lst, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(lst)
 
     def test_tuple(self):
         t = tuple(self.d.keys())
-        new = marshal.loads(marshal.dumps(t))
-        self.assertEqual(t, new)
-        marshal.dump(t, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(t, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(t)
 
     def test_sets(self):
         for constructor in (set, frozenset):
             t = constructor(self.d.keys())
-            new = marshal.loads(marshal.dumps(t))
-            self.assertEqual(t, new)
-            self.assertTrue(isinstance(new, constructor))
-            self.assertNotEqual(id(t), id(new))
-            marshal.dump(t, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(t, new)
-            os.unlink(test_support.TESTFN)
+            self.helper(t)
 
 class BugsTestCase(unittest.TestCase):
     def test_bug_5888452(self):
@@ -226,6 +169,7 @@
         s = 'c' + ('X' * 4*4) + '{' * 2**20
         self.assertRaises(ValueError, marshal.loads, s)
 
+    @test_support.impl_detail('specific recursion check')
     def test_recursion_limit(self):
         # Create a deeply nested structure.
         head = last = []
diff --git a/lib-python/2.7/test/test_memoryio.py b/lib-python/2.7/test/test_memoryio.py
--- a/lib-python/2.7/test/test_memoryio.py
+++ b/lib-python/2.7/test/test_memoryio.py
@@ -617,7 +617,7 @@
         state = memio.__getstate__()
         self.assertEqual(len(state), 3)
         bytearray(state[0]) # Check if state[0] supports the buffer interface.
-        self.assertIsInstance(state[1], int)
+        self.assertIsInstance(state[1], (int, long))
         self.assertTrue(isinstance(state[2], dict) or state[2] is None)
         memio.close()
         self.assertRaises(ValueError, memio.__getstate__)
diff --git a/lib-python/2.7/test/test_memoryview.py b/lib-python/2.7/test/test_memoryview.py
--- a/lib-python/2.7/test/test_memoryview.py
+++ b/lib-python/2.7/test/test_memoryview.py
@@ -26,7 +26,8 @@
     def check_getitem_with_type(self, tp):
         item = self.getitem_type
         b = tp(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         self.assertEqual(m[0], item(b"a"))
         self.assertIsInstance(m[0], bytes)
@@ -43,7 +44,8 @@
         self.assertRaises(TypeError, lambda: m[0.0])
         self.assertRaises(TypeError, lambda: m["a"])
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_getitem(self):
         for tp in self._types:
@@ -65,7 +67,8 @@
         if not self.ro_type:
             return
         b = self.ro_type(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         def setitem(value):
             m[0] = value
@@ -73,14 +76,16 @@
         self.assertRaises(TypeError, setitem, 65)
         self.assertRaises(TypeError, setitem, memoryview(b"a"))
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_setitem_writable(self):
         if not self.rw_type:
             return
         tp = self.rw_type
         b = self.rw_type(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         m[0] = tp(b"0")
         self._check_contents(tp, b, b"0bcdef")
@@ -110,13 +115,14 @@
         self.assertRaises(TypeError, setitem, (0,), b"a")
         self.assertRaises(TypeError, setitem, "a", b"a")
         # Trying to resize the memory object
-        self.assertRaises(ValueError, setitem, 0, b"")
-        self.assertRaises(ValueError, setitem, 0, b"ab")
+        self.assertRaises((ValueError, TypeError), setitem, 0, b"")
+        self.assertRaises((ValueError, TypeError), setitem, 0, b"ab")
         self.assertRaises(ValueError, setitem, slice(1,1), b"a")
         self.assertRaises(ValueError, setitem, slice(0,2), b"a")
 
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_delitem(self):
         for tp in self._types:
@@ -292,6 +298,7 @@
     def _check_contents(self, tp, obj, contents):
         self.assertEqual(obj[1:7], tp(contents))
 
+    @unittest.skipUnless(hasattr(sys, 'getrefcount'), "Reference counting")
     def test_refs(self):
         for tp in self._types:
             m = memoryview(tp(self._source))
diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py
--- a/lib-python/2.7/test/test_mmap.py
+++ b/lib-python/2.7/test/test_mmap.py
@@ -119,7 +119,8 @@
     def test_access_parameter(self):
         # Test for "access" keyword parameter
         mapsize = 10
-        open(TESTFN, "wb").write("a"*mapsize)
+        with open(TESTFN, "wb") as f:
+            f.write("a"*mapsize)
         f = open(TESTFN, "rb")
         m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_READ)
         self.assertEqual(m[:], 'a'*mapsize, "Readonly memory map data incorrect.")
@@ -168,9 +169,11 @@
         else:
             self.fail("Able to resize readonly memory map")
         f.close()
+        m.close()
         del m, f
-        self.assertEqual(open(TESTFN, "rb").read(), 'a'*mapsize,
-               "Readonly memory map data file was modified")
+        with open(TESTFN, "rb") as f:
+            self.assertEqual(f.read(), 'a'*mapsize,
+                "Readonly memory map data file was modified")
 
         # Opening mmap with size too big
         import sys
@@ -220,11 +223,13 @@
         self.assertEqual(m[:], 'd' * mapsize,
                "Copy-on-write memory map data not written correctly.")
         m.flush()
-        self.assertEqual(open(TESTFN, "rb").read(), 'c'*mapsize,
-               "Copy-on-write test data file should not be modified.")
+        f.close()
+        with open(TESTFN, "rb") as f:
+            self.assertEqual(f.read(), 'c'*mapsize,
+                "Copy-on-write test data file should not be modified.")
         # Ensuring copy-on-write maps cannot be resized
         self.assertRaises(TypeError, m.resize, 2*mapsize)
-        f.close()
+        m.close()
         del m, f
 
         # Ensuring invalid access parameter raises exception
@@ -287,6 +292,7 @@
         self.assertEqual(m.find('one', 1), 8)
         self.assertEqual(m.find('one', 1, -1), 8)
         self.assertEqual(m.find('one', 1, -2), -1)
+        m.close()
 
 
     def test_rfind(self):
@@ -305,6 +311,7 @@
         self.assertEqual(m.rfind('one', 0, -2), 0)
         self.assertEqual(m.rfind('one', 1, -1), 8)
         self.assertEqual(m.rfind('one', 1, -2), -1)
+        m.close()
 
 
     def test_double_close(self):
@@ -533,7 +540,8 @@
         if not hasattr(mmap, 'PROT_READ'):
             return
         mapsize = 10
-        open(TESTFN, "wb").write("a"*mapsize)
+        with open(TESTFN, "wb") as f:
+            f.write("a"*mapsize)
         f = open(TESTFN, "rb")
         m = mmap.mmap(f.fileno(), mapsize, prot=mmap.PROT_READ)
         self.assertRaises(TypeError, m.write, "foo")
@@ -545,7 +553,8 @@
 
     def test_io_methods(self):
         data = "0123456789"
-        open(TESTFN, "wb").write("x"*len(data))
+        with open(TESTFN, "wb") as f:
+            f.write("x"*len(data))
         f = open(TESTFN, "r+b")
         m = mmap.mmap(f.fileno(), len(data))
         f.close()
@@ -574,6 +583,7 @@
         self.assertEqual(m[:], "012bar6789")
         m.seek(8)
         self.assertRaises(ValueError, m.write, "bar")
+        m.close()
 
     if os.name == 'nt':
         def test_tagname(self):
@@ -611,7 +621,8 @@
             m.close()
 
             # Should not crash (Issue 5385)
-            open(TESTFN, "wb").write("x"*10)
+            with open(TESTFN, "wb") as f:
+                f.write("x"*10)
             f = open(TESTFN, "r+b")
             m = mmap.mmap(f.fileno(), 0)
             f.close()
diff --git a/lib-python/2.7/test/test_module.py b/lib-python/2.7/test/test_module.py
--- a/lib-python/2.7/test/test_module.py
+++ b/lib-python/2.7/test/test_module.py
@@ -1,6 +1,6 @@
 # Test the module type
 import unittest
-from test.test_support import run_unittest, gc_collect
+from test.test_support import run_unittest, gc_collect, check_impl_detail
 
 import sys
 ModuleType = type(sys)
@@ -10,8 +10,10 @@
         # An uninitialized module has no __dict__ or __name__,
         # and __doc__ is None
         foo = ModuleType.__new__(ModuleType)
-        self.assertTrue(foo.__dict__ is None)
-        self.assertRaises(SystemError, dir, foo)
+        self.assertFalse(foo.__dict__)
+        if check_impl_detail():
+            self.assertTrue(foo.__dict__ is None)
+            self.assertRaises(SystemError, dir, foo)
         try:
             s = foo.__name__
             self.fail("__name__ = %s" % repr(s))
diff --git a/lib-python/2.7/test/test_multibytecodec.py b/lib-python/2.7/test/test_multibytecodec.py
--- a/lib-python/2.7/test/test_multibytecodec.py
+++ b/lib-python/2.7/test/test_multibytecodec.py
@@ -42,7 +42,7 @@
         dec = codecs.getdecoder('euc-kr')
         myreplace  = lambda exc: (u'', sys.maxint+1)
         codecs.register_error('test.cjktest', myreplace)
-        self.assertRaises(IndexError, dec,
+        self.assertRaises((IndexError, OverflowError), dec,
                           'apple\x92ham\x93spam', 'test.cjktest')
 
     def test_codingspec(self):
@@ -148,7 +148,8 @@
 class Test_StreamReader(unittest.TestCase):
     def test_bug1728403(self):
         try:
-            open(TESTFN, 'w').write('\xa1')
+            with open(TESTFN, 'w') as f:
+                f.write('\xa1')
             f = codecs.open(TESTFN, encoding='cp949')
             self.assertRaises(UnicodeDecodeError, f.read, 2)
         finally:
diff --git a/lib-python/2.7/test/test_multibytecodec_support.py b/lib-python/2.7/test/test_multibytecodec_support.py
--- a/lib-python/2.7/test/test_multibytecodec_support.py
+++ b/lib-python/2.7/test/test_multibytecodec_support.py
@@ -110,8 +110,8 @@
         def myreplace(exc):
             return (u'x', sys.maxint + 1)
         codecs.register_error("test.cjktest", myreplace)
-        self.assertRaises(IndexError, self.encode, self.unmappedunicode,
-                          'test.cjktest')
+        self.assertRaises((IndexError, OverflowError), self.encode,
+                          self.unmappedunicode, 'test.cjktest')
 
     def test_callback_None_index(self):
         def myreplace(exc):
@@ -330,7 +330,7 @@
                             repr(csetch), repr(unich), exc.reason))
 
 def load_teststring(name):
-    dir = os.path.join(os.path.dirname(__file__), 'cjkencodings')
+    dir = test_support.findfile('cjkencodings')
     with open(os.path.join(dir, name + '.txt'), 'rb') as f:
         encoded = f.read()
     with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f:
diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py
--- a/lib-python/2.7/test/test_multiprocessing.py
+++ b/lib-python/2.7/test/test_multiprocessing.py
@@ -1316,6 +1316,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1605,6 +1606,10 @@
             if len(blocks) > maxblocks:
                 i = random.randrange(maxblocks)
                 del blocks[i]
+            # XXX There should be a better way to release resources for a
+            # single block
+            if i % maxblocks == 0:
+                import gc; gc.collect()
 
         # get the heap object
         heap = multiprocessing.heap.BufferWrapper._heap
@@ -1704,6 +1709,7 @@
         a = Foo()
         util.Finalize(a, conn.send, args=('a',))
         del a           # triggers callback for a
+        test_support.gc_collect()
 
         b = Foo()
         close_b = util.Finalize(b, conn.send, args=('b',))
diff --git a/lib-python/2.7/test/test_mutants.py b/lib-python/2.7/test/test_mutants.py
--- a/lib-python/2.7/test/test_mutants.py
+++ b/lib-python/2.7/test/test_mutants.py
@@ -1,4 +1,4 @@
-from test.test_support import verbose, TESTFN
+from test.test_support import verbose, TESTFN, check_impl_detail
 import random
 import os
 
@@ -137,10 +137,16 @@
     while dict1 and len(dict1) == len(dict2):
         if verbose:
             print ".",
-        if random.random() < 0.5:
-            c = cmp(dict1, dict2)
-        else:
-            c = dict1 == dict2
+        try:
+            if random.random() < 0.5:
+                c = cmp(dict1, dict2)
+            else:
+                c = dict1 == dict2
+        except RuntimeError:
+            # CPython never raises RuntimeError here, but other implementations
+            # might, and it's fine.
+            if check_impl_detail(cpython=True):
+                raise
     if verbose:
         print
 
diff --git a/lib-python/2.7/test/test_optparse.py b/lib-python/2.7/test/test_optparse.py
--- a/lib-python/2.7/test/test_optparse.py
+++ b/lib-python/2.7/test/test_optparse.py
@@ -383,6 +383,7 @@
         self.assertRaises(self.parser.remove_option, ('foo',), None,
                           ValueError, "no such option 'foo'")
 
+    @test_support.impl_detail("sys.getrefcount")
     def test_refleak(self):
         # If an OptionParser is carrying around a reference to a large
         # object, various cycles can prevent it from being GC'd in
diff --git a/lib-python/2.7/test/test_peepholer.py b/lib-python/2.7/test/test_peepholer.py
--- a/lib-python/2.7/test/test_peepholer.py
+++ b/lib-python/2.7/test/test_peepholer.py
@@ -41,7 +41,7 @@
     def test_none_as_constant(self):
         # LOAD_GLOBAL None  -->  LOAD_CONST None
         def f(x):
-            None
+            y = None
             return x
         asm = disassemble(f)
         for elem in ('LOAD_GLOBAL',):
@@ -67,10 +67,13 @@
             self.assertIn(elem, asm)
 
     def test_pack_unpack(self):
+        # On PyPy, "a, b = ..." is even more optimized, by removing
+        # the ROT_TWO.  But the ROT_TWO is not removed if assigning
+        # to more complex expressions, so check that.
         for line, elem in (
             ('a, = a,', 'LOAD_CONST',),
-            ('a, b = a, b', 'ROT_TWO',),
-            ('a, b, c = a, b, c', 'ROT_THREE',),
+            ('a[1], b = a, b', 'ROT_TWO',),
+            ('a, b[2], c = a, b, c', 'ROT_THREE',),
             ):
             asm = dis_single(line)
             self.assertIn(elem, asm)
@@ -78,6 +81,8 @@
             self.assertNotIn('UNPACK_TUPLE', asm)
 
     def test_folding_of_tuples_of_constants(self):
+        # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>"
+        # but on PyPy, it turns into "a=1;b=2;c=3".
         for line, elem in (
             ('a = 1,2,3', '((1, 2, 3))'),
             ('("a","b","c")', "(('a', 'b', 'c'))"),
@@ -86,7 +91,8 @@
             ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'),
             ):
             asm = dis_single(line)
-            self.assertIn(elem, asm)
+            self.assert_(elem in asm or (
+                line == 'a,b,c = 1,2,3' and 'UNPACK_TUPLE' not in asm))
             self.assertNotIn('BUILD_TUPLE', asm)
 
         # Bug 1053819:  Tuple of constants misidentified when presented with:
diff --git a/lib-python/2.7/test/test_pprint.py b/lib-python/2.7/test/test_pprint.py
--- a/lib-python/2.7/test/test_pprint.py
+++ b/lib-python/2.7/test/test_pprint.py
@@ -233,7 +233,16 @@
                                   frozenset([0, 2]),
                                   frozenset([0, 1])])}"""
         cube = test.test_set.cube(3)
-        self.assertEqual(pprint.pformat(cube), cube_repr_tgt)
+        # XXX issues of dictionary order, and for the case below,
+        # order of items in the frozenset([...]) representation.
+        # Whether we get precisely cube_repr_tgt or not is open
+        # to implementation-dependent choices (this test probably
+        # fails horribly in CPython if we tweak the dict order too).
+        got = pprint.pformat(cube)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertEqual(got, cube_repr_tgt)
+        else:
+            self.assertEqual(eval(got), cube)
         cubo_repr_tgt = """\
 {frozenset([frozenset([0, 2]), frozenset([0])]): frozenset([frozenset([frozenset([0,
                                                                                   2]),
@@ -393,7 +402,11 @@
                                                                                         2])])])}"""
 
         cubo = test.test_set.linegraph(cube)
-        self.assertEqual(pprint.pformat(cubo), cubo_repr_tgt)
+        got = pprint.pformat(cubo)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertEqual(got, cubo_repr_tgt)
+        else:
+            self.assertEqual(eval(got), cubo)
 
     def test_depth(self):
         nested_tuple = (1, (2, (3, (4, (5, 6)))))
diff --git a/lib-python/2.7/test/test_pydoc.py b/lib-python/2.7/test/test_pydoc.py
--- a/lib-python/2.7/test/test_pydoc.py
+++ b/lib-python/2.7/test/test_pydoc.py
@@ -267,8 +267,8 @@
         testpairs = (
             ('i_am_not_here', 'i_am_not_here'),
             ('test.i_am_not_here_either', 'i_am_not_here_either'),
-            ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
-            ('i_am_not_here.{}'.format(modname), 'i_am_not_here.{}'.format(modname)),
+            ('test.i_am_not_here.neither_am_i', 'i_am_not_here'),
+            ('i_am_not_here.{}'.format(modname), 'i_am_not_here'),
             ('test.{}'.format(modname), modname),
             )
 
@@ -292,8 +292,8 @@
                     result = run_pydoc(modname)
                 finally:
                     forget(modname)
-                expected = badimport_pattern % (modname, expectedinmsg)
-                self.assertEqual(expected, result)
+                expected = badimport_pattern % (modname, '(.+\\.)?' + expectedinmsg + '(\\..+)?$')
+                self.assertTrue(re.match(expected, result))
 
     def test_input_strip(self):
         missing_module = " test.i_am_not_here "
diff --git a/lib-python/2.7/test/test_pyexpat.py b/lib-python/2.7/test/test_pyexpat.py
--- a/lib-python/2.7/test/test_pyexpat.py
+++ b/lib-python/2.7/test/test_pyexpat.py
@@ -570,6 +570,9 @@
         self.assertEqual(self.n, 4)
 
 class MalformedInputText(unittest.TestCase):
+    # CPython seems to ship its own version of expat, they fixed it on this commit : 
+    # http://svn.python.org/view?revision=74429&view=revision
+    @unittest.skipIf(sys.platform == "darwin", "Expat is broken on Mac OS X 10.6.6")
     def test1(self):
         xml = "\0\r\n"
         parser = expat.ParserCreate()
@@ -579,6 +582,7 @@
         except expat.ExpatError as e:
             self.assertEqual(str(e), 'unclosed token: line 2, column 0')
 
+    @unittest.skipIf(sys.platform == "darwin", "Expat is broken on Mac OS X 10.6.6")
     def test2(self):
         xml = "<?xml version\xc2\x85='1.0'?>\r\n"
         parser = expat.ParserCreate()
diff --git a/lib-python/2.7/test/test_repr.py b/lib-python/2.7/test/test_repr.py
--- a/lib-python/2.7/test/test_repr.py
+++ b/lib-python/2.7/test/test_repr.py
@@ -9,6 +9,7 @@
 import unittest
 
 from test.test_support import run_unittest, check_py3k_warnings
+from test.test_support import check_impl_detail
 from repr import repr as r # Don't shadow builtin repr
 from repr import Repr
 
@@ -145,8 +146,11 @@
         # Functions
         eq(repr(hash), '<built-in function hash>')
         # Methods
-        self.assertTrue(repr(''.split).startswith(
-            '<built-in method split of str object at 0x'))
+        if check_impl_detail(cpython=True):
+            self.assertTrue(repr(''.split).startswith(
+                '<built-in method split of str object at 0x'))
+        elif check_impl_detail(pypy=True):
+            eq(repr(''.split), "<bound method str.split of ''>")
 
     def test_xrange(self):
         eq = self.assertEqual
@@ -185,7 +189,10 @@
     def test_descriptors(self):
         eq = self.assertEqual
         # method descriptors
-        eq(repr(dict.items), "<method 'items' of 'dict' objects>")
+        if check_impl_detail(cpython=True):
+            eq(repr(dict.items), "<method 'items' of 'dict' objects>")
+        elif check_impl_detail(pypy=True):
+            eq(repr(dict.items), "<unbound method dict.items>")
         # XXX member descriptors
         # XXX attribute descriptors
         # XXX slot descriptors
@@ -247,8 +254,14 @@
         eq = self.assertEqual
         touch(os.path.join(self.subpkgname, self.pkgname + os.extsep + 'py'))
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import areallylongpackageandmodulenametotestreprtruncation
-        eq(repr(areallylongpackageandmodulenametotestreprtruncation),
-           "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+        # On PyPy, we use %r to format the file name; on CPython it is done
+        # with '%s'.  It seems to me that %r is safer <arigo>.
+        if '__pypy__' in sys.builtin_module_names:
+            eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+               "<module %r from %r>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+        else:
+            eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+               "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
         eq(repr(sys), "<module 'sys' (built-in)>")
 
     def test_type(self):
diff --git a/lib-python/2.7/test/test_runpy.py b/lib-python/2.7/test/test_runpy.py
--- a/lib-python/2.7/test/test_runpy.py
+++ b/lib-python/2.7/test/test_runpy.py
@@ -5,10 +5,15 @@
 import sys
 import re
 import tempfile
-from test.test_support import verbose, run_unittest, forget
+from test.test_support import verbose, run_unittest, forget, check_impl_detail
 from test.script_helper import (temp_dir, make_script, compile_script,
                                 make_pkg, make_zip_script, make_zip_pkg)
 
+if check_impl_detail(pypy=True):
+    no_lone_pyc_file = True
+else:
+    no_lone_pyc_file = False
+
 
 from runpy import _run_code, _run_module_code, run_module, run_path
 # Note: This module can't safely test _run_module_as_main as it
@@ -168,13 +173,14 @@
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", mod_name
-            d2 = run_module(mod_name) # Read from bytecode
-            self.assertIn("x", d2)
-            self.assertTrue(d2["x"] == 1)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", mod_name
+                d2 = run_module(mod_name) # Read from bytecode
+                self.assertIn("x", d2)
+                self.assertTrue(d2["x"] == 1)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose: print "Module executed successfully"
@@ -190,13 +196,14 @@
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", pkg_name
-            d2 = run_module(pkg_name) # Read from bytecode
-            self.assertIn("x", d2)
-            self.assertTrue(d2["x"] == 1)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", pkg_name
+                d2 = run_module(pkg_name) # Read from bytecode
+                self.assertIn("x", d2)
+                self.assertTrue(d2["x"] == 1)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, pkg_name)
         if verbose: print "Package executed successfully"
@@ -244,15 +251,17 @@
             self.assertIn("sibling", d1)
             self.assertIn("nephew", d1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", mod_name
-            d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
-            self.assertIn("__package__", d2)
-            self.assertTrue(d2["__package__"] == pkg_name)
-            self.assertIn("sibling", d2)
-            self.assertIn("nephew", d2)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", mod_name
+                # Read from bytecode
+                d2 = run_module(mod_name, run_name=run_name)
+                self.assertIn("__package__", d2)
+                self.assertTrue(d2["__package__"] == pkg_name)
+                self.assertIn("sibling", d2)
+                self.assertIn("nephew", d2)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose: print "Module executed successfully"
@@ -345,6 +354,8 @@
                                script_dir, '')
 
     def test_directory_compiled(self):
+        if no_lone_pyc_file:
+            return
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
diff --git a/lib-python/2.7/test/test_scope.py b/lib-python/2.7/test/test_scope.py
--- a/lib-python/2.7/test/test_scope.py
+++ b/lib-python/2.7/test/test_scope.py
@@ -1,6 +1,6 @@
 import unittest
 from test.test_support import check_syntax_error, check_py3k_warnings, \
-                              check_warnings, run_unittest
+                              check_warnings, run_unittest, gc_collect
 
 
 class ScopeTests(unittest.TestCase):
@@ -432,6 +432,7 @@
 
         for i in range(100):
             f1()
+        gc_collect()
 
         self.assertEqual(Foo.count, 0)
 
diff --git a/lib-python/2.7/test/test_set.py b/lib-python/2.7/test/test_set.py
--- a/lib-python/2.7/test/test_set.py
+++ b/lib-python/2.7/test/test_set.py
@@ -309,6 +309,7 @@
             fo.close()
             test_support.unlink(test_support.TESTFN)
 
+    @test_support.impl_detail(pypy=False)
     def test_do_not_rehash_dict_keys(self):
         n = 10
         d = dict.fromkeys(map(HashCountingInt, xrange(n)))
@@ -559,6 +560,7 @@
         p = weakref.proxy(s)
         self.assertEqual(str(p), str(s))
         s = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, str, p)
 
     # C API test only available in a debug build
@@ -590,6 +592,7 @@
         s.__init__(self.otherword)
         self.assertEqual(s, set(self.word))
 
+    @test_support.impl_detail()
     def test_singleton_empty_frozenset(self):
         f = frozenset()
         efs = [frozenset(), frozenset([]), frozenset(()), frozenset(''),
@@ -770,9 +773,10 @@
         for v in self.set:
             self.assertIn(v, self.values)
         setiter = iter(self.set)
-        # note: __length_hint__ is an internal undocumented API,
-        # don't rely on it in your own programs
-        self.assertEqual(setiter.__length_hint__(), len(self.set))
+        if test_support.check_impl_detail():
+            # note: __length_hint__ is an internal undocumented API,
+            # don't rely on it in your own programs
+            self.assertEqual(setiter.__length_hint__(), len(self.set))
 
     def test_pickling(self):
         p = pickle.dumps(self.set)
@@ -1564,7 +1568,7 @@
             for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint):
                 for g in (G, I, Ig, L, R):
                     expected = meth(data)
-                    actual = meth(G(data))
+                    actual = meth(g(data))
                     if isinstance(expected, bool):
                         self.assertEqual(actual, expected)
                     else:
diff --git a/lib-python/2.7/test/test_sets.py b/lib-python/2.7/test/test_sets.py
--- a/lib-python/2.7/test/test_sets.py
+++ b/lib-python/2.7/test/test_sets.py
@@ -686,7 +686,9 @@
         set_list = sorted(self.set)
         self.assertEqual(len(dup_list), len(set_list))
         for i, el in enumerate(dup_list):
-            self.assertIs(el, set_list[i])
+            # Object identity is not guarnteed for immutable objects, so we
+            # can't use assertIs here.
+            self.assertEqual(el, set_list[i])
 
     def test_deep_copy(self):
         dup = copy.deepcopy(self.set)
diff --git a/lib-python/2.7/test/test_site.py b/lib-python/2.7/test/test_site.py
--- a/lib-python/2.7/test/test_site.py
+++ b/lib-python/2.7/test/test_site.py
@@ -226,6 +226,10 @@
             self.assertEqual(len(dirs), 1)
             wanted = os.path.join('xoxo', 'Lib', 'site-packages')
             self.assertEqual(dirs[0], wanted)
+        elif '__pypy__' in sys.builtin_module_names:
+            self.assertEquals(len(dirs), 1)
+            wanted = os.path.join('xoxo', 'site-packages')
+            self.assertEquals(dirs[0], wanted)
         elif os.sep == '/':
             self.assertEqual(len(dirs), 2)
             wanted = os.path.join('xoxo', 'lib', 'python' + sys.version[:3],
diff --git a/lib-python/2.7/test/test_socket.py b/lib-python/2.7/test/test_socket.py
--- a/lib-python/2.7/test/test_socket.py
+++ b/lib-python/2.7/test/test_socket.py
@@ -252,6 +252,7 @@
         self.assertEqual(p.fileno(), s.fileno())
         s.close()
         s = None
+        test_support.gc_collect()
         try:
             p.fileno()
         except ReferenceError:
@@ -285,32 +286,34 @@
             s.sendto(u'\u2620', sockname)
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, sockname)
-        self.assertIn('not complex', str(cm.exception))
+        self.assertIn('complex', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', None)
-        self.assertIn('not NoneType', str(cm.exception))
+        self.assertIn('NoneType', str(cm.exception))
         # 3 args
         with self.assertRaises(UnicodeEncodeError):
             s.sendto(u'\u2620', 0, sockname)
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, 0, sockname)
-        self.assertIn('not complex', str(cm.exception))
+        self.assertIn('complex', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 0, None)
-        self.assertIn('not NoneType', str(cm.exception))
+        if test_support.check_impl_detail():
+            self.assertIn('not NoneType', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 'bar', sockname)
-        self.assertIn('an integer is required', str(cm.exception))
+        self.assertIn('integer', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', None, None)
-        self.assertIn('an integer is required', str(cm.exception))
+        if test_support.check_impl_detail():
+            self.assertIn('an integer is required', str(cm.exception))
         # wrong number of args
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo')
-        self.assertIn('(1 given)', str(cm.exception))
+        self.assertIn(' given)', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 0, sockname, 4)
-        self.assertIn('(4 given)', str(cm.exception))
+        self.assertIn(' given)', str(cm.exception))
 
 
     def testCrucialConstants(self):
@@ -385,10 +388,10 @@
             socket.htonl(k)
             socket.htons(k)
         for k in bad_values:
-            self.assertRaises(OverflowError, socket.ntohl, k)
-            self.assertRaises(OverflowError, socket.ntohs, k)
-            self.assertRaises(OverflowError, socket.htonl, k)
-            self.assertRaises(OverflowError, socket.htons, k)
+            self.assertRaises((OverflowError, ValueError), socket.ntohl, k)
+            self.assertRaises((OverflowError, ValueError), socket.ntohs, k)
+            self.assertRaises((OverflowError, ValueError), socket.htonl, k)
+            self.assertRaises((OverflowError, ValueError), socket.htons, k)
 
     def testGetServBy(self):
         eq = self.assertEqual
@@ -428,8 +431,8 @@
         if udpport is not None:
             eq(socket.getservbyport(udpport, 'udp'), service)
         # Make sure getservbyport does not accept out of range ports.
-        self.assertRaises(OverflowError, socket.getservbyport, -1)
-        self.assertRaises(OverflowError, socket.getservbyport, 65536)
+        self.assertRaises((OverflowError, ValueError), socket.getservbyport, -1)
+        self.assertRaises((OverflowError, ValueError), socket.getservbyport, 65536)
 
     def testDefaultTimeout(self):
         # Testing default timeout
@@ -608,8 +611,8 @@
         neg_port = port - 65536
         sock = socket.socket()
         try:
-            self.assertRaises(OverflowError, sock.bind, (host, big_port))
-            self.assertRaises(OverflowError, sock.bind, (host, neg_port))
+            self.assertRaises((OverflowError, ValueError), sock.bind, (host, big_port))
+            self.assertRaises((OverflowError, ValueError), sock.bind, (host, neg_port))
             sock.bind((host, port))
         finally:
             sock.close()
@@ -1309,6 +1312,7 @@
             closed = False
             def flush(self): pass
             def close(self): self.closed = True
+            def _decref_socketios(self): pass
 
         # must not close unless we request it: the original use of _fileobject
         # by module socket requires that the underlying socket not be closed until
diff --git a/lib-python/2.7/test/test_sort.py b/lib-python/2.7/test/test_sort.py
--- a/lib-python/2.7/test/test_sort.py
+++ b/lib-python/2.7/test/test_sort.py
@@ -140,7 +140,10 @@
                 return random.random() < 0.5
 
         L = [C() for i in range(50)]
-        self.assertRaises(ValueError, L.sort)
+        try:
+            L.sort()
+        except ValueError:
+            pass
 
     def test_cmpNone(self):
         # Testing None as a comparison function.
@@ -150,8 +153,10 @@
         L.sort(None)
         self.assertEqual(L, range(50))
 
+    @test_support.impl_detail(pypy=False)
     def test_undetected_mutation(self):
         # Python 2.4a1 did not always detect mutation
+        # So does pypy...
         memorywaster = []
         for i in range(20):
             def mutating_cmp(x, y):
@@ -226,7 +231,10 @@
             def __del__(self):
                 del data[:]
                 data[:] = range(20)
-        self.assertRaises(ValueError, data.sort, key=SortKiller)
+        try:
+            data.sort(key=SortKiller)
+        except ValueError:
+            pass
 
     def test_key_with_mutating_del_and_exception(self):
         data = range(10)
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -881,6 +881,8 @@
                 c = socket.socket()
                 c.connect((HOST, port))
                 listener_gone.wait()
+                # XXX why is it necessary?
+                test_support.gc_collect()
                 try:
                     ssl_sock = ssl.wrap_socket(c)
                 except IOError:
@@ -1330,10 +1332,8 @@
 
 def test_main(verbose=False):
     global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT
-    CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir,
-                            "keycert.pem")
-    SVN_PYTHON_ORG_ROOT_CERT = os.path.join(
-        os.path.dirname(__file__) or os.curdir,
+    CERTFILE = test_support.findfile("keycert.pem")
+    SVN_PYTHON_ORG_ROOT_CERT = test_support.findfile(
         "https_svn_python_org_root.pem")
 
     if (not os.path.exists(CERTFILE) or
diff --git a/lib-python/2.7/test/test_str.py b/lib-python/2.7/test/test_str.py
--- a/lib-python/2.7/test/test_str.py
+++ b/lib-python/2.7/test/test_str.py
@@ -422,10 +422,11 @@
         for meth in ('foo'.startswith, 'foo'.endswith):
             with self.assertRaises(TypeError) as cm:
                 meth(['f'])
-            exc = str(cm.exception)
-            self.assertIn('unicode', exc)
-            self.assertIn('str', exc)
-            self.assertIn('tuple', exc)
+            if test_support.check_impl_detail():
+                exc = str(cm.exception)
+                self.assertIn('unicode', exc)
+                self.assertIn('str', exc)
+                self.assertIn('tuple', exc)
 
 def test_main():
     test_support.run_unittest(StrTest)
diff --git a/lib-python/2.7/test/test_struct.py b/lib-python/2.7/test/test_struct.py
--- a/lib-python/2.7/test/test_struct.py
+++ b/lib-python/2.7/test/test_struct.py
@@ -535,7 +535,8 @@
 
     @unittest.skipUnless(IS32BIT, "Specific to 32bit machines")
     def test_crasher(self):
-        self.assertRaises(MemoryError, struct.pack, "357913941c", "a")
+        self.assertRaises((MemoryError, struct.error), struct.pack,
+                          "357913941c", "a")
 
     def test_count_overflow(self):
         hugecount = '{}b'.format(sys.maxsize+1)
diff --git a/lib-python/2.7/test/test_subprocess.py b/lib-python/2.7/test/test_subprocess.py
--- a/lib-python/2.7/test/test_subprocess.py
+++ b/lib-python/2.7/test/test_subprocess.py
@@ -16,11 +16,11 @@
 # Depends on the following external programs: Python
 #
 
-if mswindows:
-    SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
-                                                'os.O_BINARY);')
-else:
-    SETBINARY = ''
+#if mswindows:
+#    SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
+#                                                'os.O_BINARY);')
+#else:
+#    SETBINARY = ''
 
 
 try:
@@ -420,8 +420,9 @@
         self.assertStderrEqual(stderr, "")
 
     def test_universal_newlines(self):
-        p = subprocess.Popen([sys.executable, "-c",
-                          'import sys,os;' + SETBINARY +
+        # NB. replaced SETBINARY with the -u flag
+        p = subprocess.Popen([sys.executable, "-u", "-c",
+                          'import sys,os;' + #SETBINARY +
                           'sys.stdout.write("line1\\n");'
                           'sys.stdout.flush();'
                           'sys.stdout.write("line2\\r");'
@@ -448,8 +449,9 @@
 
     def test_universal_newlines_communicate(self):
         # universal newlines through communicate()
-        p = subprocess.Popen([sys.executable, "-c",
-                          'import sys,os;' + SETBINARY +
+        # NB. replaced SETBINARY with the -u flag
+        p = subprocess.Popen([sys.executable, "-u", "-c",
+                          'import sys,os;' + #SETBINARY +
                           'sys.stdout.write("line1\\n");'
                           'sys.stdout.flush();'
                           'sys.stdout.write("line2\\r");'
diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py
--- a/lib-python/2.7/test/test_support.py
+++ b/lib-python/2.7/test/test_support.py
@@ -431,16 +431,20 @@
             rmtree(name)
 
 
-def findfile(file, here=__file__, subdir=None):
+def findfile(file, here=None, subdir=None):
     """Try to find a file on sys.path and the working directory.  If it is not
     found the argument passed to the function is returned (this does not
     necessarily signal failure; could still be the legitimate path)."""
+    import test
     if os.path.isabs(file):
         return file
     if subdir is not None:
         file = os.path.join(subdir, file)
     path = sys.path
-    path = [os.path.dirname(here)] + path
+    if here is None:
+        path = test.__path__ + path
+    else:
+        path = [os.path.dirname(here)] + path
     for dn in path:
         fn = os.path.join(dn, file)
         if os.path.exists(fn): return fn
@@ -1050,15 +1054,33 @@
     guards, default = _parse_guards(guards)
     return guards.get(platform.python_implementation().lower(), default)
 
+# ----------------------------------
+# PyPy extension: you can run::
+#     python ..../test_foo.py --pdb
+# to get a pdb prompt in case of exceptions
 
+ResultClass = unittest.TextTestRunner.resultclass
+
+class TestResultWithPdb(ResultClass):
+
+    def addError(self, testcase, exc_info):
+        ResultClass.addError(self, testcase, exc_info)
+        if '--pdb' in sys.argv:
+            import pdb, traceback
+            traceback.print_tb(exc_info[2])
+            pdb.post_mortem(exc_info[2])
+
+# ----------------------------------
 
 def _run_suite(suite):
     """Run tests from a unittest.TestSuite-derived class."""
     if verbose:
-        runner = unittest.TextTestRunner(sys.stdout, verbosity=2)
+        runner = unittest.TextTestRunner(sys.stdout, verbosity=2,
+                                         resultclass=TestResultWithPdb)
     else:
         runner = BasicTestRunner()
 
+
     result = runner.run(suite)
     if not result.wasSuccessful():
         if len(result.errors) == 1 and not result.failures:
@@ -1071,6 +1093,34 @@
                 err += "; run in verbose mode for details"
         raise TestFailed(err)
 
+# ----------------------------------
+# PyPy extension: you can run::
+#     python ..../test_foo.py --filter bar
+# to run only the test cases whose name contains bar
+
+def filter_maybe(suite):
+    try:
+        i = sys.argv.index('--filter')
+        filter = sys.argv[i+1]
+    except (ValueError, IndexError):
+        return suite
+    tests = []
+    for test in linearize_suite(suite):
+        if filter in test._testMethodName:
+            tests.append(test)
+    return unittest.TestSuite(tests)
+
+def linearize_suite(suite_or_test):
+    try:
+        it = iter(suite_or_test)
+    except TypeError:
+        yield suite_or_test
+        return
+    for subsuite in it:
+        for item in linearize_suite(subsuite):
+            yield item
+
+# ----------------------------------
 
 def run_unittest(*classes):
     """Run tests from unittest.TestCase-derived classes."""
@@ -1086,6 +1136,7 @@
             suite.addTest(cls)
         else:
             suite.addTest(unittest.makeSuite(cls))
+    suite = filter_maybe(suite)
     _run_suite(suite)
 
 
diff --git a/lib-python/2.7/test/test_syntax.py b/lib-python/2.7/test/test_syntax.py
--- a/lib-python/2.7/test/test_syntax.py
+++ b/lib-python/2.7/test/test_syntax.py
@@ -5,7 +5,8 @@
 >>> def f(x):
 ...     global x
 Traceback (most recent call last):
-SyntaxError: name 'x' is local and global (<doctest test.test_syntax[0]>, line 1)
+  File "<doctest test.test_syntax[0]>", line 1
+SyntaxError: name 'x' is local and global
 
 The tests are all raise SyntaxErrors.  They were created by checking
 each C call that raises SyntaxError.  There are several modules that
@@ -375,7 +376,7 @@
 In 2.5 there was a missing exception and an assert was triggered in a debug
 build.  The number of blocks must be greater than CO_MAXBLOCKS.  SF #1565514
 
-   >>> while 1:
+   >>> while 1: # doctest:+SKIP
    ...  while 2:
    ...   while 3:
    ...    while 4:
diff --git a/lib-python/2.7/test/test_sys.py b/lib-python/2.7/test/test_sys.py
--- a/lib-python/2.7/test/test_sys.py
+++ b/lib-python/2.7/test/test_sys.py
@@ -264,6 +264,7 @@
             self.assertEqual(sys.getdlopenflags(), oldflags+1)
             sys.setdlopenflags(oldflags)
 
+    @test.test_support.impl_detail("reference counting")
     def test_refcount(self):
         # n here must be a global in order for this test to pass while
         # tracing with a python function.  Tracing calls PyFrame_FastToLocals
@@ -287,7 +288,7 @@
             is sys._getframe().f_code
         )
 
-    # sys._current_frames() is a CPython-only gimmick.
+    @test.test_support.impl_detail("current_frames")
     def test_current_frames(self):
         have_threads = True
         try:
@@ -383,7 +384,10 @@
         self.assertEqual(len(sys.float_info), 11)
         self.assertEqual(sys.float_info.radix, 2)
         self.assertEqual(len(sys.long_info), 2)
-        self.assertTrue(sys.long_info.bits_per_digit % 5 == 0)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertTrue(sys.long_info.bits_per_digit % 5 == 0)
+        else:
+            self.assertTrue(sys.long_info.bits_per_digit >= 1)
         self.assertTrue(sys.long_info.sizeof_digit >= 1)
         self.assertEqual(type(sys.long_info.bits_per_digit), int)
         self.assertEqual(type(sys.long_info.sizeof_digit), int)
@@ -432,6 +436,7 @@
             self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
         self.assertTrue(repr(sys.flags))
 
+    @test.test_support.impl_detail("sys._clear_type_cache")
     def test_clear_type_cache(self):
         sys._clear_type_cache()
 
@@ -473,6 +478,7 @@
         p.wait()
         self.assertIn(executable, ["''", repr(sys.executable)])
 
+ at unittest.skipUnless(test.test_support.check_impl_detail(), "sys.getsizeof()")
 class SizeofTest(unittest.TestCase):
 
     TPFLAGS_HAVE_GC = 1<<14
diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py
--- a/lib-python/2.7/test/test_sys_settrace.py
+++ b/lib-python/2.7/test/test_sys_settrace.py
@@ -213,12 +213,16 @@
         "finally"
 def generator_example():
     # any() will leave the generator before its end
-    x = any(generator_function())
+    x = any(generator_function()); gc.collect()
 
     # the following lines were not traced
     for x in range(10):
         y = x
 
+# On CPython, when the generator is decref'ed to zero, we see the trace
+# for the "finally:" portion.  On PyPy, we don't see it before the next
+# garbage collection.  That's why we put gc.collect() on the same line above.
+
 generator_example.events = ([(0, 'call'),
                              (2, 'line'),
                              (-6, 'call'),
@@ -282,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
@@ -323,17 +327,24 @@
         self.run_test(tighterloop_example)
 
     def test_13_genexp(self):
-        self.run_test(generator_example)
-        # issue1265: if the trace function contains a generator,
-        # and if the traced function contains another generator
-        # that is not completely exhausted, the trace stopped.
-        # Worse: the 'finally' clause was not invoked.
-        tracer = Tracer()
-        sys.settrace(tracer.traceWithGenexp)
-        generator_example()
-        sys.settrace(None)
-        self.compare_events(generator_example.__code__.co_firstlineno,
-                            tracer.events, generator_example.events)
+        if self.using_gc:
+            test_support.gc_collect()
+            gc.enable()
+        try:
+            self.run_test(generator_example)
+            # issue1265: if the trace function contains a generator,
+            # and if the traced function contains another generator
+            # that is not completely exhausted, the trace stopped.
+            # Worse: the 'finally' clause was not invoked.
+            tracer = Tracer()
+            sys.settrace(tracer.traceWithGenexp)
+            generator_example()
+            sys.settrace(None)
+            self.compare_events(generator_example.__code__.co_firstlineno,
+                                tracer.events, generator_example.events)
+        finally:
+            if self.using_gc:
+                gc.disable()
 
     def test_14_onliner_if(self):
         def onliners():
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -209,13 +209,22 @@
 
         self.assertEqual(get_platform(), 'macosx-10.4-fat64')
 
-        for arch in ('ppc', 'i386', 'x86_64', 'ppc64'):
+        for arch in ('ppc', 'i386', 'ppc64', 'x86_64'):
             get_config_vars()['CFLAGS'] = ('-arch %s -isysroot '
                                            '/Developer/SDKs/MacOSX10.4u.sdk  '
                                            '-fno-strict-aliasing -fno-common '
                                            '-dynamic -DNDEBUG -g -O3'%(arch,))
 
             self.assertEqual(get_platform(), 'macosx-10.4-%s'%(arch,))
+        
+        # macosx with ARCHFLAGS set and empty _CONFIG_VARS
+        os.environ['ARCHFLAGS'] = '-arch i386'
+        sysconfig._CONFIG_VARS = None
+        
+        # this will attempt to recreate the _CONFIG_VARS based on environment 
+        # variables; used to check a problem with the PyPy's _init_posix
+        # implementation; see: issue 705
+        get_config_vars() 
 
         # linux debian sarge
         os.name = 'posix'
@@ -235,7 +244,7 @@
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
-                  'posix_home', 'posix_prefix', 'posix_user')
+                  'posix_home', 'posix_prefix', 'posix_user', 'pypy')
         self.assertEqual(get_scheme_names(), wanted)
 
     def test_symlink(self):
diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/2.7/test/test_tarfile.py
--- a/lib-python/2.7/test/test_tarfile.py
+++ b/lib-python/2.7/test/test_tarfile.py
@@ -169,6 +169,7 @@
         except tarfile.ReadError:
             self.fail("tarfile.open() failed on empty archive")
         self.assertListEqual(tar.getmembers(), [])
+        tar.close()
 
     def test_null_tarfile(self):
         # Test for issue6123: Allow opening empty archives.
@@ -207,16 +208,21 @@
         fobj = open(self.tarname, "rb")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, os.path.abspath(fobj.name))
+        tar.close()
 
     def test_no_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self.assertRaises(AttributeError, getattr, fobj, "name")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, None)
 
     def test_empty_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         fobj.name = ""
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
@@ -515,6 +521,7 @@
         self.tar = tarfile.open(self.tarname, mode=self.mode, encoding="iso8859-1")
         tarinfo = self.tar.getmember("pax/umlauts-&#65533;&#65533;&#65533;&#65533;&#65533;&#65533;&#65533;")
         self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self.tar.close()
 
 
 class LongnameTest(ReadTest):
@@ -675,6 +682,7 @@
             tar = tarfile.open(tmpname, self.mode)
             tarinfo = tar.gettarinfo(path)
             self.assertEqual(tarinfo.size, 0)
+            tar.close()
         finally:
             os.rmdir(path)
 
@@ -692,6 +700,7 @@
                 tar.gettarinfo(target)
                 tarinfo = tar.gettarinfo(link)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(target)
                 os.remove(link)
@@ -704,6 +713,7 @@
                 tar = tarfile.open(tmpname, self.mode)
                 tarinfo = tar.gettarinfo(path)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(path)
 
@@ -722,6 +732,7 @@
         tar.add(dstname)
         os.chdir(cwd)
         self.assertTrue(tar.getnames() == [], "added the archive to itself")
+        tar.close()
 
     def test_exclude(self):
         tempdir = os.path.join(TEMPDIR, "exclude")
@@ -742,6 +753,7 @@
             tar = tarfile.open(tmpname, "r")
             self.assertEqual(len(tar.getmembers()), 1)
             self.assertEqual(tar.getnames()[0], "empty_dir")
+            tar.close()
         finally:
             shutil.rmtree(tempdir)
 
@@ -947,7 +959,9 @@
             fobj.close()
         elif self.mode.endswith("bz2"):
             dec = bz2.BZ2Decompressor()
-            data = open(tmpname, "rb").read()
+            f = open(tmpname, "rb")
+            data = f.read()
+            f.close()
             data = dec.decompress(data)
             self.assertTrue(len(dec.unused_data) == 0,
                     "found trailing data")
@@ -1026,6 +1040,7 @@
                 "unable to read longname member")
         self.assertEqual(tarinfo.linkname, member.linkname,
                 "unable to read longname member")
+        tar.close()
 
     def test_longname_1023(self):
         self._test(("longnam/" * 127) + "longnam")
@@ -1118,6 +1133,7 @@
         else:
             n = tar.getmembers()[0].name
             self.assertTrue(name == n, "PAX longname creation failed")
+        tar.close()
 
     def test_pax_global_header(self):
         pax_headers = {
@@ -1146,6 +1162,7 @@
                     tarfile.PAX_NUMBER_FIELDS[key](val)
                 except (TypeError, ValueError):
                     self.fail("unable to convert pax header field")
+        tar.close()
 
     def test_pax_extended_header(self):
         # The fields from the pax header have priority over the
@@ -1165,6 +1182,7 @@
         self.assertEqual(t.pax_headers, pax_headers)
         self.assertEqual(t.name, "foo")
         self.assertEqual(t.uid, 123)
+        tar.close()
 
 
 class UstarUnicodeTest(unittest.TestCase):
@@ -1208,6 +1226,7 @@
         tarinfo.name = "foo"
         tarinfo.uname = u"&#65533;&#65533;&#65533;"
         self.assertRaises(UnicodeError, tar.addfile, tarinfo)
+        tar.close()
 
     def test_unicode_argument(self):
         tar = tarfile.open(tarname, "r", encoding="iso8859-1", errors="strict")
@@ -1262,6 +1281,7 @@
             tar = tarfile.open(tmpname, format=self.format, encoding="ascii",
                     errors=handler)
             self.assertEqual(tar.getnames()[0], name)
+            tar.close()
 
         self.assertRaises(UnicodeError, tarfile.open, tmpname,
                 encoding="ascii", errors="strict")
@@ -1274,6 +1294,7 @@
         tar = tarfile.open(tmpname, format=self.format, encoding="iso8859-1",
                 errors="utf-8")
         self.assertEqual(tar.getnames()[0], "&#65533;&#65533;&#65533;/" + u"&#65533;".encode("utf8"))
+        tar.close()
 
 
 class AppendTest(unittest.TestCase):
@@ -1301,6 +1322,7 @@
     def _test(self, names=["bar"], fileobj=None):
         tar = tarfile.open(self.tarname, fileobj=fileobj)
         self.assertEqual(tar.getnames(), names)
+        tar.close()
 
     def test_non_existing(self):
         self._add_testfile()
@@ -1319,7 +1341,9 @@
 
     def test_fileobj(self):
         self._create_testtar()
-        data = open(self.tarname).read()
+        f = open(self.tarname)
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self._add_testfile(fobj)
         fobj.seek(0)
@@ -1345,7 +1369,9 @@
     # Append mode is supposed to fail if the tarfile to append to
     # does not end with a zero block.
     def _test_error(self, data):
-        open(self.tarname, "wb").write(data)
+        f = open(self.tarname, "wb")
+        f.write(data)
+        f.close()
         self.assertRaises(tarfile.ReadError, self._add_testfile)
 
     def test_null(self):
diff --git a/lib-python/2.7/test/test_tempfile.py b/lib-python/2.7/test/test_tempfile.py
--- a/lib-python/2.7/test/test_tempfile.py
+++ b/lib-python/2.7/test/test_tempfile.py
@@ -23,8 +23,8 @@
 
 # TEST_FILES may need to be tweaked for systems depending on the maximum
 # number of files that can be opened at one time (see ulimit -n)
-if sys.platform in ('openbsd3', 'openbsd4'):
-    TEST_FILES = 48
+if sys.platform.startswith("openbsd"):
+    TEST_FILES = 64 # ulimit -n defaults to 128 for normal users
 else:
     TEST_FILES = 100
 
@@ -244,6 +244,7 @@
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir).write("blat")
+            test_support.gc_collect()
         finally:
             os.rmdir(dir)
 
@@ -528,12 +529,15 @@
         self.do_create(suf="b")
         self.do_create(pre="a", suf="b")
         self.do_create(pre="aa", suf=".txt")
+        test_support.gc_collect()
 
     def test_many(self):
         # mktemp can choose many usable file names (stochastic)
         extant = range(TEST_FILES)
         for i in extant:
             extant[i] = self.do_create(pre="aa")
+        del extant
+        test_support.gc_collect()
 
 ##     def test_warning(self):
 ##         # mktemp issues a warning when used
diff --git a/lib-python/2.7/test/test_thread.py b/lib-python/2.7/test/test_thread.py
--- a/lib-python/2.7/test/test_thread.py
+++ b/lib-python/2.7/test/test_thread.py
@@ -128,6 +128,7 @@
         del task
         while not done:
             time.sleep(0.01)
+            test_support.gc_collect()
         self.assertEqual(thread._count(), orig)
 
 
diff --git a/lib-python/2.7/test/test_threading.py b/lib-python/2.7/test/test_threading.py
--- a/lib-python/2.7/test/test_threading.py
+++ b/lib-python/2.7/test/test_threading.py
@@ -161,6 +161,7 @@
 
     # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently)
     # exposed at the Python level.  This test relies on ctypes to get at it.
+    @test.test_support.cpython_only
     def test_PyThreadState_SetAsyncExc(self):
         try:
             import ctypes
@@ -266,6 +267,7 @@
         finally:
             threading._start_new_thread = _start_new_thread
 
+    @test.test_support.cpython_only
     def test_finalize_runnning_thread(self):
         # Issue 1402: the PyGILState_Ensure / _Release functions may be called
         # very late on python exit: on deallocation of a running thread for
@@ -383,6 +385,7 @@
         finally:
             sys.setcheckinterval(old_interval)
 
+    @test.test_support.cpython_only
     def test_no_refcycle_through_target(self):
         class RunSelfFunction(object):
             def __init__(self, should_raise):
@@ -425,6 +428,9 @@
             def joiningfunc(mainthread):
                 mainthread.join()
                 print 'end of thread'
+                # stdout is fully buffered because not a tty, we have to flush
+                # before exit.
+                sys.stdout.flush()
         \n""" + script
 
         p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE)
diff --git a/lib-python/2.7/test/test_threading_local.py b/lib-python/2.7/test/test_threading_local.py
--- a/lib-python/2.7/test/test_threading_local.py
+++ b/lib-python/2.7/test/test_threading_local.py
@@ -173,8 +173,9 @@
         obj = cls()
         obj.x = 5
         self.assertEqual(obj.__dict__, {'x': 5})
-        with self.assertRaises(AttributeError):
-            obj.__dict__ = {}
+        if test_support.check_impl_detail():
+            with self.assertRaises(AttributeError):
+                obj.__dict__ = {}
         with self.assertRaises(AttributeError):
             del obj.__dict__
 
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -5,7 +5,8 @@
 import sys
 import unittest
 from imp import reload
-from test.test_support import run_unittest, is_jython, Error
+from test.test_support import run_unittest, Error
+from test.test_support import impl_detail, check_impl_detail
 
 import traceback
 
@@ -49,10 +50,8 @@
         self.assertTrue(err[2].count('\n') == 1) # and no additional newline
         self.assertTrue(err[1].find("+") == err[2].find("^")) # in the right place
 
+    @impl_detail("other implementations may add a caret (why shouldn't they?)")
     def test_nocaret(self):
-        if is_jython:
-            # jython adds a caret in this case (why shouldn't it?)
-            return
         err = self.get_exception_format(self.syntax_error_without_caret,
                                         SyntaxError)
         self.assertTrue(len(err) == 3)
@@ -63,8 +62,11 @@
                                         IndentationError)
         self.assertTrue(len(err) == 4)
         self.assertTrue(err[1].strip() == "print 2")
-        self.assertIn("^", err[2])
-        self.assertTrue(err[1].find("2") == err[2].find("^"))
+        if check_impl_detail():
+            # on CPython, there is a "^" at the end of the line
+            # on PyPy, there is a "^" too, but at the start, more logically
+            self.assertIn("^", err[2])
+            self.assertTrue(err[1].find("2") == err[2].find("^"))
 
     def test_bug737473(self):
         import os, tempfile, time
@@ -74,7 +76,8 @@
         try:
             sys.path.insert(0, testdir)
             testfile = os.path.join(testdir, 'test_bug737473.py')
-            print >> open(testfile, 'w'), """
+            with open(testfile, 'w') as f:
+                print >> f, """
 def test():
     raise ValueError"""
 
@@ -96,7 +99,8 @@
             # three seconds are needed for this test to pass reliably :-(
             time.sleep(4)
 
-            print >> open(testfile, 'w'), """
+            with open(testfile, 'w') as f:
+                print >> f, """
 def test():
     raise NotImplementedError"""
             reload(test_bug737473)
diff --git a/lib-python/2.7/test/test_types.py b/lib-python/2.7/test/test_types.py
--- a/lib-python/2.7/test/test_types.py
+++ b/lib-python/2.7/test/test_types.py
@@ -1,7 +1,8 @@
 # Python test set -- part 6, built-in types
 
 from test.test_support import run_unittest, have_unicode, run_with_locale, \
-                              check_py3k_warnings
+                              check_py3k_warnings, \
+                              impl_detail, check_impl_detail
 import unittest
 import sys
 import locale
@@ -289,9 +290,14 @@
         # array.array() returns an object that does not implement a char buffer,
         # something which int() uses for conversion.
         import array
-        try: int(buffer(array.array('c')))
+        try: int(buffer(array.array('c', '5')))
         except TypeError: pass
-        else: self.fail("char buffer (at C level) not working")
+        else:
+            if check_impl_detail():
+                self.fail("char buffer (at C level) not working")
+            #else:
+            #   it works on PyPy, which does not have the distinction
+            #   between char buffer and binary buffer.  XXX fine enough?
 
     def test_int__format__(self):
         def test(i, format_spec, result):
@@ -741,6 +747,7 @@
         for code in 'xXobns':
             self.assertRaises(ValueError, format, 0, ',' + code)
 
+    @impl_detail("the types' internal size attributes are CPython-only")
     def test_internal_sizes(self):
         self.assertGreater(object.__basicsize__, 0)
         self.assertGreater(tuple.__itemsize__, 0)
diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -448,10 +448,11 @@
                 meth('\xff')
             with self.assertRaises(TypeError) as cm:
                 meth(['f'])
-            exc = str(cm.exception)
-            self.assertIn('unicode', exc)
-            self.assertIn('str', exc)
-            self.assertIn('tuple', exc)
+            if test_support.check_impl_detail():
+                exc = str(cm.exception)
+                self.assertIn('unicode', exc)
+                self.assertIn('str', exc)
+                self.assertIn('tuple', exc)
 
     @test_support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
     def test_format_float(self):
@@ -1062,7 +1063,8 @@
         # to take a 64-bit long, this test should apply to all platforms.
         if sys.maxint > (1 << 32) or struct.calcsize('P') != 4:
             return
-        self.assertRaises(OverflowError, u't\tt\t'.expandtabs, sys.maxint)
+        self.assertRaises((OverflowError, MemoryError),
+                          u't\tt\t'.expandtabs, sys.maxint)
 
     def test__format__(self):
         def test(value, format, expected):
diff --git a/lib-python/2.7/test/test_unicodedata.py b/lib-python/2.7/test/test_unicodedata.py
--- a/lib-python/2.7/test/test_unicodedata.py
+++ b/lib-python/2.7/test/test_unicodedata.py
@@ -233,10 +233,12 @@
         # been loaded in this process.
         popen = subprocess.Popen(args, stderr=subprocess.PIPE)
         popen.wait()
-        self.assertEqual(popen.returncode, 1)
-        error = "SyntaxError: (unicode error) \N escapes not supported " \
-            "(can't load unicodedata module)"
-        self.assertIn(error, popen.stderr.read())
+        self.assertIn(popen.returncode, [0, 1]) # at least it did not segfault
+        if test.test_support.check_impl_detail():
+            self.assertEqual(popen.returncode, 1)
+            error = "SyntaxError: (unicode error) \N escapes not supported " \
+                "(can't load unicodedata module)"
+            self.assertIn(error, popen.stderr.read())
 
     def test_decimal_numeric_consistent(self):
         # Test that decimal and numeric are consistent,
diff --git a/lib-python/2.7/test/test_unpack.py b/lib-python/2.7/test/test_unpack.py
--- a/lib-python/2.7/test/test_unpack.py
+++ b/lib-python/2.7/test/test_unpack.py
@@ -62,14 +62,14 @@
     >>> a, b = t
     Traceback (most recent call last):
       ...
-    ValueError: too many values to unpack
+    ValueError: expected length 2, got 3
 
 Unpacking tuple of wrong size
 
     >>> a, b = l
     Traceback (most recent call last):
       ...
-    ValueError: too many values to unpack
+    ValueError: expected length 2, got 3
 
 Unpacking sequence too short
 
diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py
--- a/lib-python/2.7/test/test_urllib2.py
+++ b/lib-python/2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/2.7/test/test_warnings.py b/lib-python/2.7/test/test_warnings.py
--- a/lib-python/2.7/test/test_warnings.py
+++ b/lib-python/2.7/test/test_warnings.py
@@ -355,7 +355,8 @@
     # test_support.import_fresh_module utility function
     def test_accelerated(self):
         self.assertFalse(original_warnings is self.module)
-        self.assertFalse(hasattr(self.module.warn, 'func_code'))
+        self.assertFalse(hasattr(self.module.warn, 'func_code') and
+                         hasattr(self.module.warn.func_code, 'co_filename'))
 
 class PyWarnTests(BaseTest, WarnTests):
     module = py_warnings
@@ -364,7 +365,8 @@
     # test_support.import_fresh_module utility function
     def test_pure_python(self):
         self.assertFalse(original_warnings is self.module)
-        self.assertTrue(hasattr(self.module.warn, 'func_code'))
+        self.assertTrue(hasattr(self.module.warn, 'func_code') and
+                        hasattr(self.module.warn.func_code, 'co_filename'))
 
 
 class WCmdLineTests(unittest.TestCase):
diff --git a/lib-python/2.7/test/test_weakref.py b/lib-python/2.7/test/test_weakref.py
--- a/lib-python/2.7/test/test_weakref.py
+++ b/lib-python/2.7/test/test_weakref.py
@@ -1,4 +1,3 @@
-import gc
 import sys
 import unittest
 import UserList
@@ -6,6 +5,7 @@
 import operator
 
 from test import test_support
+from test.test_support import gc_collect
 
 # Used in ReferencesTestCase.test_ref_created_during_del() .
 ref_from_del = None
@@ -70,6 +70,7 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del o
+        gc_collect()
         self.assertTrue(ref1() is None,
                      "expected reference to be invalidated")
         self.assertTrue(ref2() is None,
@@ -101,13 +102,16 @@
         ref1 = weakref.proxy(o, self.callback)
         ref2 = weakref.proxy(o, self.callback)
         del o
+        gc_collect()
 
         def check(proxy):
             proxy.bar
 
         self.assertRaises(weakref.ReferenceError, check, ref1)
         self.assertRaises(weakref.ReferenceError, check, ref2)
-        self.assertRaises(weakref.ReferenceError, bool, weakref.proxy(C()))
+        ref3 = weakref.proxy(C())
+        gc_collect()
+        self.assertRaises(weakref.ReferenceError, bool, ref3)
         self.assertTrue(self.cbcalled == 2)
 
     def check_basic_ref(self, factory):
@@ -124,6 +128,7 @@
         o = factory()
         ref = weakref.ref(o, self.callback)
         del o
+        gc_collect()
         self.assertTrue(self.cbcalled == 1,
                      "callback did not properly set 'cbcalled'")
         self.assertTrue(ref() is None,
@@ -148,6 +153,7 @@
         self.assertTrue(weakref.getweakrefcount(o) == 2,
                      "wrong weak ref count for object")
         del proxy
+        gc_collect()
         self.assertTrue(weakref.getweakrefcount(o) == 1,
                      "wrong weak ref count for object after deleting proxy")
 
@@ -325,6 +331,7 @@
                      "got wrong number of weak reference objects")
 
         del ref1, ref2, proxy1, proxy2
+        gc_collect()
         self.assertTrue(weakref.getweakrefcount(o) == 0,
                      "weak reference objects not unlinked from"
                      " referent when discarded.")
@@ -338,6 +345,7 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del ref1
+        gc_collect()
         self.assertTrue(weakref.getweakrefs(o) == [ref2],
                      "list of refs does not match")
 
@@ -345,10 +353,12 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del ref2
+        gc_collect()
         self.assertTrue(weakref.getweakrefs(o) == [ref1],
                      "list of refs does not match")
 
         del ref1
+        gc_collect()
         self.assertTrue(weakref.getweakrefs(o) == [],
                      "list of refs not cleared")
 
@@ -400,13 +410,11 @@
         # when the second attempt to remove the instance from the "list
         # of all objects" occurs.
 
-        import gc
-
         class C(object):
             pass
 
         c = C()
-        wr = weakref.ref(c, lambda ignore: gc.collect())
+        wr = weakref.ref(c, lambda ignore: gc_collect())
         del c
 
         # There endeth the first part.  It gets worse.
@@ -414,7 +422,7 @@
 
         c1 = C()
         c1.i = C()
-        wr = weakref.ref(c1.i, lambda ignore: gc.collect())
+        wr = weakref.ref(c1.i, lambda ignore: gc_collect())
 
         c2 = C()
         c2.c1 = c1
@@ -430,8 +438,6 @@
         del c2
 
     def test_callback_in_cycle_1(self):
-        import gc
-
         class J(object):
             pass
 
@@ -467,11 +473,9 @@
         # search II.__mro__, but that's NULL.   The result was a segfault in
         # a release build, and an assert failure in a debug build.
         del I, J, II
-        gc.collect()
+        gc_collect()
 
     def test_callback_in_cycle_2(self):
-        import gc
-
         # This is just like test_callback_in_cycle_1, except that II is an
         # old-style class.  The symptom is different then:  an instance of an
         # old-style class looks in its own __dict__ first.  'J' happens to
@@ -496,11 +500,9 @@
         I.wr = weakref.ref(J, I.acallback)
 
         del I, J, II
-        gc.collect()
+        gc_collect()
 
     def test_callback_in_cycle_3(self):
-        import gc
-
         # This one broke the first patch that fixed the last two.  In this
         # case, the objects reachable from the callback aren't also reachable
         # from the object (c1) *triggering* the callback:  you can get to
@@ -520,11 +522,9 @@
         c2.wr = weakref.ref(c1, c2.cb)
 
         del c1, c2
-        gc.collect()
+        gc_collect()
 
     def test_callback_in_cycle_4(self):
-        import gc
-
         # Like test_callback_in_cycle_3, except c2 and c1 have different
         # classes.  c2's class (C) isn't reachable from c1 then, so protecting
         # objects reachable from the dying object (c1) isn't enough to stop
@@ -548,11 +548,9 @@
         c2.wr = weakref.ref(c1, c2.cb)
 
         del c1, c2, C, D
-        gc.collect()
+        gc_collect()
 
     def test_callback_in_cycle_resurrection(self):
-        import gc
-
         # Do something nasty in a weakref callback:  resurrect objects
         # from dead cycles.  For this to be attempted, the weakref and
         # its callback must also be part of the cyclic trash (else the
@@ -583,7 +581,7 @@
         del c1, c2, C   # make them all trash
         self.assertEqual(alist, [])  # del isn't enough to reclaim anything
 
-        gc.collect()
+        gc_collect()
         # c1.wr and c2.wr were part of the cyclic trash, so should have
         # been cleared without their callbacks executing.  OTOH, the weakref
         # to C is bound to a function local (wr), and wasn't trash, so that
@@ -593,12 +591,10 @@
         self.assertEqual(wr(), None)
 
         del alist[:]
-        gc.collect()
+        gc_collect()
         self.assertEqual(alist, [])
 
     def test_callbacks_on_callback(self):
-        import gc
-
         # Set up weakref callbacks *on* weakref callbacks.
         alist = []
         def safe_callback(ignore):
@@ -626,12 +622,12 @@
 
         del callback, c, d, C
         self.assertEqual(alist, [])  # del isn't enough to clean up cycles
-        gc.collect()
+        gc_collect()
         self.assertEqual(alist, ["safe_callback called"])
         self.assertEqual(external_wr(), None)
 
         del alist[:]
-        gc.collect()
+        gc_collect()
         self.assertEqual(alist, [])
 
     def test_gc_during_ref_creation(self):
@@ -641,9 +637,11 @@
         self.check_gc_during_creation(weakref.proxy)
 
     def check_gc_during_creation(self, makeref):
-        thresholds = gc.get_threshold()
-        gc.set_threshold(1, 1, 1)
-        gc.collect()
+        if test_support.check_impl_detail():
+            import gc
+            thresholds = gc.get_threshold()
+            gc.set_threshold(1, 1, 1)
+        gc_collect()
         class A:
             pass
 
@@ -663,7 +661,8 @@
             weakref.ref(referenced, callback)
 
         finally:
-            gc.set_threshold(*thresholds)
+            if test_support.check_impl_detail():
+                gc.set_threshold(*thresholds)
 
     def test_ref_created_during_del(self):
         # Bug #1377858
@@ -683,7 +682,7 @@
         r = weakref.ref(Exception)
         self.assertRaises(TypeError, r.__init__, 0, 0, 0, 0, 0)
         # No exception should be raised here
-        gc.collect()
+        gc_collect()
 
     def test_classes(self):
         # Check that both old-style classes and new-style classes
@@ -696,12 +695,12 @@
         weakref.ref(int)
         a = weakref.ref(A, l.append)
         A = None
-        gc.collect()
+        gc_collect()
         self.assertEqual(a(), None)
         self.assertEqual(l, [a])
         b = weakref.ref(B, l.append)
         B = None
-        gc.collect()
+        gc_collect()
         self.assertEqual(b(), None)
         self.assertEqual(l, [a, b])
 
@@ -722,6 +721,7 @@
         self.assertTrue(mr.called)
         self.assertEqual(mr.value, 24)
         del o
+        gc_collect()
         self.assertTrue(mr() is None)
         self.assertTrue(mr.called)
 
@@ -738,9 +738,11 @@
         self.assertEqual(weakref.getweakrefcount(o), 3)
         refs = weakref.getweakrefs(o)
         self.assertEqual(len(refs), 3)
-        self.assertTrue(r2 is refs[0])
-        self.assertIn(r1, refs[1:])
-        self.assertIn(r3, refs[1:])
+        assert set(refs) == set((r1, r2, r3))
+        if test_support.check_impl_detail():
+            self.assertTrue(r2 is refs[0])
+            self.assertIn(r1, refs[1:])
+            self.assertIn(r3, refs[1:])
 
     def test_subclass_refs_dont_conflate_callbacks(self):
         class MyRef(weakref.ref):
@@ -839,15 +841,18 @@
         del items1, items2
         self.assertTrue(len(dict) == self.COUNT)
         del objects[0]
+        gc_collect()
         self.assertTrue(len(dict) == (self.COUNT - 1),
                      "deleting object did not cause dictionary update")
         del objects, o
+        gc_collect()
         self.assertTrue(len(dict) == 0,
                      "deleting the values did not clear the dictionary")
         # regression on SF bug #447152:
         dict = weakref.WeakValueDictionary()
         self.assertRaises(KeyError, dict.__getitem__, 1)
         dict[2] = C()
+        gc_collect()
         self.assertRaises(KeyError, dict.__getitem__, 2)
 
     def test_weak_keys(self):
@@ -868,9 +873,11 @@
         del items1, items2
         self.assertTrue(len(dict) == self.COUNT)
         del objects[0]
+        gc_collect()
         self.assertTrue(len(dict) == (self.COUNT - 1),
                      "deleting object did not cause dictionary update")
         del objects, o
+        gc_collect()
         self.assertTrue(len(dict) == 0,
                      "deleting the keys did not clear the dictionary")
         o = Object(42)
@@ -986,13 +993,13 @@
         self.assertTrue(len(weakdict) == 2)
         k, v = weakdict.popitem()
         self.assertTrue(len(weakdict) == 1)
-        if k is key1:
+        if k == key1:
             self.assertTrue(v is value1)
         else:
             self.assertTrue(v is value2)
         k, v = weakdict.popitem()
         self.assertTrue(len(weakdict) == 0)
-        if k is key1:
+        if k == key1:
             self.assertTrue(v is value1)
         else:
             self.assertTrue(v is value2)
@@ -1137,6 +1144,7 @@
         for o in objs:
             count += 1
             del d[o]
+        gc_collect()
         self.assertEqual(len(d), 0)
         self.assertEqual(count, 2)
 
@@ -1177,6 +1185,7 @@
 >>> o is o2
 True
 >>> del o, o2
+>>> gc_collect()
 >>> print r()
 None
 
@@ -1229,6 +1238,7 @@
 >>> id2obj(a_id) is a
 True
 >>> del a
+>>> gc_collect()
 >>> try:
 ...     id2obj(a_id)
 ... except KeyError:
diff --git a/lib-python/2.7/test/test_weakset.py b/lib-python/2.7/test/test_weakset.py
--- a/lib-python/2.7/test/test_weakset.py
+++ b/lib-python/2.7/test/test_weakset.py
@@ -57,6 +57,7 @@
         self.assertEqual(len(self.s), len(self.d))
         self.assertEqual(len(self.fs), 1)
         del self.obj
+        test_support.gc_collect()
         self.assertEqual(len(self.fs), 0)
 
     def test_contains(self):
@@ -66,6 +67,7 @@
         self.assertNotIn(1, self.s)
         self.assertIn(self.obj, self.fs)
         del self.obj
+        test_support.gc_collect()
         self.assertNotIn(SomeClass('F'), self.fs)
 
     def test_union(self):
@@ -204,6 +206,7 @@
         self.assertEqual(self.s, dup)
         self.assertRaises(TypeError, self.s.add, [])
         self.fs.add(Foo())
+        test_support.gc_collect()
         self.assertTrue(len(self.fs) == 1)
         self.fs.add(self.obj)
         self.assertTrue(len(self.fs) == 1)
@@ -330,10 +333,11 @@
         next(it)             # Trigger internal iteration
         # Destroy an item
         del items[-1]
-        gc.collect()    # just in case
+        test_support.gc_collect()
         # We have removed either the first consumed items, or another one
         self.assertIn(len(list(it)), [len(items), len(items) - 1])
         del it
+        test_support.gc_collect()
         # The removal has been committed
         self.assertEqual(len(s), len(items))
 
diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -1633,10 +1633,10 @@
 
     Check reference leak.
     >>> xmltoolkit63()
-    >>> count = sys.getrefcount(None)
+    >>> count = sys.getrefcount(None)  #doctest: +SKIP
     >>> for i in range(1000):
     ...     xmltoolkit63()
-    >>> sys.getrefcount(None) - count
+    >>> sys.getrefcount(None) - count  #doctest: +SKIP
     0
 
     """
diff --git a/lib-python/2.7/test/test_xmlrpc.py b/lib-python/2.7/test/test_xmlrpc.py
--- a/lib-python/2.7/test/test_xmlrpc.py
+++ b/lib-python/2.7/test/test_xmlrpc.py
@@ -308,7 +308,7 @@
         global ADDR, PORT, URL
         ADDR, PORT = serv.socket.getsockname()
         #connect to IP address directly.  This avoids socket.create_connection()
-        #trying to connect to "localhost" using all address families, which
+        #trying to connect to to "localhost" using all address families, which
         #causes slowdown e.g. on vista which supports AF_INET6.  The server listens
         #on AF_INET only.
         URL = "http://%s:%d"%(ADDR, PORT)
@@ -367,7 +367,7 @@
         global ADDR, PORT, URL
         ADDR, PORT = serv.socket.getsockname()
         #connect to IP address directly.  This avoids socket.create_connection()
-        #trying to connect to "localhost" using all address families, which
+        #trying to connect to to "localhost" using all address families, which
         #causes slowdown e.g. on vista which supports AF_INET6.  The server listens
         #on AF_INET only.
         URL = "http://%s:%d"%(ADDR, PORT)
@@ -435,6 +435,7 @@
 
     def tearDown(self):
         # wait on the server thread to terminate
+        test_support.gc_collect() # to close the active connections
         self.evt.wait(10)
 
         # disable traceback reporting
@@ -472,9 +473,6 @@
                 # protocol error; provide additional information in test output
                 self.fail("%s\n%s" % (e, getattr(e, "headers", "")))
 
-    def test_unicode_host(self):
-        server = xmlrpclib.ServerProxy(u"http://%s:%d/RPC2"%(ADDR, PORT))
-        self.assertEqual(server.add("a", u"\xe9"), u"a\xe9")
 
     # [ch] The test 404 is causing lots of false alarms.
     def XXXtest_404(self):
@@ -589,12 +587,6 @@
         # This avoids waiting for the socket timeout.
         self.test_simple1()
 
-    def test_partial_post(self):
-        # Check that a partial POST doesn't make the server loop: issue #14001.
-        conn = httplib.HTTPConnection(ADDR, PORT)
-        conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye')
-        conn.close()
-
 class MultiPathServerTestCase(BaseServerTestCase):
     threadFunc = staticmethod(http_multi_server)
     request_count = 2
diff --git a/lib-python/2.7/test/test_zlib.py b/lib-python/2.7/test/test_zlib.py
--- a/lib-python/2.7/test/test_zlib.py
+++ b/lib-python/2.7/test/test_zlib.py
@@ -1,6 +1,7 @@
 import unittest
 from test.test_support import TESTFN, run_unittest, import_module, unlink, requires
 import binascii
+import os
 import random
 from test.test_support import precisionbigmemtest, _1G, _4G
 import sys
@@ -99,14 +100,7 @@
 
 class BaseCompressTestCase(object):
     def check_big_compress_buffer(self, size, compress_func):
-        _1M = 1024 * 1024
-        fmt = "%%0%dx" % (2 * _1M)
-        # Generate 10MB worth of random, and expand it by repeating it.
-        # The assumption is that zlib's memory is not big enough to exploit
-        # such spread out redundancy.
-        data = ''.join([binascii.a2b_hex(fmt % random.getrandbits(8 * _1M))
-                        for i in range(10)])
-        data = data * (size // len(data) + 1)
+        data = os.urandom(size)
         try:
             compress_func(data)
         finally:
diff --git a/lib-python/2.7/trace.py b/lib-python/2.7/trace.py
--- a/lib-python/2.7/trace.py
+++ b/lib-python/2.7/trace.py
@@ -559,6 +559,10 @@
             if len(funcs) == 1:
                 dicts = [d for d in gc.get_referrers(funcs[0])
                              if isinstance(d, dict)]
+                if len(dicts) == 0:
+                    # PyPy may store functions directly on the class
+                    # (more exactly: the container is not a Python object)
+                    dicts = funcs
                 if len(dicts) == 1:
                     classes = [c for c in gc.get_referrers(dicts[0])
                                    if hasattr(c, "__bases__")]
diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py
--- a/lib-python/2.7/urllib2.py
+++ b/lib-python/2.7/urllib2.py
@@ -1171,6 +1171,7 @@
             except TypeError: #buffering kw not supported
                 r = h.getresponse()
         except socket.error, err: # XXX what error?
+            h.close()
             raise URLError(err)
 
         # Pick apart the HTTPResponse object to get the addinfourl
diff --git a/lib-python/2.7/uuid.py b/lib-python/2.7/uuid.py
--- a/lib-python/2.7/uuid.py
+++ b/lib-python/2.7/uuid.py
@@ -406,8 +406,12 @@
             continue
         if hasattr(lib, 'uuid_generate_random'):
             _uuid_generate_random = lib.uuid_generate_random
+            _uuid_generate_random.argtypes = [ctypes.c_char * 16]
+            _uuid_generate_random.restype = None
         if hasattr(lib, 'uuid_generate_time'):
             _uuid_generate_time = lib.uuid_generate_time
+            _uuid_generate_time.argtypes = [ctypes.c_char * 16]
+            _uuid_generate_time.restype = None
 
     # The uuid_generate_* functions are broken on MacOS X 10.5, as noted
     # in issue #8621 the function generates the same sequence of values
@@ -436,6 +440,9 @@
         lib = None
     _UuidCreate = getattr(lib, 'UuidCreateSequential',
                           getattr(lib, 'UuidCreate', None))
+    if _UuidCreate is not None:
+        _UuidCreate.argtypes = [ctypes.c_char * 16]
+        _UuidCreate.restype = ctypes.c_int
 except:
     pass
 
diff --git a/lib-python/modified-2.7/UserDict.py b/lib-python/modified-2.7/UserDict.py
deleted file mode 100644
--- a/lib-python/modified-2.7/UserDict.py
+++ /dev/null
@@ -1,189 +0,0 @@
-"""A more or less complete user-defined wrapper around dictionary objects."""
-
-# XXX This is a bit of a hack (as usual :-))
-# the actual content of the file is not changed, but we put it here to make
-# virtualenv happy (because its internal logic expects at least one of the
-# REQUIRED_MODULES to be in modified-*)
-
-class UserDict:
-    def __init__(self, dict=None, **kwargs):
-        self.data = {}
-        if dict is not None:
-            self.update(dict)
-        if len(kwargs):
-            self.update(kwargs)
-    def __repr__(self): return repr(self.data)
-    def __cmp__(self, dict):
-        if isinstance(dict, UserDict):
-            return cmp(self.data, dict.data)
-        else:
-            return cmp(self.data, dict)
-    __hash__ = None # Avoid Py3k warning
-    def __len__(self): return len(self.data)
-    def __getitem__(self, key):
-        if key in self.data:
-            return self.data[key]
-        if hasattr(self.__class__, "__missing__"):
-            return self.__class__.__missing__(self, key)
-        raise KeyError(key)
-    def __setitem__(self, key, item): self.data[key] = item
-    def __delitem__(self, key): del self.data[key]
-    def clear(self): self.data.clear()
-    def copy(self):
-        if self.__class__ is UserDict:
-            return UserDict(self.data.copy())
-        import copy
-        data = self.data
-        try:
-            self.data = {}
-            c = copy.copy(self)
-        finally:
-            self.data = data
-        c.update(self)
-        return c
-    def keys(self): return self.data.keys()
-    def items(self): return self.data.items()
-    def iteritems(self): return self.data.iteritems()
-    def iterkeys(self): return self.data.iterkeys()
-    def itervalues(self): return self.data.itervalues()
-    def values(self): return self.data.values()
-    def has_key(self, key): return key in self.data
-    def update(self, dict=None, **kwargs):
-        if dict is None:
-            pass
-        elif isinstance(dict, UserDict):
-            self.data.update(dict.data)
-        elif isinstance(dict, type({})) or not hasattr(dict, 'items'):
-            self.data.update(dict)
-        else:
-            for k, v in dict.items():
-                self[k] = v
-        if len(kwargs):
-            self.data.update(kwargs)
-    def get(self, key, failobj=None):
-        if key not in self:
-            return failobj
-        return self[key]
-    def setdefault(self, key, failobj=None):
-        if key not in self:
-            self[key] = failobj
-        return self[key]
-    def pop(self, key, *args):
-        return self.data.pop(key, *args)
-    def popitem(self):
-        return self.data.popitem()
-    def __contains__(self, key):
-        return key in self.data
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        d = cls()
-        for key in iterable:
-            d[key] = value
-        return d
-
-class IterableUserDict(UserDict):
-    def __iter__(self):
-        return iter(self.data)
-
-try:
-    import _abcoll
-except ImportError:
-    pass    # e.g. no '_weakref' module on this pypy
-else:
-    _abcoll.MutableMapping.register(IterableUserDict)
-
-
-class DictMixin:
-    # Mixin defining all dictionary methods for classes that already have
-    # a minimum dictionary interface including getitem, setitem, delitem,
-    # and keys. Without knowledge of the subclass constructor, the mixin
-    # does not define __init__() or copy().  In addition to the four base
-    # methods, progressively more efficiency comes with defining
-    # __contains__(), __iter__(), and iteritems().
-
-    # second level definitions support higher levels
-    def __iter__(self):
-        for k in self.keys():
-            yield k
-    def has_key(self, key):
-        try:
-            self[key]
-        except KeyError:
-            return False
-        return True
-    def __contains__(self, key):
-        return self.has_key(key)
-
-    # third level takes advantage of second level definitions
-    def iteritems(self):
-        for k in self:
-            yield (k, self[k])
-    def iterkeys(self):
-        return self.__iter__()
-
-    # fourth level uses definitions from lower levels
-    def itervalues(self):
-        for _, v in self.iteritems():
-            yield v
-    def values(self):
-        return [v for _, v in self.iteritems()]
-    def items(self):
-        return list(self.iteritems())
-    def clear(self):
-        for key in self.keys():
-            del self[key]
-    def setdefault(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            self[key] = default
-        return default
-    def pop(self, key, *args):
-        if len(args) > 1:
-            raise TypeError, "pop expected at most 2 arguments, got "\
-                              + repr(1 + len(args))
-        try:
-            value = self[key]
-        except KeyError:
-            if args:
-                return args[0]
-            raise
-        del self[key]
-        return value
-    def popitem(self):
-        try:
-            k, v = self.iteritems().next()
-        except StopIteration:
-            raise KeyError, 'container is empty'
-        del self[k]
-        return (k, v)
-    def update(self, other=None, **kwargs):
-        # Make progressively weaker assumptions about "other"
-        if other is None:
-            pass
-        elif hasattr(other, 'iteritems'):  # iteritems saves memory and lookups
-            for k, v in other.iteritems():
-                self[k] = v
-        elif hasattr(other, 'keys'):
-            for k in other.keys():
-                self[k] = other[k]
-        else:
-            for k, v in other:
-                self[k] = v
-        if kwargs:
-            self.update(kwargs)
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-    def __repr__(self):
-        return repr(dict(self.iteritems()))
-    def __cmp__(self, other):
-        if other is None:
-            return 1
-        if isinstance(other, DictMixin):
-            other = dict(other.iteritems())
-        return cmp(dict(self.iteritems()), other)
-    def __len__(self):
-        return len(self.keys())
diff --git a/lib-python/modified-2.7/_threading_local.py b/lib-python/modified-2.7/_threading_local.py
deleted file mode 100644
--- a/lib-python/modified-2.7/_threading_local.py
+++ /dev/null
@@ -1,251 +0,0 @@
-"""Thread-local objects.
-
-(Note that this module provides a Python version of the threading.local
- class.  Depending on the version of Python you're using, there may be a
- faster one available.  You should always import the `local` class from
- `threading`.)
-
-Thread-local objects support the management of thread-local data.
-If you have data that you want to be local to a thread, simply create
-a thread-local object and use its attributes:
-
-  >>> mydata = local()
-  >>> mydata.number = 42
-  >>> mydata.number
-  42
-
-You can also access the local-object's dictionary:
-
-  >>> mydata.__dict__
-  {'number': 42}
-  >>> mydata.__dict__.setdefault('widgets', [])
-  []
-  >>> mydata.widgets
-  []
-
-What's important about thread-local objects is that their data are
-local to a thread. If we access the data in a different thread:
-
-  >>> log = []
-  >>> def f():
-  ...     items = mydata.__dict__.items()
-  ...     items.sort()
-  ...     log.append(items)
-  ...     mydata.number = 11
-  ...     log.append(mydata.number)
-
-  >>> import threading
-  >>> thread = threading.Thread(target=f)
-  >>> thread.start()
-  >>> thread.join()
-  >>> log
-  [[], 11]
-
-we get different data.  Furthermore, changes made in the other thread
-don't affect data seen in this thread:
-
-  >>> mydata.number
-  42
-
-Of course, values you get from a local object, including a __dict__
-attribute, are for whatever thread was current at the time the
-attribute was read.  For that reason, you generally don't want to save
-these values across threads, as they apply only to the thread they
-came from.
-
-You can create custom local objects by subclassing the local class:
-
-  >>> class MyLocal(local):
-  ...     number = 2
-  ...     initialized = False
-  ...     def __init__(self, **kw):
-  ...         if self.initialized:
-  ...             raise SystemError('__init__ called too many times')
-  ...         self.initialized = True
-  ...         self.__dict__.update(kw)
-  ...     def squared(self):
-  ...         return self.number ** 2
-
-This can be useful to support default values, methods and
-initialization.  Note that if you define an __init__ method, it will be
-called each time the local object is used in a separate thread.  This
-is necessary to initialize each thread's dictionary.
-
-Now if we create a local object:
-
-  >>> mydata = MyLocal(color='red')
-
-Now we have a default number:
-
-  >>> mydata.number
-  2
-
-an initial color:
-
-  >>> mydata.color
-  'red'
-  >>> del mydata.color
-
-And a method that operates on the data:
-
-  >>> mydata.squared()
-  4
-
-As before, we can access the data in a separate thread:
-
-  >>> log = []
-  >>> thread = threading.Thread(target=f)
-  >>> thread.start()
-  >>> thread.join()
-  >>> log
-  [[('color', 'red'), ('initialized', True)], 11]
-
-without affecting this thread's data:
-
-  >>> mydata.number
-  2
-  >>> mydata.color
-  Traceback (most recent call last):
-  ...
-  AttributeError: 'MyLocal' object has no attribute 'color'
-
-Note that subclasses can define slots, but they are not thread
-local. They are shared across threads:
-
-  >>> class MyLocal(local):
-  ...     __slots__ = 'number'
-
-  >>> mydata = MyLocal()
-  >>> mydata.number = 42
-  >>> mydata.color = 'red'
-
-So, the separate thread:
-
-  >>> thread = threading.Thread(target=f)
-  >>> thread.start()
-  >>> thread.join()
-
-affects what we see:
-
-  >>> mydata.number
-  11
-
->>> del mydata
-"""
-
-__all__ = ["local"]
-
-# We need to use objects from the threading module, but the threading
-# module may also want to use our `local` class, if support for locals
-# isn't compiled in to the `thread` module.  This creates potential problems
-# with circular imports.  For that reason, we don't import `threading`
-# until the bottom of this file (a hack sufficient to worm around the
-# potential problems).  Note that almost all platforms do have support for
-# locals in the `thread` module, and there is no circular import problem
-# then, so problems introduced by fiddling the order of imports here won't
-# manifest on most boxes.
-
-class _localbase(object):
-    __slots__ = '_local__key', '_local__args', '_local__lock'
-
-    def __new__(cls, *args, **kw):
-        self = object.__new__(cls)
-        key = '_local__key', 'thread.local.' + str(id(self))
-        object.__setattr__(self, '_local__key', key)
-        object.__setattr__(self, '_local__args', (args, kw))
-        object.__setattr__(self, '_local__lock', RLock())
-
-        if (args or kw) and (cls.__init__ == object.__init__):
-            raise TypeError("Initialization arguments are not supported")
-
-        # We need to create the thread dict in anticipation of
-        # __init__ being called, to make sure we don't call it
-        # again ourselves.
-        dict = object.__getattribute__(self, '__dict__')
-        current_thread().__dict__[key] = dict
-
-        return self
-
-def _patch(self):
-    key = object.__getattribute__(self, '_local__key')
-    d = current_thread().__dict__.get(key)
-    if d is None:
-        d = {}
-        current_thread().__dict__[key] = d
-        object.__setattr__(self, '__dict__', d)
-
-        # we have a new instance dict, so call out __init__ if we have
-        # one
-        cls = type(self)
-        if cls.__init__ is not object.__init__:
-            args, kw = object.__getattribute__(self, '_local__args')
-            cls.__init__(self, *args, **kw)
-    else:
-        object.__setattr__(self, '__dict__', d)
-
-class local(_localbase):
-
-    def __getattribute__(self, name):
-        lock = object.__getattribute__(self, '_local__lock')
-        lock.acquire()
-        try:
-            _patch(self)
-            return object.__getattribute__(self, name)
-        finally:
-            lock.release()
-
-    def __setattr__(self, name, value):
-        if name == '__dict__':
-            raise AttributeError(
-                "%r object attribute '__dict__' is read-only"
-                % self.__class__.__name__)
-        lock = object.__getattribute__(self, '_local__lock')
-        lock.acquire()
-        try:
-            _patch(self)
-            return object.__setattr__(self, name, value)
-        finally:
-            lock.release()
-
-    def __delattr__(self, name):
-        if name == '__dict__':
-            raise AttributeError(
-                "%r object attribute '__dict__' is read-only"
-                % self.__class__.__name__)
-        lock = object.__getattribute__(self, '_local__lock')
-        lock.acquire()
-        try:
-            _patch(self)
-            return object.__delattr__(self, name)
-        finally:
-            lock.release()
-
-    def __del__(self):
-        import threading
-
-        key = object.__getattribute__(self, '_local__key')
-
-        try:
-            # We use the non-locking API since we might already hold the lock
-            # (__del__ can be called at any point by the cyclic GC).
-            threads = threading._enumerate()
-        except:
-            # If enumerating the current threads fails, as it seems to do
-            # during shutdown, we'll skip cleanup under the assumption
-            # that there is nothing to clean up.
-            return
-
-        for thread in threads:
-            try:
-                __dict__ = thread.__dict__
-            except AttributeError:
-                # Thread is dying, rest in peace.
-                continue
-
-            if key in __dict__:
-                try:
-                    del __dict__[key]
-                except KeyError:
-                    pass # didn't have anything in this thread
-
-from threading import current_thread, RLock
diff --git a/lib-python/modified-2.7/ctypes/__init__.py b/lib-python/modified-2.7/ctypes/__init__.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/__init__.py
+++ /dev/null
@@ -1,554 +0,0 @@
-######################################################################
-#  This file should be kept compatible with Python 2.3, see PEP 291. #
-######################################################################
-"""create and manipulate C data types in Python"""
-
-import os as _os, sys as _sys
-
-__version__ = "1.1.0"
-
-import _ffi
-from _ctypes import Union, Structure, Array
-from _ctypes import _Pointer
-from _ctypes import CFuncPtr as _CFuncPtr
-from _ctypes import __version__ as _ctypes_version
-from _ctypes import RTLD_LOCAL, RTLD_GLOBAL
-from _ctypes import ArgumentError
-
-from struct import calcsize as _calcsize
-
-if __version__ != _ctypes_version:
-    raise Exception("Version number mismatch", __version__, _ctypes_version)
-
-if _os.name in ("nt", "ce"):
-    from _ctypes import FormatError
-
-DEFAULT_MODE = RTLD_LOCAL
-if _os.name == "posix" and _sys.platform == "darwin":
-    # On OS X 10.3, we use RTLD_GLOBAL as default mode
-    # because RTLD_LOCAL does not work at least on some
-    # libraries.  OS X 10.3 is Darwin 7, so we check for
-    # that.
-
-    if int(_os.uname()[2].split('.')[0]) < 8:
-        DEFAULT_MODE = RTLD_GLOBAL
-
-from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \
-     FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI, \
-     FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \
-     FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR
-
-"""
-WINOLEAPI -> HRESULT
-WINOLEAPI_(type)
-
-STDMETHODCALLTYPE
-
-STDMETHOD(name)
-STDMETHOD_(type, name)
-
-STDAPICALLTYPE
-"""
-
-def create_string_buffer(init, size=None):
-    """create_string_buffer(aString) -> character array
-    create_string_buffer(anInteger) -> character array
-    create_string_buffer(aString, anInteger) -> character array
-    """
-    if isinstance(init, (str, unicode)):
-        if size is None:
-            size = len(init)+1
-        buftype = c_char * size
-        buf = buftype()
-        buf.value = init
-        return buf
-    elif isinstance(init, (int, long)):
-        buftype = c_char * init
-        buf = buftype()
-        return buf
-    raise TypeError(init)
-
-def c_buffer(init, size=None):
-##    "deprecated, use create_string_buffer instead"
-##    import warnings
-##    warnings.warn("c_buffer is deprecated, use create_string_buffer instead",
-##                  DeprecationWarning, stacklevel=2)
-    return create_string_buffer(init, size)
-
-_c_functype_cache = {}
-def CFUNCTYPE(restype, *argtypes, **kw):
-    """CFUNCTYPE(restype, *argtypes,
-                 use_errno=False, use_last_error=False) -> function prototype.
-
-    restype: the result type
-    argtypes: a sequence specifying the argument types
-
-    The function prototype can be called in different ways to create a
-    callable object:
-
-    prototype(integer address) -> foreign function
-    prototype(callable) -> create and return a C callable function from callable
-    prototype(integer index, method name[, paramflags]) -> foreign function calling a COM method
-    prototype((ordinal number, dll object)[, paramflags]) -> foreign function exported by ordinal
-    prototype((function name, dll object)[, paramflags]) -> foreign function exported by name
-    """
-    flags = _FUNCFLAG_CDECL
-    if kw.pop("use_errno", False):
-        flags |= _FUNCFLAG_USE_ERRNO
-    if kw.pop("use_last_error", False):
-        flags |= _FUNCFLAG_USE_LASTERROR
-    if kw:
-        raise ValueError("unexpected keyword argument(s) %s" % kw.keys())
-    try:
-        return _c_functype_cache[(restype, argtypes, flags)]
-    except KeyError:
-        class CFunctionType(_CFuncPtr):
-            _argtypes_ = argtypes
-            _restype_ = restype
-            _flags_ = flags
-        _c_functype_cache[(restype, argtypes, flags)] = CFunctionType
-        return CFunctionType
-
-if _os.name in ("nt", "ce"):
-    from _ctypes import LoadLibrary as _dlopen
-    from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL
-    if _os.name == "ce":
-        # 'ce' doesn't have the stdcall calling convention
-        _FUNCFLAG_STDCALL = _FUNCFLAG_CDECL
-
-    _win_functype_cache = {}
-    def WINFUNCTYPE(restype, *argtypes, **kw):
-        # docstring set later (very similar to CFUNCTYPE.__doc__)
-        flags = _FUNCFLAG_STDCALL
-        if kw.pop("use_errno", False):
-            flags |= _FUNCFLAG_USE_ERRNO
-        if kw.pop("use_last_error", False):
-            flags |= _FUNCFLAG_USE_LASTERROR
-        if kw:
-            raise ValueError("unexpected keyword argument(s) %s" % kw.keys())
-        try:
-            return _win_functype_cache[(restype, argtypes, flags)]
-        except KeyError:
-            class WinFunctionType(_CFuncPtr):
-                _argtypes_ = argtypes
-                _restype_ = restype
-                _flags_ = flags
-            _win_functype_cache[(restype, argtypes, flags)] = WinFunctionType
-            return WinFunctionType
-    if WINFUNCTYPE.__doc__:
-        WINFUNCTYPE.__doc__ = CFUNCTYPE.__doc__.replace("CFUNCTYPE", "WINFUNCTYPE")
-
-elif _os.name == "posix":
-    from _ctypes import dlopen as _dlopen
-
-from _ctypes import sizeof, byref, addressof, alignment, resize
-from _ctypes import get_errno, set_errno
-from _ctypes import _SimpleCData
-
-def _check_size(typ, typecode=None):
-    # Check if sizeof(ctypes_type) against struct.calcsize.  This
-    # should protect somewhat against a misconfigured libffi.
-    from struct import calcsize
-    if typecode is None:
-        # Most _type_ codes are the same as used in struct
-        typecode = typ._type_
-    actual, required = sizeof(typ), calcsize(typecode)
-    if actual != required:
-        raise SystemError("sizeof(%s) wrong: %d instead of %d" % \
-                          (typ, actual, required))
-
-class py_object(_SimpleCData):
-    _type_ = "O"
-    def __repr__(self):
-        try:
-            return super(py_object, self).__repr__()
-        except ValueError:
-            return "%s(<NULL>)" % type(self).__name__
-_check_size(py_object, "P")
-
-class c_short(_SimpleCData):
-    _type_ = "h"
-_check_size(c_short)
-
-class c_ushort(_SimpleCData):
-    _type_ = "H"
-_check_size(c_ushort)
-
-class c_long(_SimpleCData):
-    _type_ = "l"
-_check_size(c_long)
-
-class c_ulong(_SimpleCData):
-    _type_ = "L"
-_check_size(c_ulong)
-
-if _calcsize("i") == _calcsize("l"):
-    # if int and long have the same size, make c_int an alias for c_long
-    c_int = c_long
-    c_uint = c_ulong
-else:
-    class c_int(_SimpleCData):
-        _type_ = "i"
-    _check_size(c_int)
-
-    class c_uint(_SimpleCData):
-        _type_ = "I"
-    _check_size(c_uint)
-
-class c_float(_SimpleCData):
-    _type_ = "f"
-_check_size(c_float)
-
-class c_double(_SimpleCData):
-    _type_ = "d"
-_check_size(c_double)
-
-class c_longdouble(_SimpleCData):
-    _type_ = "g"
-if sizeof(c_longdouble) == sizeof(c_double):
-    c_longdouble = c_double
-
-if _calcsize("l") == _calcsize("q"):
-    # if long and long long have the same size, make c_longlong an alias for c_long
-    c_longlong = c_long
-    c_ulonglong = c_ulong
-else:
-    class c_longlong(_SimpleCData):
-        _type_ = "q"
-    _check_size(c_longlong)
-
-    class c_ulonglong(_SimpleCData):
-        _type_ = "Q"
-    ##    def from_param(cls, val):
-    ##        return ('d', float(val), val)
-    ##    from_param = classmethod(from_param)
-    _check_size(c_ulonglong)
-
-class c_ubyte(_SimpleCData):
-    _type_ = "B"
-c_ubyte.__ctype_le__ = c_ubyte.__ctype_be__ = c_ubyte
-# backward compatibility:
-##c_uchar = c_ubyte
-_check_size(c_ubyte)
-
-class c_byte(_SimpleCData):
-    _type_ = "b"
-c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte
-_check_size(c_byte)
-
-class c_char(_SimpleCData):
-    _type_ = "c"
-c_char.__ctype_le__ = c_char.__ctype_be__ = c_char
-_check_size(c_char)
-
-class c_char_p(_SimpleCData):
-    _type_ = "z"
-    if _os.name == "nt":
-        def __repr__(self):
-            if not windll.kernel32.IsBadStringPtrA(self, -1):
-                return "%s(%r)" % (self.__class__.__name__, self.value)
-            return "%s(%s)" % (self.__class__.__name__, cast(self, c_void_p).value)
-    else:
-        def __repr__(self):
-            return "%s(%s)" % (self.__class__.__name__, cast(self, c_void_p).value)
-_check_size(c_char_p, "P")
-
-class c_void_p(_SimpleCData):
-    _type_ = "P"
-c_voidp = c_void_p # backwards compatibility (to a bug)
-_check_size(c_void_p)
-
-class c_bool(_SimpleCData):
-    _type_ = "?"
-
-from _ctypes import POINTER, pointer, _pointer_type_cache
-
-try:
-    from _ctypes import set_conversion_mode
-except ImportError:
-    pass
-else:
-    if _os.name in ("nt", "ce"):
-        set_conversion_mode("mbcs", "ignore")
-    else:
-        set_conversion_mode("ascii", "strict")
-
-    class c_wchar_p(_SimpleCData):
-        _type_ = "Z"
-
-    class c_wchar(_SimpleCData):
-        _type_ = "u"
-
-    POINTER(c_wchar).from_param = c_wchar_p.from_param #_SimpleCData.c_wchar_p_from_param
-
-    def create_unicode_buffer(init, size=None):
-        """create_unicode_buffer(aString) -> character array
-        create_unicode_buffer(anInteger) -> character array
-        create_unicode_buffer(aString, anInteger) -> character array
-        """
-        if isinstance(init, (str, unicode)):
-            if size is None:
-                size = len(init)+1
-            buftype = c_wchar * size
-            buf = buftype()
-            buf.value = init
-            return buf
-        elif isinstance(init, (int, long)):
-            buftype = c_wchar * init
-            buf = buftype()
-            return buf
-        raise TypeError(init)
-
-POINTER(c_char).from_param = c_char_p.from_param #_SimpleCData.c_char_p_from_param
-
-# XXX Deprecated
-def SetPointerType(pointer, cls):
-    if _pointer_type_cache.get(cls, None) is not None:
-        raise RuntimeError("This type already exists in the cache")
-    if id(pointer) not in _pointer_type_cache:
-        raise RuntimeError("What's this???")
-    pointer.set_type(cls)
-    _pointer_type_cache[cls] = pointer
-    del _pointer_type_cache[id(pointer)]
-
-# XXX Deprecated
-def ARRAY(typ, len):
-    return typ * len
-
-################################################################
-
-
-class CDLL(object):
-    """An instance of this class represents a loaded dll/shared
-    library, exporting functions using the standard C calling
-    convention (named 'cdecl' on Windows).
-
-    The exported functions can be accessed as attributes, or by
-    indexing with the function name.  Examples:
-
-    <obj>.qsort -> callable object
-    <obj>['qsort'] -> callable object
-
-    Calling the functions releases the Python GIL during the call and
-    reacquires it afterwards.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL
-    _func_restype_ = c_int
-
-    def __init__(self, name, mode=DEFAULT_MODE, handle=None,
-                 use_errno=False,
-                 use_last_error=False):
-        self._name = name
-        flags = self._func_flags_
-        if use_errno:
-            flags |= _FUNCFLAG_USE_ERRNO
-        if use_last_error:
-            flags |= _FUNCFLAG_USE_LASTERROR
-
-        class _FuncPtr(_CFuncPtr):
-            _flags_ = flags
-            _restype_ = self._func_restype_
-        self._FuncPtr = _FuncPtr
-
-        if handle is None:
-            self._handle = _ffi.CDLL(name, mode)
-        else:
-            self._handle = handle
-
-    def __repr__(self):
-        return "<%s '%s', handle %r at %x>" % \
-               (self.__class__.__name__, self._name,
-                (self._handle),
-                id(self) & (_sys.maxint*2 + 1))
-
-
-    def __getattr__(self, name):
-        if name.startswith('__') and name.endswith('__'):
-            raise AttributeError(name)
-        func = self.__getitem__(name)
-        setattr(self, name, func)
-        return func
-
-    def __getitem__(self, name_or_ordinal):
-        func = self._FuncPtr((name_or_ordinal, self))
-        if not isinstance(name_or_ordinal, (int, long)):
-            func.__name__ = name_or_ordinal
-        return func
-
-class PyDLL(CDLL):
-    """This class represents the Python library itself.  It allows to
-    access Python API functions.  The GIL is not released, and
-    Python exceptions are handled correctly.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
-
-if _os.name in ("nt", "ce"):
-
-    class WinDLL(CDLL):
-        """This class represents a dll exporting functions using the
-        Windows stdcall calling convention.
-        """
-        _func_flags_ = _FUNCFLAG_STDCALL
-
-    # XXX Hm, what about HRESULT as normal parameter?
-    # Mustn't it derive from c_long then?
-    from _ctypes import _check_HRESULT, _SimpleCData
-    class HRESULT(_SimpleCData):
-        _type_ = "l"
-        # _check_retval_ is called with the function's result when it
-        # is used as restype.  It checks for the FAILED bit, and
-        # raises a WindowsError if it is set.
-        #
-        # The _check_retval_ method is implemented in C, so that the
-        # method definition itself is not included in the traceback
-        # when it raises an error - that is what we want (and Python
-        # doesn't have a way to raise an exception in the caller's
-        # frame).
-        _check_retval_ = _check_HRESULT
-
-    class OleDLL(CDLL):
-        """This class represents a dll exporting functions using the
-        Windows stdcall calling convention, and returning HRESULT.
-        HRESULT error values are automatically raised as WindowsError
-        exceptions.
-        """
-        _func_flags_ = _FUNCFLAG_STDCALL
-        _func_restype_ = HRESULT
-
-class LibraryLoader(object):
-    def __init__(self, dlltype):
-        self._dlltype = dlltype
-
-    def __getattr__(self, name):
-        if name[0] == '_':
-            raise AttributeError(name)
-        dll = self._dlltype(name)
-        setattr(self, name, dll)
-        return dll
-
-    def __getitem__(self, name):
-        return getattr(self, name)
-
-    def LoadLibrary(self, name):
-        return self._dlltype(name)
-
-cdll = LibraryLoader(CDLL)
-pydll = LibraryLoader(PyDLL)
-
-if _os.name in ("nt", "ce"):
-    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
-elif _sys.platform == "cygwin":
-    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
-else:
-    pythonapi = PyDLL(None)
-
-
-if _os.name in ("nt", "ce"):
-    windll = LibraryLoader(WinDLL)
-    oledll = LibraryLoader(OleDLL)
-
-    if _os.name == "nt":
-        GetLastError = windll.kernel32.GetLastError
-    else:
-        GetLastError = windll.coredll.GetLastError
-    from _ctypes import get_last_error, set_last_error
-
-    def WinError(code=None, descr=None):
-        if code is None:
-            code = GetLastError()
-        if descr is None:
-            descr = FormatError(code).strip()
-        return WindowsError(code, descr)
-
-_pointer_type_cache[None] = c_void_p
-
-if sizeof(c_uint) == sizeof(c_void_p):
-    c_size_t = c_uint
-    c_ssize_t = c_int
-elif sizeof(c_ulong) == sizeof(c_void_p):
-    c_size_t = c_ulong
-    c_ssize_t = c_long
-elif sizeof(c_ulonglong) == sizeof(c_void_p):
-    c_size_t = c_ulonglong
-    c_ssize_t = c_longlong
-
-# functions
-
-from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr
-
-## void *memmove(void *, const void *, size_t);
-memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr)
-
-## void *memset(void *, int, size_t)
-memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr)
-
-def PYFUNCTYPE(restype, *argtypes):
-    class CFunctionType(_CFuncPtr):
-        _argtypes_ = argtypes
-        _restype_ = restype
-        _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
-    return CFunctionType
-
-def cast(obj, typ):
-    try:
-        c_void_p.from_param(obj)
-    except TypeError, e:
-        raise ArgumentError(str(e))
-    return _cast_addr(obj, obj, typ)
-
-_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
-def string_at(ptr, size=-1):
-    """string_at(addr[, size]) -> string
-
-    Return the string at addr."""
-    return _string_at(ptr, size)
-
-try:
-    from _ctypes import _wstring_at_addr
-except ImportError:
-    pass
-else:
-    _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr)
-    def wstring_at(ptr, size=-1):
-        """wstring_at(addr[, size]) -> string
-
-        Return the string at addr."""
-        return _wstring_at(ptr, size)
-
-
-if _os.name in ("nt", "ce"): # COM stuff
-    def DllGetClassObject(rclsid, riid, ppv):
-        try:
-            ccom = __import__("comtypes.server.inprocserver", globals(), locals(), ['*'])
-        except ImportError:
-            return -2147221231 # CLASS_E_CLASSNOTAVAILABLE
-        else:
-            return ccom.DllGetClassObject(rclsid, riid, ppv)
-
-    def DllCanUnloadNow():
-        try:
-            ccom = __import__("comtypes.server.inprocserver", globals(), locals(), ['*'])
-        except ImportError:
-            return 0 # S_OK
-        return ccom.DllCanUnloadNow()
-
-from ctypes._endian import BigEndianStructure, LittleEndianStructure
-
-# Fill in specifically-sized types
-c_int8 = c_byte
-c_uint8 = c_ubyte
-for kind in [c_short, c_int, c_long, c_longlong]:
-    if sizeof(kind) == 2: c_int16 = kind
-    elif sizeof(kind) == 4: c_int32 = kind
-    elif sizeof(kind) == 8: c_int64 = kind
-for kind in [c_ushort, c_uint, c_ulong, c_ulonglong]:
-    if sizeof(kind) == 2: c_uint16 = kind
-    elif sizeof(kind) == 4: c_uint32 = kind
-    elif sizeof(kind) == 8: c_uint64 = kind
-del(kind)
-
-# XXX for whatever reasons, creating the first instance of a callback
-# function is needed for the unittests on Win64 to succeed.  This MAY
-# be a compiler bug, since the problem occurs only when _ctypes is
-# compiled with the MS SDK compiler.  Or an uninitialized variable?
-CFUNCTYPE(c_int)(lambda: None)
diff --git a/lib-python/modified-2.7/ctypes/_endian.py b/lib-python/modified-2.7/ctypes/_endian.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/_endian.py
+++ /dev/null
@@ -1,60 +0,0 @@
-######################################################################
-#  This file should be kept compatible with Python 2.3, see PEP 291. #
-######################################################################
-import sys
-from ctypes import *
-
-_array_type = type(c_int * 3)
-
-def _other_endian(typ):
-    """Return the type with the 'other' byte order.  Simple types like
-    c_int and so on already have __ctype_be__ and __ctype_le__
-    attributes which contain the types, for more complicated types
-    only arrays are supported.
-    """
-    try:
-        return getattr(typ, _OTHER_ENDIAN)
-    except AttributeError:
-        if type(typ) == _array_type:
-            return _other_endian(typ._type_) * typ._length_
-        raise TypeError("This type does not support other endian: %s" % typ)
-
-class _swapped_meta(type(Structure)):
-    def __setattr__(self, attrname, value):
-        if attrname == "_fields_":
-            fields = []
-            for desc in value:
-                name = desc[0]
-                typ = desc[1]
-                rest = desc[2:]
-                fields.append((name, _other_endian(typ)) + rest)
-            value = fields
-        super(_swapped_meta, self).__setattr__(attrname, value)
-
-################################################################
-
-# Note: The Structure metaclass checks for the *presence* (not the
-# value!) of a _swapped_bytes_ attribute to determine the bit order in
-# structures containing bit fields.
-
-if sys.byteorder == "little":
-    _OTHER_ENDIAN = "__ctype_be__"
-
-    LittleEndianStructure = Structure
-
-    class BigEndianStructure(Structure):
-        """Structure with big endian byte order"""
-        __metaclass__ = _swapped_meta
-        _swappedbytes_ = None
-
-elif sys.byteorder == "big":
-    _OTHER_ENDIAN = "__ctype_le__"
-
-    BigEndianStructure = Structure
-    class LittleEndianStructure(Structure):
-        """Structure with little endian byte order"""
-        __metaclass__ = _swapped_meta
-        _swappedbytes_ = None
-
-else:
-    raise RuntimeError("Invalid byteorder")
diff --git a/lib-python/modified-2.7/ctypes/macholib/README.ctypes b/lib-python/modified-2.7/ctypes/macholib/README.ctypes
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/macholib/README.ctypes
+++ /dev/null
@@ -1,7 +0,0 @@
-Files in this directory from from Bob Ippolito's py2app.
-
-License: Any components of the py2app suite may be distributed under
-the MIT or PSF open source licenses.
-
-This is version 1.0, SVN revision 789, from 2006/01/25.
-The main repository is http://svn.red-bean.com/bob/macholib/trunk/macholib/
\ No newline at end of file
diff --git a/lib-python/modified-2.7/ctypes/macholib/__init__.py b/lib-python/modified-2.7/ctypes/macholib/__init__.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/macholib/__init__.py
+++ /dev/null
@@ -1,12 +0,0 @@
-######################################################################
-#  This file should be kept compatible with Python 2.3, see PEP 291. #
-######################################################################
-"""
-Enough Mach-O to make your head spin.
-
-See the relevant header files in /usr/include/mach-o
-
-And also Apple's documentation.
-"""
-
-__version__ = '1.0'
diff --git a/lib-python/modified-2.7/ctypes/macholib/dyld.py b/lib-python/modified-2.7/ctypes/macholib/dyld.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/macholib/dyld.py
+++ /dev/null
@@ -1,169 +0,0 @@
-######################################################################
-#  This file should be kept compatible with Python 2.3, see PEP 291. #
-######################################################################
-"""
-dyld emulation
-"""
-
-import os
-from framework import framework_info
-from dylib import dylib_info
-from itertools import *
-
-__all__ = [
-    'dyld_find', 'framework_find',
-    'framework_info', 'dylib_info',
-]
-
-# These are the defaults as per man dyld(1)
-#
-DEFAULT_FRAMEWORK_FALLBACK = [
-    os.path.expanduser("~/Library/Frameworks"),
-    "/Library/Frameworks",
-    "/Network/Library/Frameworks",
-    "/System/Library/Frameworks",
-]
-
-DEFAULT_LIBRARY_FALLBACK = [
-    os.path.expanduser("~/lib"),
-    "/usr/local/lib",
-    "/lib",
-    "/usr/lib",
-]
-
-def ensure_utf8(s):
-    """Not all of PyObjC and Python understand unicode paths very well yet"""
-    if isinstance(s, unicode):
-        return s.encode('utf8')
-    return s
-
-def dyld_env(env, var):
-    if env is None:
-        env = os.environ
-    rval = env.get(var)
-    if rval is None:
-        return []
-    return rval.split(':')
-
-def dyld_image_suffix(env=None):
-    if env is None:
-        env = os.environ
-    return env.get('DYLD_IMAGE_SUFFIX')
-
-def dyld_framework_path(env=None):
-    return dyld_env(env, 'DYLD_FRAMEWORK_PATH')
-
-def dyld_library_path(env=None):
-    return dyld_env(env, 'DYLD_LIBRARY_PATH')
-
-def dyld_fallback_framework_path(env=None):
-    return dyld_env(env, 'DYLD_FALLBACK_FRAMEWORK_PATH')
-
-def dyld_fallback_library_path(env=None):
-    return dyld_env(env, 'DYLD_FALLBACK_LIBRARY_PATH')
-
-def dyld_image_suffix_search(iterator, env=None):
-    """For a potential path iterator, add DYLD_IMAGE_SUFFIX semantics"""
-    suffix = dyld_image_suffix(env)
-    if suffix is None:
-        return iterator
-    def _inject(iterator=iterator, suffix=suffix):
-        for path in iterator:
-            if path.endswith('.dylib'):
-                yield path[:-len('.dylib')] + suffix + '.dylib'
-            else:
-                yield path + suffix
-            yield path
-    return _inject()
-
-def dyld_override_search(name, env=None):
-    # If DYLD_FRAMEWORK_PATH is set and this dylib_name is a
-    # framework name, use the first file that exists in the framework
-    # path if any.  If there is none go on to search the DYLD_LIBRARY_PATH
-    # if any.
-
-    framework = framework_info(name)
-
-    if framework is not None:
-        for path in dyld_framework_path(env):
-            yield os.path.join(path, framework['name'])
-
-    # If DYLD_LIBRARY_PATH is set then use the first file that exists
-    # in the path.  If none use the original name.
-    for path in dyld_library_path(env):
-        yield os.path.join(path, os.path.basename(name))
-
-def dyld_executable_path_search(name, executable_path=None):
-    # If we haven't done any searching and found a library and the
-    # dylib_name starts with "@executable_path/" then construct the
-    # library name.
-    if name.startswith('@executable_path/') and executable_path is not None:
-        yield os.path.join(executable_path, name[len('@executable_path/'):])
-
-def dyld_default_search(name, env=None):
-    yield name
-
-    framework = framework_info(name)
-
-    if framework is not None:
-        fallback_framework_path = dyld_fallback_framework_path(env)
-        for path in fallback_framework_path:
-            yield os.path.join(path, framework['name'])
-
-    fallback_library_path = dyld_fallback_library_path(env)
-    for path in fallback_library_path:
-        yield os.path.join(path, os.path.basename(name))
-
-    if framework is not None and not fallback_framework_path:
-        for path in DEFAULT_FRAMEWORK_FALLBACK:
-            yield os.path.join(path, framework['name'])
-
-    if not fallback_library_path:
-        for path in DEFAULT_LIBRARY_FALLBACK:
-            yield os.path.join(path, os.path.basename(name))
-
-def dyld_find(name, executable_path=None, env=None):
-    """
-    Find a library or framework using dyld semantics
-    """
-    name = ensure_utf8(name)
-    executable_path = ensure_utf8(executable_path)
-    for path in dyld_image_suffix_search(chain(
-                dyld_override_search(name, env),
-                dyld_executable_path_search(name, executable_path),
-                dyld_default_search(name, env),
-            ), env):
-        if os.path.isfile(path):
-            return path
-    raise ValueError("dylib %s could not be found" % (name,))
-
-def framework_find(fn, executable_path=None, env=None):
-    """
-    Find a framework using dyld semantics in a very loose manner.
-
-    Will take input such as:
-        Python
-        Python.framework
-        Python.framework/Versions/Current
-    """
-    try:
-        return dyld_find(fn, executable_path=executable_path, env=env)
-    except ValueError, e:
-        pass
-    fmwk_index = fn.rfind('.framework')
-    if fmwk_index == -1:
-        fmwk_index = len(fn)
-        fn += '.framework'
-    fn = os.path.join(fn, os.path.basename(fn[:fmwk_index]))
-    try:
-        return dyld_find(fn, executable_path=executable_path, env=env)
-    except ValueError:
-        raise e
-
-def test_dyld_find():
-    env = {}
-    assert dyld_find('libSystem.dylib') == '/usr/lib/libSystem.dylib'
-    assert dyld_find('System.framework/System') == '/System/Library/Frameworks/System.framework/System'
-
-if __name__ == '__main__':
-    test_dyld_find()
diff --git a/lib-python/modified-2.7/ctypes/macholib/dylib.py b/lib-python/modified-2.7/ctypes/macholib/dylib.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/macholib/dylib.py
+++ /dev/null
@@ -1,66 +0,0 @@
-######################################################################
-#  This file should be kept compatible with Python 2.3, see PEP 291. #
-######################################################################
-"""
-Generic dylib path manipulation
-"""
-
-import re
-
-__all__ = ['dylib_info']
-
-DYLIB_RE = re.compile(r"""(?x)
-(?P<location>^.*)(?:^|/)
-(?P<name>
-    (?P<shortname>\w+?)
-    (?:\.(?P<version>[^._]+))?
-    (?:_(?P<suffix>[^._]+))?
-    \.dylib$
-)
-""")
-
-def dylib_info(filename):
-    """
-    A dylib name can take one of the following four forms:
-        Location/Name.SomeVersion_Suffix.dylib
-        Location/Name.SomeVersion.dylib
-        Location/Name_Suffix.dylib
-        Location/Name.dylib
-
-    returns None if not found or a mapping equivalent to:
-        dict(
-            location='Location',
-            name='Name.SomeVersion_Suffix.dylib',
-            shortname='Name',
-            version='SomeVersion',
-            suffix='Suffix',
-        )
-
-    Note that SomeVersion and Suffix are optional and may be None
-    if not present.
-    """
-    is_dylib = DYLIB_RE.match(filename)
-    if not is_dylib:
-        return None
-    return is_dylib.groupdict()
-
-
-def test_dylib_info():
-    def d(location=None, name=None, shortname=None, version=None, suffix=None):
-        return dict(
-            location=location,
-            name=name,
-            shortname=shortname,
-            version=version,
-            suffix=suffix
-        )
-    assert dylib_info('completely/invalid') is None
-    assert dylib_info('completely/invalide_debug') is None
-    assert dylib_info('P/Foo.dylib') == d('P', 'Foo.dylib', 'Foo')
-    assert dylib_info('P/Foo_debug.dylib') == d('P', 'Foo_debug.dylib', 'Foo', suffix='debug')
-    assert dylib_info('P/Foo.A.dylib') == d('P', 'Foo.A.dylib', 'Foo', 'A')
-    assert dylib_info('P/Foo_debug.A.dylib') == d('P', 'Foo_debug.A.dylib', 'Foo_debug', 'A')
-    assert dylib_info('P/Foo.A_debug.dylib') == d('P', 'Foo.A_debug.dylib', 'Foo', 'A', 'debug')
-
-if __name__ == '__main__':
-    test_dylib_info()
diff --git a/lib-python/modified-2.7/ctypes/macholib/fetch_macholib b/lib-python/modified-2.7/ctypes/macholib/fetch_macholib
deleted file mode 100755
--- a/lib-python/modified-2.7/ctypes/macholib/fetch_macholib
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/modified-2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/modified-2.7/ctypes/macholib/fetch_macholib.bat
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/macholib/fetch_macholib.bat
+++ /dev/null
@@ -1,1 +0,0 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/modified-2.7/ctypes/macholib/framework.py b/lib-python/modified-2.7/ctypes/macholib/framework.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/macholib/framework.py
+++ /dev/null
@@ -1,68 +0,0 @@
-######################################################################
-#  This file should be kept compatible with Python 2.3, see PEP 291. #
-######################################################################
-"""
-Generic framework path manipulation
-"""
-
-import re
-
-__all__ = ['framework_info']
-
-STRICT_FRAMEWORK_RE = re.compile(r"""(?x)
-(?P<location>^.*)(?:^|/)
-(?P<name>
-    (?P<shortname>\w+).framework/
-    (?:Versions/(?P<version>[^/]+)/)?
-    (?P=shortname)
-    (?:_(?P<suffix>[^_]+))?
-)$
-""")
-
-def framework_info(filename):
-    """
-    A framework name can take one of the following four forms:
-        Location/Name.framework/Versions/SomeVersion/Name_Suffix
-        Location/Name.framework/Versions/SomeVersion/Name
-        Location/Name.framework/Name_Suffix
-        Location/Name.framework/Name
-
-    returns None if not found, or a mapping equivalent to:
-        dict(
-            location='Location',
-            name='Name.framework/Versions/SomeVersion/Name_Suffix',
-            shortname='Name',
-            version='SomeVersion',
-            suffix='Suffix',
-        )
-
-    Note that SomeVersion and Suffix are optional and may be None
-    if not present
-    """
-    is_framework = STRICT_FRAMEWORK_RE.match(filename)
-    if not is_framework:
-        return None
-    return is_framework.groupdict()
-
-def test_framework_info():
-    def d(location=None, name=None, shortname=None, version=None, suffix=None):
-        return dict(
-            location=location,
-            name=name,
-            shortname=shortname,
-            version=version,
-            suffix=suffix
-        )
-    assert framework_info('completely/invalid') is None
-    assert framework_info('completely/invalid/_debug') is None
-    assert framework_info('P/F.framework') is None
-    assert framework_info('P/F.framework/_debug') is None
-    assert framework_info('P/F.framework/F') == d('P', 'F.framework/F', 'F')
-    assert framework_info('P/F.framework/F_debug') == d('P', 'F.framework/F_debug', 'F', suffix='debug')
-    assert framework_info('P/F.framework/Versions') is None
-    assert framework_info('P/F.framework/Versions/A') is None
-    assert framework_info('P/F.framework/Versions/A/F') == d('P', 'F.framework/Versions/A/F', 'F', 'A')
-    assert framework_info('P/F.framework/Versions/A/F_debug') == d('P', 'F.framework/Versions/A/F_debug', 'F', 'A', 'debug')
-
-if __name__ == '__main__':
-    test_framework_info()
diff --git a/lib-python/modified-2.7/ctypes/test/__init__.py b/lib-python/modified-2.7/ctypes/test/__init__.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/test/__init__.py
+++ /dev/null
@@ -1,221 +0,0 @@
-import os, sys, unittest, getopt, time
-
-use_resources = []
-
-class ResourceDenied(Exception):
-    """Test skipped because it requested a disallowed resource.
-
-    This is raised when a test calls requires() for a resource that
-    has not be enabled.  Resources are defined by test modules.
-    """
-
-def is_resource_enabled(resource):
-    """Test whether a resource is enabled.
-
-    If the caller's module is __main__ then automatically return True."""
-    if sys._getframe().f_back.f_globals.get("__name__") == "__main__":
-        return True
-    result = use_resources is not None and \
-           (resource in use_resources or "*" in use_resources)
-    if not result:
-        _unavail[resource] = None
-    return result
-
-_unavail = {}
-def requires(resource, msg=None):
-    """Raise ResourceDenied if the specified resource is not available.
-
-    If the caller's module is __main__ then automatically return True."""
-    # see if the caller's module is __main__ - if so, treat as if
-    # the resource was set
-    if sys._getframe().f_back.f_globals.get("__name__") == "__main__":
-        return
-    if not is_resource_enabled(resource):
-        if msg is None:
-            msg = "Use of the `%s' resource not enabled" % resource
-        raise ResourceDenied(msg)
-
-def find_package_modules(package, mask):
-    import fnmatch
-    if (hasattr(package, "__loader__") and
-            hasattr(package.__loader__, '_files')):
-        path = package.__name__.replace(".", os.path.sep)
-        mask = os.path.join(path, mask)
-        for fnm in package.__loader__._files.iterkeys():
-            if fnmatch.fnmatchcase(fnm, mask):
-                yield os.path.splitext(fnm)[0].replace(os.path.sep, ".")
-    else:
-        path = package.__path__[0]
-        for fnm in os.listdir(path):
-            if fnmatch.fnmatchcase(fnm, mask):
-                yield "%s.%s" % (package.__name__, os.path.splitext(fnm)[0])
-
-def get_tests(package, mask, verbosity, exclude=()):
-    """Return a list of skipped test modules, and a list of test cases."""
-    tests = []
-    skipped = []
-    for modname in find_package_modules(package, mask):
-        if modname.split(".")[-1] in exclude:
-            skipped.append(modname)
-            if verbosity > 1:
-                print >> sys.stderr, "Skipped %s: excluded" % modname
-            continue
-        try:
-            mod = __import__(modname, globals(), locals(), ['*'])
-        except ResourceDenied, detail:
-            skipped.append(modname)
-            if verbosity > 1:
-                print >> sys.stderr, "Skipped %s: %s" % (modname, detail)
-            continue
-        for name in dir(mod):
-            if name.startswith("_"):
-                continue
-            o = getattr(mod, name)
-            if type(o) is type(unittest.TestCase) and issubclass(o, unittest.TestCase):
-                tests.append(o)
-    return skipped, tests
-
-def usage():
-    print __doc__
-    return 1
-
-def test_with_refcounts(runner, verbosity, testcase):
-    """Run testcase several times, tracking reference counts."""
-    import gc
-    import ctypes
-    ptc = ctypes._pointer_type_cache.copy()
-    cfc = ctypes._c_functype_cache.copy()
-    wfc = ctypes._win_functype_cache.copy()
-
-    # when searching for refcount leaks, we have to manually reset any
-    # caches that ctypes has.
-    def cleanup():
-        ctypes._pointer_type_cache = ptc.copy()
-        ctypes._c_functype_cache = cfc.copy()
-        ctypes._win_functype_cache = wfc.copy()
-        gc.collect()
-
-    test = unittest.makeSuite(testcase)
-    for i in range(5):
-        rc = sys.gettotalrefcount()
-        runner.run(test)
-        cleanup()
-    COUNT = 5
-    refcounts = [None] * COUNT
-    for i in range(COUNT):
-        rc = sys.gettotalrefcount()
-        runner.run(test)
-        cleanup()
-        refcounts[i] = sys.gettotalrefcount() - rc
-    if filter(None, refcounts):
-        print "%s leaks:\n\t" % testcase, refcounts
-    elif verbosity:
-        print "%s: ok." % testcase
-
-class TestRunner(unittest.TextTestRunner):
-    def run(self, test, skipped):
-        "Run the given test case or test suite."
-        # Same as unittest.TextTestRunner.run, except that it reports
-        # skipped tests.
-        result = self._makeResult()
-        startTime = time.time()
-        test(result)
-        stopTime = time.time()
-        timeTaken = stopTime - startTime
-        result.printErrors()
-        self.stream.writeln(result.separator2)
-        run = result.testsRun
-        if _unavail: #skipped:
-            requested = _unavail.keys()
-            requested.sort()
-            self.stream.writeln("Ran %d test%s in %.3fs (%s module%s skipped)" %
-                                (run, run != 1 and "s" or "", timeTaken,
-                                 len(skipped),
-                                 len(skipped) != 1 and "s" or ""))
-            self.stream.writeln("Unavailable resources: %s" % ", ".join(requested))
-        else:
-            self.stream.writeln("Ran %d test%s in %.3fs" %
-                                (run, run != 1 and "s" or "", timeTaken))
-        self.stream.writeln()
-        if not result.wasSuccessful():
-            self.stream.write("FAILED (")
-            failed, errored = map(len, (result.failures, result.errors))
-            if failed:
-                self.stream.write("failures=%d" % failed)
-            if errored:
-                if failed: self.stream.write(", ")
-                self.stream.write("errors=%d" % errored)
-            self.stream.writeln(")")
-        else:
-            self.stream.writeln("OK")
-        return result
-
-
-def main(*packages):
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "rqvu:x:")
-    except getopt.error:
-        return usage()
-
-    verbosity = 1
-    search_leaks = False
-    exclude = []
-    for flag, value in opts:
-        if flag == "-q":
-            verbosity -= 1
-        elif flag == "-v":
-            verbosity += 1
-        elif flag == "-r":
-            try:
-                sys.gettotalrefcount
-            except AttributeError:
-                print >> sys.stderr, "-r flag requires Python debug build"
-                return -1
-            search_leaks = True
-        elif flag == "-u":
-            use_resources.extend(value.split(","))
-        elif flag == "-x":
-            exclude.extend(value.split(","))
-
-    mask = "test_*.py"
-    if args:
-        mask = args[0]
-
-    for package in packages:
-        run_tests(package, mask, verbosity, search_leaks, exclude)
-
-
-def run_tests(package, mask, verbosity, search_leaks, exclude):
-    skipped, testcases = get_tests(package, mask, verbosity, exclude)
-    runner = TestRunner(verbosity=verbosity)
-
-    suites = [unittest.makeSuite(o) for o in testcases]
-    suite = unittest.TestSuite(suites)
-    result = runner.run(suite, skipped)
-
-    if search_leaks:
-        # hunt for refcount leaks
-        runner = BasicTestRunner()
-        for t in testcases:
-            test_with_refcounts(runner, verbosity, t)
-
-    return bool(result.errors)
-
-class BasicTestRunner:
-    def run(self, test):
-        result = unittest.TestResult()
-        test(result)
-        return result
-
-def xfail(method):
-    """
-    Poor's man xfail: remove it when all the failures have been fixed
-    """
-    def new_method(self, *args, **kwds):
-        try:
-            method(self, *args, **kwds)
-        except:
-            pass
-        else:
-            self.assertTrue(False, "DID NOT RAISE")
-    return new_method
diff --git a/lib-python/modified-2.7/ctypes/test/runtests.py b/lib-python/modified-2.7/ctypes/test/runtests.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/test/runtests.py
+++ /dev/null
@@ -1,19 +0,0 @@
-"""Usage: runtests.py [-q] [-r] [-v] [-u resources] [mask]
-
-Run all tests found in this directory, and print a summary of the results.
-Command line flags:
-  -q     quiet mode: don't prnt anything while the tests are running
-  -r     run tests repeatedly, look for refcount leaks
-  -u<resources>
-         Add resources to the lits of allowed resources. '*' allows all
-         resources.
-  -v     verbose mode: print the test currently executed
-  -x<test1[,test2...]>
-         Exclude specified tests.
-  mask   mask to select filenames containing testcases, wildcards allowed
-"""
-import sys
-import ctypes.test
-
-if __name__ == "__main__":
-    sys.exit(ctypes.test.main(ctypes.test))
diff --git a/lib-python/modified-2.7/ctypes/test/test_anon.py b/lib-python/modified-2.7/ctypes/test/test_anon.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/test/test_anon.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import unittest
-from ctypes import *
-
-class AnonTest(unittest.TestCase):
-
-    def test_anon(self):
-        class ANON(Union):
-            _fields_ = [("a", c_int),
-                        ("b", c_int)]
-
-        class Y(Structure):
-            _fields_ = [("x", c_int),
-                        ("_", ANON),
-                        ("y", c_int)]
-            _anonymous_ = ["_"]
-
-        self.assertEqual(Y.a.offset, sizeof(c_int))
-        self.assertEqual(Y.b.offset, sizeof(c_int))
-
-        self.assertEqual(ANON.a.offset, 0)
-        self.assertEqual(ANON.b.offset, 0)
-
-    def test_anon_nonseq(self):
-        # TypeError: _anonymous_ must be a sequence
-        self.assertRaises(TypeError,
-                              lambda: type(Structure)("Name",
-                                                      (Structure,),
-                                                      {"_fields_": [], "_anonymous_": 42}))
-
-    def test_anon_nonmember(self):
-        # AttributeError: type object 'Name' has no attribute 'x'
-        self.assertRaises(AttributeError,
-                              lambda: type(Structure)("Name",
-                                                      (Structure,),
-                                                      {"_fields_": [],
-                                                       "_anonymous_": ["x"]}))
-
-    def test_nested(self):
-        class ANON_S(Structure):
-            _fields_ = [("a", c_int)]
-
-        class ANON_U(Union):
-            _fields_ = [("_", ANON_S),
-                        ("b", c_int)]
-            _anonymous_ = ["_"]
-
-        class Y(Structure):
-            _fields_ = [("x", c_int),
-                        ("_", ANON_U),
-                        ("y", c_int)]
-            _anonymous_ = ["_"]
-
-        self.assertEqual(Y.x.offset, 0)
-        self.assertEqual(Y.a.offset, sizeof(c_int))
-        self.assertEqual(Y.b.offset, sizeof(c_int))
-        self.assertEqual(Y._.offset, sizeof(c_int))
-        self.assertEqual(Y.y.offset, sizeof(c_int) * 2)
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/lib-python/modified-2.7/ctypes/test/test_array_in_pointer.py b/lib-python/modified-2.7/ctypes/test/test_array_in_pointer.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/test/test_array_in_pointer.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import unittest
-from ctypes import *
-from binascii import hexlify
-import re
-
-def dump(obj):
-    # helper function to dump memory contents in hex, with a hyphen
-    # between the bytes.
-    h = hexlify(memoryview(obj))
-    return re.sub(r"(..)", r"\1-", h)[:-1]
-
-
-class Value(Structure):
-    _fields_ = [("val", c_byte)]
-
-class Container(Structure):
-    _fields_ = [("pvalues", POINTER(Value))]
-
-class Test(unittest.TestCase):
-    def test(self):
-        # create an array of 4 values
-        val_array = (Value * 4)()
-
-        # create a container, which holds a pointer to the pvalues array.
-        c = Container()
-        c.pvalues = val_array
-
-        # memory contains 4 NUL bytes now, that's correct
-        self.assertEqual("00-00-00-00", dump(val_array))
-
-        # set the values of the array through the pointer:
-        for i in range(4):
-            c.pvalues[i].val = i + 1
-
-        values = [c.pvalues[i].val for i in range(4)]
-
-        # These are the expected results: here s the bug!
-        self.assertEqual(
-            (values, dump(val_array)),
-            ([1, 2, 3, 4], "01-02-03-04")
-        )
-
-    def test_2(self):
-
-        val_array = (Value * 4)()
-
-        # memory contains 4 NUL bytes now, that's correct
-        self.assertEqual("00-00-00-00", dump(val_array))
-
-        ptr = cast(val_array, POINTER(Value))
-        # set the values of the array through the pointer:
-        for i in range(4):
-            ptr[i].val = i + 1
-
-        values = [ptr[i].val for i in range(4)]
-
-        # These are the expected results: here s the bug!
-        self.assertEqual(
-            (values, dump(val_array)),
-            ([1, 2, 3, 4], "01-02-03-04")
-        )
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/lib-python/modified-2.7/ctypes/test/test_arrays.py b/lib-python/modified-2.7/ctypes/test/test_arrays.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/test/test_arrays.py
+++ /dev/null
@@ -1,145 +0,0 @@
-import unittest
-from ctypes import *
-from test.test_support import impl_detail
-
-formats = "bBhHiIlLqQfd"
-
-# c_longdouble commented out for PyPy, look at the commend in test_longdouble
-formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \
-          c_long, c_ulonglong, c_float, c_double #, c_longdouble
-
-class ArrayTestCase(unittest.TestCase):
-
-    @impl_detail('long double not supported by PyPy', pypy=False)
-    def test_longdouble(self):
-        """
-        This test is empty. It's just here to remind that we commented out
-        c_longdouble in "formats". If pypy will ever supports c_longdouble, we
-        should kill this test and uncomment c_longdouble inside formats.
-        """
-
-    def test_simple(self):
-        # create classes holding simple numeric types, and check
-        # various properties.
-
-        init = range(15, 25)
-
-        for fmt in formats:
-            alen = len(init)
-            int_array = ARRAY(fmt, alen)
-
-            ia = int_array(*init)
-            # length of instance ok?
-            self.assertEqual(len(ia), alen)
-
-            # slot values ok?
-            values = [ia[i] for i in range(len(init))]
-            self.assertEqual(values, init)
-
-            # change the items
-            from operator import setitem
-            new_values = range(42, 42+alen)
-            [setitem(ia, n, new_values[n]) for n in range(alen)]
-            values = [ia[i] for i in range(len(init))]
-            self.assertEqual(values, new_values)
-
-            # are the items initialized to 0?
-            ia = int_array()
-            values = [ia[i] for i in range(len(init))]
-            self.assertEqual(values, [0] * len(init))
-
-            # Too many initializers should be caught
-            self.assertRaises(IndexError, int_array, *range(alen*2))
-
-        CharArray = ARRAY(c_char, 3)
-
-        ca = CharArray("a", "b", "c")
-
-        # Should this work? It doesn't:
-        # CharArray("abc")
-        self.assertRaises(TypeError, CharArray, "abc")
-
-        self.assertEqual(ca[0], "a")
-        self.assertEqual(ca[1], "b")
-        self.assertEqual(ca[2], "c")
-        self.assertEqual(ca[-3], "a")
-        self.assertEqual(ca[-2], "b")
-        self.assertEqual(ca[-1], "c")
-
-        self.assertEqual(len(ca), 3)
-
-        # slicing is now supported, but not extended slicing (3-argument)!
-        from operator import getslice, delitem
-        self.assertRaises(TypeError, getslice, ca, 0, 1, -1)
-
-        # cannot delete items
-        self.assertRaises(TypeError, delitem, ca, 0)
-
-    def test_numeric_arrays(self):
-
-        alen = 5
-
-        numarray = ARRAY(c_int, alen)
-
-        na = numarray()
-        values = [na[i] for i in range(alen)]
-        self.assertEqual(values, [0] * alen)
-
-        na = numarray(*[c_int()] * alen)
-        values = [na[i] for i in range(alen)]
-        self.assertEqual(values, [0]*alen)
-
-        na = numarray(1, 2, 3, 4, 5)
-        values = [i for i in na]
-        self.assertEqual(values, [1, 2, 3, 4, 5])
-
-        na = numarray(*map(c_int, (1, 2, 3, 4, 5)))
-        values = [i for i in na]
-        self.assertEqual(values, [1, 2, 3, 4, 5])
-
-    def test_classcache(self):
-        self.assertTrue(not ARRAY(c_int, 3) is ARRAY(c_int, 4))
-        self.assertTrue(ARRAY(c_int, 3) is ARRAY(c_int, 3))
-
-    def test_from_address(self):
-        # Failed with 0.9.8, reported by JUrner
-        p = create_string_buffer("foo")
-        sz = (c_char * 3).from_address(addressof(p))
-        self.assertEqual(sz[:], "foo")
-        self.assertEqual(sz[::], "foo")
-        self.assertEqual(sz[::-1], "oof")
-        self.assertEqual(sz[::3], "f")
-        self.assertEqual(sz[1:4:2], "o")
-        self.assertEqual(sz.value, "foo")
-
-    try:
-        create_unicode_buffer
-    except NameError:
-        pass
-    else:
-        def test_from_addressW(self):
-            p = create_unicode_buffer("foo")
-            sz = (c_wchar * 3).from_address(addressof(p))
-            self.assertEqual(sz[:], "foo")
-            self.assertEqual(sz[::], "foo")
-            self.assertEqual(sz[::-1], "oof")
-            self.assertEqual(sz[::3], "f")
-            self.assertEqual(sz[1:4:2], "o")
-            self.assertEqual(sz.value, "foo")
-
-    def test_cache(self):
-        # Array types are cached internally in the _ctypes extension,
-        # in a WeakValueDictionary.  Make sure the array type is
-        # removed from the cache when the itemtype goes away.  This
-        # test will not fail, but will show a leak in the testsuite.
-
-        # Create a new type:
-        class my_int(c_int):
-            pass
-        # Create a new array type based on it:
-        t1 = my_int * 1
-        t2 = my_int * 1
-        self.assertTrue(t1 is t2)
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/lib-python/modified-2.7/ctypes/test/test_as_parameter.py b/lib-python/modified-2.7/ctypes/test/test_as_parameter.py
deleted file mode 100644
--- a/lib-python/modified-2.7/ctypes/test/test_as_parameter.py
+++ /dev/null
@@ -1,227 +0,0 @@
-import unittest
-from ctypes import *
-import _ctypes_test
-
-dll = CDLL(_ctypes_test.__file__)
-
-try:
-    CALLBACK_FUNCTYPE = WINFUNCTYPE
-except NameError:
-    # fake to enable this test on Linux
-    CALLBACK_FUNCTYPE = CFUNCTYPE
-
-class POINT(Structure):
-    _fields_ = [("x", c_int), ("y", c_int)]
-
-class BasicWrapTestCase(unittest.TestCase):
-    def wrap(self, param):
-        return param
-
-    def test_wchar_parm(self):
-        try:
-            c_wchar
-        except NameError:
-            return
-        f = dll._testfunc_i_bhilfd
-        f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double]
-        result = f(self.wrap(1), self.wrap(u"x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0))
-        self.assertEqual(result, 139)
-        self.assertTrue(type(result), int)
-
-    def test_pointers(self):
-        f = dll._testfunc_p_p
-        f.restype = POINTER(c_int)
-        f.argtypes = [POINTER(c_int)]
-
-        # This only works if the value c_int(42) passed to the
-        # function is still alive while the pointer (the result) is
-        # used.
-
-        v = c_int(42)
-
-        self.assertEqual(pointer(v).contents.value, 42)
-        result = f(self.wrap(pointer(v)))
-        self.assertEqual(type(result), POINTER(c_int))
-        self.assertEqual(result.contents.value, 42)
-
-        # This on works...
-        result = f(self.wrap(pointer(v)))
-        self.assertEqual(result.contents.value, v.value)
-
-        p = pointer(c_int(99))
-        result = f(self.wrap(p))
-        self.assertEqual(result.contents.value, 99)
-
-    def test_shorts(self):
-        f = dll._testfunc_callback_i_if
-
-        args = []
-        expected = [262144, 131072, 65536, 32768, 16384, 8192, 4096, 2048,
-                    1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1]
-
-        def callback(v):
-            args.append(v)
-            return v
-
-        CallBack = CFUNCTYPE(c_int, c_int)
-
-        cb = CallBack(callback)
-        f(self.wrap(2**18), self.wrap(cb))
-        self.assertEqual(args, expected)
-
-    ################################################################
-
-    def test_callbacks(self):
-        f = dll._testfunc_callback_i_if
-        f.restype = c_int
-
-        MyCallback = CFUNCTYPE(c_int, c_int)
-
-        def callback(value):
-            #print "called back with", value
-            return value
-
-        cb = MyCallback(callback)
-
-        result = f(self.wrap(-10), self.wrap(cb))
-        self.assertEqual(result, -18)
-
-        # test with prototype
-        f.argtypes = [c_int, MyCallback]
-        cb = MyCallback(callback)
-
-        result = f(self.wrap(-10), self.wrap(cb))
-        self.assertEqual(result, -18)
-
-        result = f(self.wrap(-10), self.wrap(cb))
-        self.assertEqual(result, -18)
-
-        AnotherCallback = CALLBACK_FUNCTYPE(c_int, c_int, c_int, c_int, c_int)
-
-        # check that the prototype works: we call f with wrong
-        # argument types
-        cb = AnotherCallback(callback)
-        self.assertRaises(ArgumentError, f, self.wrap(-10), self.wrap(cb))
-
-    def test_callbacks_2(self):
-        # Can also use simple datatypes as argument type specifiers
-        # for the callback function.
-        # In this case the call receives an instance of that type
-        f = dll._testfunc_callback_i_if
-        f.restype = c_int
-
-        MyCallback = CFUNCTYPE(c_int, c_int)
-
-        f.argtypes = [c_int, MyCallback]
-
-        def callback(value):
-            #print "called back with", value
-            self.assertEqual(type(value), int)
-            return value
-
-        cb = MyCallback(callback)
-        result = f(self.wrap(-10), self.wrap(cb))
-        self.assertEqual(result, -18)
-
-    def test_longlong_callbacks(self):
-
-        f = dll._testfunc_callback_q_qf
-        f.restype = c_longlong
-
-        MyCallback = CFUNCTYPE(c_longlong, c_longlong)
-
-        f.argtypes = [c_longlong, MyCallback]
-
-        def callback(value):
-            self.assertTrue(isinstance(value, (int, long)))
-            return value & 0x7FFFFFFF
-
-        cb = MyCallback(callback)
-
-        self.assertEqual(13577625587, int(f(self.wrap(1000000000000), self.wrap(cb))))
-
-    def test_byval(self):
-        # without prototype
-        ptin = POINT(1, 2)
-        ptout = POINT()
-        # EXPORT int _testfunc_byval(point in, point *pout)
-        result = dll._testfunc_byval(ptin, byref(ptout))
-        got = result, ptout.x, ptout.y
-        expected = 3, 1, 2
-        self.assertEqual(got, expected)
-
-        # with prototype
-        ptin = POINT(101, 102)
-        ptout = POINT()
-        dll._testfunc_byval.argtypes = (POINT, POINTER(POINT))
-        dll._testfunc_byval.restype = c_int
-        result = dll._testfunc_byval(self.wrap(ptin), byref(ptout))
-        got = result, ptout.x, ptout.y
-        expected = 203, 101, 102
-        self.assertEqual(got, expected)
-
-    def test_struct_return_2H(self):
-        class S2H(Structure):
-            _fields_ = [("x", c_short),
-                        ("y", c_short)]
-        dll.ret_2h_func.restype = S2H
-        dll.ret_2h_func.argtypes = [S2H]
-        inp = S2H(99, 88)
-        s2h = dll.ret_2h_func(self.wrap(inp))
-        self.assertEqual((s2h.x, s2h.y), (99*2, 88*3))
-
-    def test_struct_return_8H(self):
-        class S8I(Structure):
-            _fields_ = [("a", c_int),
-                        ("b", c_int),
-                        ("c", c_int),
-                        ("d", c_int),
-                        ("e", c_int),
-                        ("f", c_int),
-                        ("g", c_int),
-                        ("h", c_int)]

From noreply at buildbot.pypy.org  Thu Apr 12 11:31:59 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 12 Apr 2012 11:31:59 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: run sympy benchmarks the normal
 number of times, they are fast
Message-ID: <20120412093159.73E5582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r181:062b90951d4b
Date: 2012-04-12 11:30 +0200
http://bitbucket.org/pypy/benchmarks/changeset/062b90951d4b/

Log:	run sympy benchmarks the normal number of times, they are fast

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -49,8 +49,7 @@
 for name in ['expand', 'integrate', 'sum', 'str']:
     _register_new_bm('bm_sympy', 'sympy_' + name,
                      globals(), bm_env={'PYTHONPATH': relative('lib/sympy')},
-                     extra_args=['--benchmark=' + name],
-                     iteration_scaling=0.1)
+                     extra_args=['--benchmark=' + name])
 
 for name in ['xml', 'text']:
     _register_new_bm('bm_genshi', 'genshi_' + name,

From noreply at buildbot.pypy.org  Thu Apr 12 12:17:22 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 12:17:22 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: draft next numpy status update
Message-ID: <20120412101722.2311982F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r4187:ab6a378f28e1
Date: 2012-04-12 10:47 +0200
http://bitbucket.org/pypy/extradoc/changeset/ab6a378f28e1/

Log:	draft next numpy status update

diff --git a/blog/draft/numpy-status-update-3.rst b/blog/draft/numpy-status-update-3.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/numpy-status-update-3.rst
@@ -0,0 +1,43 @@
+NumPy on PyPy progress report
+=============================
+
+Hello.
+
+A lot of things happened in March, like `pycon`_. I was also busy doing other
+things (pictured), so apologies for the late numpy status update.
+
+However, a lot of things have happened and numpy continues to be on of the
+main points of entry for hacking on PyPy. Apologies to all the people who's
+patches I don't review in timely manner, but seriously, you do **a lot** of
+work.
+
+The list of things is definitely not exhaustive, and I might be forgetting
+important contributions. In a loose order:
+
+* Matti Picus made ``out`` parameter working for a lot (but not all)
+  functions.
+
+* We merged record dtypes support. The only missing dtypes left are complex
+  (important), datetime (less important) and object (which will probably
+  never be implemented).
+
+* Taavi Burns and others implemented lots of details, including lots of ufuncs.
+  On the completely unscientific measure of "implemented functions" on
+  `numpypy status page`_, we're close to 50% of numpy working. In reality
+  it might be more or less, but after complex dtypes we're getting very close
+  to running real programs.
+
+* Bool indexing of arrays of the same size should work, leaving only
+  arrays-of-ints indexing as the last missing element of fancy indexing.
+
+* I did some very early experiments on SSE. This work is **seriously**
+  preliminary - in fact the only implemented operation is addition of
+  float single-dimension numpy arrays. However, results are encouraging,
+  given that our assembler generator is far from ideal:
+
+  +
+
+Next step would be to just continue implementing missing features. Future
+is hard to predict, but we're not far off!
+
+.. _`pycon`: http://us.pycon.org

From noreply at buildbot.pypy.org  Thu Apr 12 12:17:23 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 12:17:23 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20120412101723.5AFBF82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r4188:34f623b889f3
Date: 2012-04-12 12:17 +0200
http://bitbucket.org/pypy/extradoc/changeset/34f623b889f3/

Log:	merge

diff --git a/blog/draft/py3k-status-update-3.rst b/blog/draft/py3k-status-update-3.rst
--- a/blog/draft/py3k-status-update-3.rst
+++ b/blog/draft/py3k-status-update-3.rst
@@ -1,22 +1,22 @@
 Py3k status update #3
 ---------------------
 
-This is the third status update about my work on the `py3k branch`_, which I
+This is the third status update about our work on the `py3k branch`_, which we
 can work on thanks to all of the people who donated_ to the `py3k proposal`_.
 
 A lot of work has been done during the last month: as usual, the list of
-changes is too big to be reported in a detalied way, so this is just a summary
+changes is too big to be reported in a detailed way, so this is just a summary
 of what happened.
 
 One of the most active areas was killing old and deprecated features. In
 particular, we killed support for the ``__cmp__`` special method and its
-counsins, the ``cmp`` builtin function and keyword argument for
-``list.sort()`` and ``sorted()``.  Killing is easy, but then you have to fix
-all the places which breaks because of this, including all the types which
-relied on ``__cmp__`` to be comparable,, fixing all the tests which tried to
-order objects which are no longer ordeable now, or implementing new behavior
-like forbidding calling ``hash()`` on objects which implement ``__eq__`` but
-not ``__hash__``.
+counsins, the ``cmp`` builtin function and keyword argument for ``list.sort()``
+and ``sorted()``.  Killing is easy, but then you have to fix all the places
+which breaks because of this, including all the types which relied on
+``__cmp__`` to be comparable and all the tests which tried to order objects
+which are no longer ordeable. New behavior, like forbidding calling ``hash()``
+on objects which implement ``__eq__`` but not ``__hash__``, also has to be
+implemented.
 
 Among the other features, we killed lots of now-gone functions in the
 ``operator`` module, the builtins ``apply()``, ``reduce()`` and ``buffer``,
diff --git a/blog/draft/pycon-wrapup.rst b/blog/draft/pycon-wrapup.rst
--- a/blog/draft/pycon-wrapup.rst
+++ b/blog/draft/pycon-wrapup.rst
@@ -6,12 +6,13 @@
 
 From the PyPy perspective, a lot at PyCon was about PyPy. Listing things:
 
-* David Beazley did an excellent keynote on trying to dive head-first into
-  PyPy and at least partly failing. He however did not fail to explain
-  bits and pieces about PyPy's architecture. `Video`_ is available.
+* David Beazley presented an excellent keynote describing his experience
+  diving head-first into PyPy and at least partly failing. He, however, did
+  not fail to explain bits and pieces about PyPy's architecture.
+  `Video`_ is available.
 
 * We gave tons of talks, including the `tutorial`_, `why pypy by example`_
-  and `pypy's JIT architecturew`_
+  and `pypy's JIT architecture`_
 
 * We had a giant influx of new commiters, easily doubling the amount of pull
   requests ever created for PyPy. The main topics for newcomers were numpy and
@@ -23,7 +24,7 @@
 We would like to thank everyone who talked to us, shared ideas and especially
 those who participated in sprints - we're always happy to welcome newcomers!
 
-I'm sure there is tons of things I forgot, but thank you all!
+I'm sure there are tons of things I forgot, but thank you all!
 
 Cheers,
 fijal
@@ -31,5 +32,5 @@
 .. _`Video`: http://pyvideo.org/video/659/keynote-david-beazley
 .. _`tutorial`: http://pyvideo.org/video/612/how-to-get-the-most-out-of-your-pypy
 .. _`why pypy by example`: http://pyvideo.org/video/661/why-pypy-by-example
-.. _`pypy's JIT architecturew`: http://pyvideo.org/video/662/how-the-pypy-jit-works
+.. _`pypy's JIT architecture`: http://pyvideo.org/video/662/how-the-pypy-jit-works
 .. _`prove him correct`: http://mrjoes.github.com/2011/12/15/sockjs-bench.html

From noreply at buildbot.pypy.org  Thu Apr 12 14:15:49 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 14:15:49 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: a branch to experiment with
	pinning
Message-ID: <20120412121549.7972482F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54303:cb0a5d8aa8af
Date: 2012-04-12 13:52 +0200
http://bitbucket.org/pypy/pypy/changeset/cb0a5d8aa8af/

Log:	a branch to experiment with pinning

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -18,6 +18,24 @@
     """
     pass
 
+def pin(obj):
+    pass
+
+def unpin(obj):
+    pass
+
+
+class pinned_object(object):
+    def __init__(self, obj):
+        self.obj = obj
+        
+    def __enter__(self):
+        pin(self.obj)
+        return self
+    
+    def __exit__(self, *args):
+        unpin(self.obj)
+
 # ____________________________________________________________
 # Annotation and specialization
 
diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py
--- a/pypy/rlib/test/test_rgc.py
+++ b/pypy/rlib/test/test_rgc.py
@@ -171,3 +171,9 @@
     x1 = X()
     n = rgc.get_rpy_memory_usage(rgc.cast_instance_to_gcref(x1))
     assert n >= 8 and n <= 64
+
+def test_pin_obj():
+    l = []
+    with rgc.pinned_object(l):
+        l.append(3)
+    assert l == [3]

From noreply at buildbot.pypy.org  Thu Apr 12 14:15:51 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 14:15:51 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: Start a branch to
 experiment with pinning on the GC. Boilerplate so far
Message-ID: <20120412121551.9A24682F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54304:43d51ad02ec5
Date: 2012-04-12 14:14 +0200
http://bitbucket.org/pypy/pypy/changeset/43d51ad02ec5/

Log:	Start a branch to experiment with pinning on the GC. Boilerplate so
	far

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -493,3 +493,25 @@
     def specialize_call(self, hop):
         hop.exception_is_here()
         return hop.genop('gc_typeids_z', [], resulttype = hop.r_result)
+
+class PinEntry(ExtRegistryEntry):
+    _about_ = pin
+
+    def compute_result_annotation(self, s_arg):
+        pass
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        v_obj, = hop.inputargs(hop.args_r[0])
+        hop.genop('gc_pin', [v_obj])
+
+class UnpinEntry(ExtRegistryEntry):
+    _about_ = unpin
+
+    def compute_result_annotation(self, s_arg):
+        pass
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        v_obj, = hop.inputargs(hop.args_r[0])
+        hop.genop('gc_unpin', [v_obj])
diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py
--- a/pypy/rlib/test/test_rgc.py
+++ b/pypy/rlib/test/test_rgc.py
@@ -177,3 +177,12 @@
     with rgc.pinned_object(l):
         l.append(3)
     assert l == [3]
+
+def test_interp_pin_obj():
+    def f(i):
+        l = []
+        with rgc.pinned_object(l):
+            l.append(i)
+        return l[0]
+    
+    assert interpret(f, [3]) == 3
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -890,6 +890,12 @@
     def op_gc_stack_bottom(self):
         pass       # marker for trackgcroot.py
 
+    def op_gc_pin(self, obj):
+        self.heap.pin(obj)
+
+    def op_gc_unpin(self, obj):
+        self.heap.unpin(obj)
+
     def op_gc_shadowstackref_new(self):   # stacklet+shadowstack
         raise NotImplementedError("gc_shadowstackref_new")
     def op_gc_shadowstackref_context(self):
diff --git a/pypy/rpython/lltypesystem/llheap.py b/pypy/rpython/lltypesystem/llheap.py
--- a/pypy/rpython/lltypesystem/llheap.py
+++ b/pypy/rpython/lltypesystem/llheap.py
@@ -35,3 +35,10 @@
 
 def thread_die():
     pass
+
+def pin(obj):
+    pass
+
+def unpin(obj):
+    pass
+
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -469,6 +469,8 @@
     'gc_assume_young_pointers': LLOp(canrun=True),
     'gc_writebarrier_before_copy': LLOp(canrun=True),
     'gc_heap_stats'       : LLOp(canmallocgc=True),
+    'gc_pin'              : LLOp(canrun=True),
+    'gc_unpin'            : LLOp(canrun=True),
 
     'gc_get_rpy_roots'    : LLOp(),
     'gc_get_rpy_referents': LLOp(),

From noreply at buildbot.pypy.org  Thu Apr 12 14:16:08 2012
From: noreply at buildbot.pypy.org (mikefc)
Date: Thu, 12 Apr 2012 14:16:08 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: small changes
Message-ID: <20120412121608.BD93A82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Michael Cheng <coolbutuseless at gmail.com>
Branch: extradoc
Changeset: r4189:ea0070e7a081
Date: 2012-04-12 22:13 +1000
http://bitbucket.org/pypy/extradoc/changeset/ea0070e7a081/

Log:	small changes

diff --git a/blog/draft/numpy-status-update-3.rst b/blog/draft/numpy-status-update-3.rst
--- a/blog/draft/numpy-status-update-3.rst
+++ b/blog/draft/numpy-status-update-3.rst
@@ -6,15 +6,15 @@
 A lot of things happened in March, like `pycon`_. I was also busy doing other
 things (pictured), so apologies for the late numpy status update.
 
-However, a lot of things have happened and numpy continues to be on of the
-main points of entry for hacking on PyPy. Apologies to all the people who's
+However, a lot of things have happened and numpy continues to be one of the
+main points of entry for hacking on PyPy. Apologies to all the people whose
 patches I don't review in timely manner, but seriously, you do **a lot** of
 work.
 
 The list of things is definitely not exhaustive, and I might be forgetting
 important contributions. In a loose order:
 
-* Matti Picus made ``out`` parameter working for a lot (but not all)
+* Matti Picus made ``out`` parameter work for a lot of (but not all)
   functions.
 
 * We merged record dtypes support. The only missing dtypes left are complex

From noreply at buildbot.pypy.org  Thu Apr 12 14:36:14 2012
From: noreply at buildbot.pypy.org (mikefc)
Date: Thu, 12 Apr 2012 14:36:14 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: more ToDos mentioned at end
Message-ID: <20120412123614.9001D82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Michael Cheng <coolbutuseless at gmail.com>
Branch: extradoc
Changeset: r4190:bfc2761330f2
Date: 2012-04-12 22:35 +1000
http://bitbucket.org/pypy/extradoc/changeset/bfc2761330f2/

Log:	more ToDos mentioned at end

diff --git a/blog/draft/numpy-status-update-3.rst b/blog/draft/numpy-status-update-3.rst
--- a/blog/draft/numpy-status-update-3.rst
+++ b/blog/draft/numpy-status-update-3.rst
@@ -11,7 +11,7 @@
 patches I don't review in timely manner, but seriously, you do **a lot** of
 work.
 
-The list of things is definitely not exhaustive, and I might be forgetting
+This list of changes is definitely not exhaustive, and I might be forgetting
 important contributions. In a loose order:
 
 * Matti Picus made ``out`` parameter work for a lot of (but not all)
@@ -19,7 +19,7 @@
 
 * We merged record dtypes support. The only missing dtypes left are complex
   (important), datetime (less important) and object (which will probably
-  never be implemented).
+  never be implemented because XXXXXXX).
 
 * Taavi Burns and others implemented lots of details, including lots of ufuncs.
   On the completely unscientific measure of "implemented functions" on
@@ -37,7 +37,15 @@
 
   +
 
-Next step would be to just continue implementing missing features. Future
-is hard to predict, but we're not far off!
+Next step would be to just continue implementing missing features such as
+
+* specialised arrays i.e. masked arrays and matrixes
+
+* core modules such as ``fft``, ``linalg``, ``random``.  
+
+* numpy's testing framework
+
+The future is hard to predict, but we're not far off!
 
 .. _`pycon`: http://us.pycon.org
+.. _`numpypy status page`: http://buildbot.pypy.org/numpy-status/latest.html

From noreply at buildbot.pypy.org  Thu Apr 12 14:55:17 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 12 Apr 2012 14:55:17 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: import py missing
Message-ID: <20120412125517.8F37882F4E@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54305:7b4d81251b58
Date: 2012-04-12 14:55 +0200
http://bitbucket.org/pypy/pypy/changeset/7b4d81251b58/

Log:	import py missing

diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -1,3 +1,4 @@
+import py
 from pypy.conftest import gettestobjspace, option
 from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
 from pypy.objspace.std.kwargsdict import *

From noreply at buildbot.pypy.org  Thu Apr 12 15:13:33 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 12 Apr 2012 15:13:33 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: optimize the keys method
Message-ID: <20120412131333.DCB5382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54306:f37a2d944ce1
Date: 2012-04-12 15:13 +0200
http://bitbucket.org/pypy/pypy/changeset/f37a2d944ce1/

Log:	optimize the keys method

diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -102,8 +102,8 @@
             return w_dict.getitem(w_key)
 
     def w_keys(self, w_dict):
-        l = [self.wrap(key) for key in self.unerase(w_dict.dstorage)[0]]
-        return self.space.newlist(l)
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
 
     def values(self, w_dict):
         return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -76,6 +76,17 @@
     assert d.strategy is not strategy
     assert "StringDictStrategy" == d.strategy.__class__.__name__
 
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
 from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
 def get_impl(self):
     storage = strategy.erase(([], []))

From noreply at buildbot.pypy.org  Thu Apr 12 15:23:46 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 12 Apr 2012 15:23:46 +0200 (CEST)
Subject: [pypy-commit] pypy kwargsdict-strategy: close before impending merge
Message-ID: <20120412132346.C5E7C82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: kwargsdict-strategy
Changeset: r54307:c02cd7a449c0
Date: 2012-04-12 15:22 +0200
http://bitbucket.org/pypy/pypy/changeset/c02cd7a449c0/

Log:	close before impending merge


From noreply at buildbot.pypy.org  Thu Apr 12 15:23:48 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 12 Apr 2012 15:23:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge kwargsdict-strategy:
Message-ID: <20120412132348.B09EA82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54308:97c57afceef4
Date: 2012-04-12 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/97c57afceef4/

Log:	merge kwargsdict-strategy:

	add a new dict strategy that is created for being passed into **args
	arguments. This makes decorators that just receive **args and pass
	them on to another call a lot faster.

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -169,9 +169,11 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +188,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +226,26 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
+        if self.keywords is None:
+            self.keywords = keywords[:] # copy to make non-resizable
+            self.keywords_w = keywords_w[:]
+        else:
+            # looks quadratic, but the JIT should remove all of it nicely.
+            # Also, all the lists should be small
+            for key in keywords:
+                for otherkey in self.keywords:
+                    if otherkey == key:
+                        raise operationerrfmt(self.space.w_TypeError,
+                                              "got multiple values "
+                                              "for keyword argument "
+                                              "'%s'", key)
+            self.keywords = self.keywords + keywords
+            self.keywords_w = self.keywords_w + keywords_w
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +404,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+

From noreply at buildbot.pypy.org  Thu Apr 12 15:24:48 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 12 Apr 2012 15:24:48 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: adapt sandbox interaction
 test to unified site
Message-ID: <20120412132448.4554382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54309:08c38e4707f6
Date: 2012-04-12 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/08c38e4707f6/

Log:	adapt sandbox interaction test to unified site

diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -42,8 +42,6 @@
     lib_python = os.path.join(prefix, 'lib-python')
     python_std_lib = os.path.join(lib_python, dirname)
     checkdir(python_std_lib)
-    python_std_lib_modified = os.path.join(lib_python, 'modified-' + dirname)
-    checkdir(python_std_lib_modified)
     
     lib_pypy = os.path.join(prefix, 'lib_pypy')
     checkdir(lib_pypy)
@@ -56,12 +54,9 @@
         importlist.append(lib_extensions)
     #
     importlist.append(lib_pypy)
-    importlist.append(python_std_lib_modified)
     importlist.append(python_std_lib)
     #
-    lib_tk_modified = os.path.join(python_std_lib_modified, 'lib-tk')
     lib_tk = os.path.join(python_std_lib, 'lib-tk')
-    importlist.append(lib_tk_modified)
     importlist.append(lib_tk)
     #
     # List here the extra platform-specific paths.
diff --git a/pypy/tool/lib_pypy.py b/pypy/tool/lib_pypy.py
--- a/pypy/tool/lib_pypy.py
+++ b/pypy/tool/lib_pypy.py
@@ -5,9 +5,7 @@
 
 LIB_ROOT = py.path.local(pypy.__path__[0]).dirpath()
 LIB_PYPY =  LIB_ROOT.join('lib_pypy')
-LIB_PYTHON = LIB_ROOT.join('lib-python')
-LIB_PYTHON_VANILLA = LIB_PYTHON.join('%d.%d' % CPYTHON_VERSION[:2])
-LIB_PYTHON_MODIFIED = LIB_PYTHON.join('modified-%d.%d' % CPYTHON_VERSION[:2])
+LIB_PYTHON = LIB_ROOT.join('lib-python', '%d.%d' % CPYTHON_VERSION[:2])
 
 
 def import_from_lib_pypy(modname):
diff --git a/pypy/tool/stdlib_opcode.py b/pypy/tool/stdlib_opcode.py
--- a/pypy/tool/stdlib_opcode.py
+++ b/pypy/tool/stdlib_opcode.py
@@ -106,8 +106,8 @@
     opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT)
 
 def load_pypy_opcode():
-    from pypy.tool.lib_pypy import LIB_PYTHON_MODIFIED
-    opcode_path = LIB_PYTHON_MODIFIED.join('opcode.py')
+    from pypy.tool.lib_pypy import LIB_PYTHON
+    opcode_path = LIB_PYTHON.join('opcode.py')
     d = {}
     execfile(str(opcode_path), d)
     for name in __all__:
diff --git a/pypy/translator/sandbox/test/test_pypy_interact.py b/pypy/translator/sandbox/test/test_pypy_interact.py
--- a/pypy/translator/sandbox/test/test_pypy_interact.py
+++ b/pypy/translator/sandbox/test/test_pypy_interact.py
@@ -2,14 +2,12 @@
 import os, sys, stat, errno
 from pypy.translator.sandbox.pypy_interact import PyPySandboxedProc
 from pypy.translator.interactive import Translation
+
 from pypy.module.sys.version import CPYTHON_VERSION
+from pypy.tool.lib_pypy import LIB_PYTHON
 
 VERSION = '%d.%d' % CPYTHON_VERSION[:2]
-SITE_PY_CONTENT = open(os.path.join(autopath.pypydir,
-                                    '..',
-                                    'lib-python',
-                                    'modified-' + VERSION, 'site.py'),
-                       'rb').read()
+SITE_PY_CONTENT = LIB_PYTHON.join('site.py').read()
 ERROR_TEXT = os.strerror(errno.ENOENT)
 
 def assert_(cond, text):
@@ -39,15 +37,14 @@
         pass
     else:
         assert_(False, "os.stat('site') should have failed")
-    st = os.stat('/bin/lib-python/modified-%s/site.py' % VERSION)
-    assert_(stat.S_ISREG(st.st_mode), "bad st_mode for .../site.py")
+
     try:
-        os.stat('/bin/lib-python/modified-%s/site.pyc' % VERSION)
+        os.stat('/bin/lib-python/%s/site.pyc' % VERSION)
     except OSError:
         pass
     else:
         assert_(False, "os.stat('....pyc') should have failed")
-    fd = os.open('/bin/lib-python/modified-%s/site.py' % VERSION,
+    fd = os.open('/bin/lib-python/%s/site.py' % VERSION,
                  os.O_RDONLY, 0666)
     length = 8192
     ofs = 0

From noreply at buildbot.pypy.org  Thu Apr 12 15:37:00 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 15:37:00 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: test and more boilerplate
Message-ID: <20120412133700.51DC082F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54310:afd55b84ac33
Date: 2012-04-12 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/afd55b84ac33/

Log:	test and more boilerplate

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -503,7 +503,9 @@
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
         v_obj, = hop.inputargs(hop.args_r[0])
-        hop.genop('gc_pin', [v_obj])
+        v_addr = hop.genop('cast_ptr_to_adr', [v_obj],
+                           resulttype=llmemory.Address)
+        hop.genop('gc_pin', [v_addr])
 
 class UnpinEntry(ExtRegistryEntry):
     _about_ = unpin
@@ -514,4 +516,6 @@
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
         v_obj, = hop.inputargs(hop.args_r[0])
-        hop.genop('gc_unpin', [v_obj])
+        v_addr = hop.genop('cast_ptr_to_adr', [v_obj],
+                           resulttype=llmemory.Address)
+        hop.genop('gc_unpin', [v_addr])
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -18,9 +18,11 @@
     needs_write_barrier = False
     malloc_zero_filled = False
     prebuilt_gc_objects_are_static_roots = True
+    can_always_pin_objects = False
     object_minimal_size = 0
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
 
+
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
                  translated_to_c=True):
         self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
@@ -176,6 +178,12 @@
     def can_move(self, addr):
         return False
 
+    def pin(self, addr):
+        pass
+
+    def unpin(self, addr):
+        pass
+
     def set_max_heap_size(self, size):
         raise NotImplementedError
 
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -130,6 +130,7 @@
     needs_write_barrier = True
     prebuilt_gc_objects_are_static_roots = False
     malloc_zero_filled = True    # xxx experiment with False
+    can_always_pin_objects = True
     gcflag_extra = GCFLAG_FINALIZATION_ORDERING
 
     # All objects start with a HDR, i.e. with a field 'tid' which contains
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -123,6 +123,12 @@
     def can_move(self, addr):
         return self.gc.can_move(addr)
 
+    def pin(self, addr):
+        self.gc.pin(addr)
+
+    def unpin(self, addr):
+        self.gc.unpin(addr)    
+
     def weakref_create_getlazy(self, objgetter):
         # we have to be lazy in reading the llinterp variable containing
         # the 'obj' pointer, because the gc.malloc() call below could
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -746,6 +746,25 @@
         res = self.interpret(fn, [])
         assert res == ord('y')
 
+    def test_pinning(self):
+        def f(i):
+            s = str(i)
+            if not rgc.can_move(s):
+                return 13
+            sum = 0
+            with rgc.pinned_object(s):
+                sum += int(rgc.can_move(s))
+            sum += 10 * int(rgc.can_move(s))
+            return sum
+
+        res = self.interpret(f, [10])
+        if not self.GCClass.moving_gc:
+            assert res == 13
+        elif self.GCClass.can_always_pin_objects:
+            assert res == 10
+        else:
+            assert res == 11
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):

From noreply at buildbot.pypy.org  Thu Apr 12 15:37:01 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 15:37:01 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: enough to make the test
 pass, but obviously it does not work
Message-ID: <20120412133701.92B1482F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54311:55a65a85e0d6
Date: 2012-04-12 15:32 +0200
http://bitbucket.org/pypy/pypy/changeset/55a65a85e0d6/

Log:	enough to make the test pass, but obviously it does not work

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -113,8 +113,9 @@
 # one bit per 'card_page_indices' indices.
 GCFLAG_HAS_CARDS    = first_gcflag << 5
 GCFLAG_CARDS_SET    = first_gcflag << 6     # <- at least one card bit is set
+GCFLAG_PINNED       = first_gcflag << 7
 
-TID_MASK            = (first_gcflag << 7) - 1
+TID_MASK            = (first_gcflag << 8) - 1
 
 
 FORWARDSTUB = lltype.GcStruct('forwarding_stub',
@@ -304,6 +305,13 @@
         # GCFLAG_HAS_SHADOW to their future location at the next
         # minor collection.
         self.nursery_objects_shadows = self.AddressDict()
+        # all pinned objects in the nursery
+        self.pinned_objects = self.AddressStack()
+        # all pinned objects that were in the nursery *before* last
+        # minor collect. This is a sorted stack that should be consulted when
+        # considering next nursery ceiling
+        self.nursery_barriers = self.AddressStack()
+    
         #
         # Allocate a nursery.  In case of auto_nursery_size, start by
         # allocating a very small nursery, enough to do things like look
@@ -762,8 +770,14 @@
 
     def can_move(self, obj):
         """Overrides the parent can_move()."""
-        return self.is_in_nursery(obj)
+        return (self.is_in_nursery(obj) and
+                not self.header(obj).tid & GCFLAG_PINNED)
 
+    def pin(self, obj):
+        self.header(obj).tid |= GCFLAG_PINNED
+
+    def unpin(self, obj):
+        self.header(obj).tid &= ~GCFLAG_PINNED
 
     def shrink_array(self, obj, smallerlength):
         #
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -763,7 +763,7 @@
         elif self.GCClass.can_always_pin_objects:
             assert res == 10
         else:
-            assert res == 11
+            assert res == 11 or res == 13 # sometimes fresh objs can't move
 
 from pypy.rlib.objectmodel import UnboxedValue
 

From noreply at buildbot.pypy.org  Thu Apr 12 15:39:11 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 12 Apr 2012 15:39:11 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: fix the note on using the
	stdlib test_set
Message-ID: <20120412133911.BE83782F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54312:e0fe8d719183
Date: 2012-04-12 15:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e0fe8d719183/

Log:	fix the note on using the stdlib test_set

diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -1,9 +1,10 @@
 """
 The main test for the set implementation is located
-at:
-    pypy-dist/lib-python/modified-2.5.2/test/test_set.py
-    go there and invoke
+in the stdlibs test/test_set.py which is located in lib-python
+go there and invoke::
+
     ../../../pypy/bin/py.py test_set.py
+
 This file just contains some basic tests that make sure, the implementation
 is not too wrong.
 """

From noreply at buildbot.pypy.org  Thu Apr 12 15:39:13 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 12 Apr 2012 15:39:13 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: todo note in tool/pytest
Message-ID: <20120412133913.0049282F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54313:4a9a542ed7e4
Date: 2012-04-12 15:38 +0200
http://bitbucket.org/pypy/pypy/changeset/4a9a542ed7e4/

Log:	todo note in tool/pytest

diff --git a/pypy/tool/pytest/htmlreport.py b/pypy/tool/pytest/htmlreport.py
--- a/pypy/tool/pytest/htmlreport.py
+++ b/pypy/tool/pytest/htmlreport.py
@@ -75,6 +75,7 @@
 
     def render_test_references(self, result): 
         dest = self.make_single_test_result(result)
+        #XXX: ask hg for differences between test and vendor branch
         modified = result.ismodifiedtest() and " [mod]" or ""
         return html.div(html.a(result.path.purebasename + modified, 
                       href=self.getrelpath(dest)),

From noreply at buildbot.pypy.org  Thu Apr 12 16:35:50 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 12 Apr 2012 16:35:50 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: rework zipfile and test_zipfile to
 respect open files
Message-ID: <20120412143550.E66BD82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54314:ba500a555707
Date: 2012-04-12 17:35 +0300
http://bitbucket.org/pypy/pypy/changeset/ba500a555707/

Log:	rework zipfile and test_zipfile to respect open files

diff --git a/lib-python/2.7/test/test_zipfile.py b/lib-python/modified-2.7/test/test_zipfile.py
copy from lib-python/2.7/test/test_zipfile.py
copy to lib-python/modified-2.7/test/test_zipfile.py
--- a/lib-python/2.7/test/test_zipfile.py
+++ b/lib-python/modified-2.7/test/test_zipfile.py
@@ -234,8 +234,9 @@
 
         # Read the ZIP archive
         with zipfile.ZipFile(f, "r") as zipfp:
-            for line, zipline in zip(self.line_gen, zipfp.open(TESTFN)):
-                self.assertEqual(zipline, line + '\n')
+            with zipfp.open(TESTFN) as f:
+                for line, zipline in zip(self.line_gen, f):
+                    self.assertEqual(zipline, line + '\n')
 
     def test_readline_read_stored(self):
         # Issue #7610: calls to readline() interleaved with calls to read().
@@ -340,7 +341,8 @@
         produces the expected result."""
         with zipfile.ZipFile(TESTFN2, "w") as zipfp:
             zipfp.write(TESTFN)
-            self.assertEqual(zipfp.read(TESTFN), open(TESTFN).read())
+            with open(TESTFN) as f:
+                self.assertEqual(zipfp.read(TESTFN), f.read())
 
     @skipUnless(zlib, "requires zlib")
     def test_per_file_compression(self):
@@ -382,7 +384,8 @@
                 self.assertEqual(writtenfile, correctfile)
 
                 # make sure correct data is in correct file
-                self.assertEqual(fdata, open(writtenfile, "rb").read())
+                with open(writtenfile, "rb") as fid:
+                    self.assertEqual(fdata, fid.read())
                 os.remove(writtenfile)
 
         # remove the test file subdirectories
@@ -401,24 +404,25 @@
                 else:
                     outfile = os.path.join(os.getcwd(), fpath)
 
-                self.assertEqual(fdata, open(outfile, "rb").read())
+                with  open(outfile, "rb") as fid:
+                    self.assertEqual(fdata, fid.read())
                 os.remove(outfile)
 
         # remove the test file subdirectories
         shutil.rmtree(os.path.join(os.getcwd(), 'ziptest2dir'))
 
     def test_writestr_compression(self):
-        zipfp = zipfile.ZipFile(TESTFN2, "w")
-        zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED)
-        if zlib:
-            zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED)
+        with zipfile.ZipFile(TESTFN2, "w") as zipfp:
+            zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED)
+            if zlib:
+                zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED)
 
-        info = zipfp.getinfo('a.txt')
-        self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
+            info = zipfp.getinfo('a.txt')
+            self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
 
-        if zlib:
-            info = zipfp.getinfo('b.txt')
-            self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
+            if zlib:
+                info = zipfp.getinfo('b.txt')
+                self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
 
 
     def zip_test_writestr_permissions(self, f, compression):
@@ -646,7 +650,8 @@
 
     def test_write_non_pyfile(self):
         with zipfile.PyZipFile(TemporaryFile(), "w") as zipfp:
-            open(TESTFN, 'w').write('most definitely not a python file')
+            with open(TESTFN, 'w') as f:
+                f.write('most definitely not a python file')
             self.assertRaises(RuntimeError, zipfp.writepy, TESTFN)
             os.remove(TESTFN)
 
@@ -795,7 +800,8 @@
         self.assertRaises(RuntimeError, zipf.open, "foo.txt")
         self.assertRaises(RuntimeError, zipf.testzip)
         self.assertRaises(RuntimeError, zipf.writestr, "bogus.txt", "bogus")
-        open(TESTFN, 'w').write('zipfile test data')
+        with open(TESTFN, 'w') as fp:
+            fp.write('zipfile test data')
         self.assertRaises(RuntimeError, zipf.write, TESTFN)
 
     def test_bad_constructor_mode(self):
@@ -803,7 +809,6 @@
         self.assertRaises(RuntimeError, zipfile.ZipFile, TESTFN, "q")
 
     def test_bad_open_mode(self):
-        """Check that bad modes passed to ZipFile.open are caught."""
         with zipfile.ZipFile(TESTFN, mode="w") as zipf:
             zipf.writestr("foo.txt", "O, for a Muse of Fire!")
 
@@ -851,7 +856,6 @@
 
     def test_comments(self):
         """Check that comments on the archive are handled properly."""
-
         # check default comment is empty
         with zipfile.ZipFile(TESTFN, mode="w") as zipf:
             self.assertEqual(zipf.comment, '')
@@ -953,14 +957,16 @@
         with zipfile.ZipFile(TESTFN, mode="w") as zipf:
             pass
         try:
-            zipf = zipfile.ZipFile(TESTFN, mode="r")
+            with zipfile.ZipFile(TESTFN, mode="r") as zipf:
+                pass
         except zipfile.BadZipfile:
             self.fail("Unable to create empty ZIP file in 'w' mode")
 
         with zipfile.ZipFile(TESTFN, mode="a") as zipf:
             pass
         try:
-            zipf = zipfile.ZipFile(TESTFN, mode="r")
+            with zipfile.ZipFile(TESTFN, mode="r") as zipf:
+                pass
         except:
             self.fail("Unable to create empty ZIP file in 'a' mode")
 
@@ -1160,6 +1166,8 @@
             data1 += zopen1.read(500)
             data2 += zopen2.read(500)
             self.assertEqual(data1, data2)
+            zopen1.close()
+            zopen2.close()
 
     def test_different_file(self):
         # Verify that (when the ZipFile is in control of creating file objects)
@@ -1207,9 +1215,9 @@
 
     def test_store_dir(self):
         os.mkdir(os.path.join(TESTFN2, "x"))
-        zipf = zipfile.ZipFile(TESTFN, "w")
-        zipf.write(os.path.join(TESTFN2, "x"), "x")
-        self.assertTrue(zipf.filelist[0].filename.endswith("x/"))
+        with zipfile.ZipFile(TESTFN, "w") as zipf:
+            zipf.write(os.path.join(TESTFN2, "x"), "x")
+            self.assertTrue(zipf.filelist[0].filename.endswith("x/"))
 
     def tearDown(self):
         shutil.rmtree(TESTFN2)
@@ -1226,7 +1234,8 @@
         for n, s in enumerate(self.seps):
             self.arcdata[s] = s.join(self.line_gen) + s
             self.arcfiles[s] = '%s-%d' % (TESTFN, n)
-            open(self.arcfiles[s], "wb").write(self.arcdata[s])
+            with open(self.arcfiles[s], "wb") as f:
+                f.write(self.arcdata[s])
 
     def make_test_archive(self, f, compression):
         # Create the ZIP archive
@@ -1295,8 +1304,9 @@
         # Read the ZIP archive
         with zipfile.ZipFile(f, "r") as zipfp:
             for sep, fn in self.arcfiles.items():
-                for line, zipline in zip(self.line_gen, zipfp.open(fn, "rU")):
-                    self.assertEqual(zipline, line + '\n')
+                with zipfp.open(fn, "rU") as f:
+                    for line, zipline in zip(self.line_gen, f):
+                        self.assertEqual(zipline, line + '\n')
 
     def test_read_stored(self):
         for f in (TESTFN2, TemporaryFile(), StringIO()):
diff --git a/lib-python/2.7/zipfile.py b/lib-python/modified-2.7/zipfile.py
copy from lib-python/2.7/zipfile.py
copy to lib-python/modified-2.7/zipfile.py
--- a/lib-python/2.7/zipfile.py
+++ b/lib-python/modified-2.7/zipfile.py
@@ -648,6 +648,10 @@
         return data
 
 
+class ZipExtFileWithClose(ZipExtFile):
+    def close(self):
+        self._fileobj.close()
+
 
 class ZipFile:
     """ Class with methods to open, read, write, close, list zip files.
@@ -843,9 +847,9 @@
             try:
                 # Read by chunks, to avoid an OverflowError or a
                 # MemoryError with very large embedded files.
-                f = self.open(zinfo.filename, "r")
-                while f.read(chunk_size):     # Check CRC-32
-                    pass
+                with self.open(zinfo.filename, "r") as f:
+                    while f.read(chunk_size):     # Check CRC-32
+                        pass
             except BadZipfile:
                 return zinfo.filename
 
@@ -864,7 +868,9 @@
 
     def read(self, name, pwd=None):
         """Return file bytes (as a string) for name."""
-        return self.open(name, "r", pwd).read()
+        with self.open(name, "r", pwd) as f:
+            retval = f.read()
+            return retval
 
     def open(self, name, mode="r", pwd=None):
         """Return file-like object for 'name'."""
@@ -881,59 +887,66 @@
         else:
             zef_file = open(self.filename, 'rb')
 
-        # Make sure we have an info object
-        if isinstance(name, ZipInfo):
-            # 'name' is already an info object
-            zinfo = name
+        try:
+            # Make sure we have an info object
+            if isinstance(name, ZipInfo):
+                # 'name' is already an info object
+                zinfo = name
+            else:
+                # Get info object for name
+                zinfo = self.getinfo(name)
+
+            zef_file.seek(zinfo.header_offset, 0)
+
+            # Skip the file header:
+            fheader = zef_file.read(sizeFileHeader)
+            if fheader[0:4] != stringFileHeader:
+                raise BadZipfile, "Bad magic number for file header"
+
+            fheader = struct.unpack(structFileHeader, fheader)
+            fname = zef_file.read(fheader[_FH_FILENAME_LENGTH])
+            if fheader[_FH_EXTRA_FIELD_LENGTH]:
+                zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
+
+            if fname != zinfo.orig_filename:
+                raise BadZipfile, \
+                          'File name in directory "%s" and header "%s" differ.' % (
+                              zinfo.orig_filename, fname)
+
+            # check for encrypted flag & handle password
+            is_encrypted = zinfo.flag_bits & 0x1
+            zd = None
+            if is_encrypted:
+                if not pwd:
+                    pwd = self.pwd
+                if not pwd:
+                    raise RuntimeError, "File %s is encrypted, " \
+                          "password required for extraction" % name
+
+                zd = _ZipDecrypter(pwd)
+                # The first 12 bytes in the cypher stream is an encryption header
+                #  used to strengthen the algorithm. The first 11 bytes are
+                #  completely random, while the 12th contains the MSB of the CRC,
+                #  or the MSB of the file time depending on the header type
+                #  and is used to check the correctness of the password.
+                bytes = zef_file.read(12)
+                h = map(zd, bytes[0:12])
+                if zinfo.flag_bits & 0x8:
+                    # compare against the file type from extended local headers
+                    check_byte = (zinfo._raw_time >> 8) & 0xff
+                else:
+                    # compare against the CRC otherwise
+                    check_byte = (zinfo.CRC >> 24) & 0xff
+                if ord(h[11]) != check_byte:
+                    raise RuntimeError("Bad password for file", name)
+        except:
+            if not self._filePassed:
+                zef_file.close()
+            raise    
+        if self._filePassed:
+            return  ZipExtFile(zef_file, mode, zinfo, zd)
         else:
-            # Get info object for name
-            zinfo = self.getinfo(name)
-
-        zef_file.seek(zinfo.header_offset, 0)
-
-        # Skip the file header:
-        fheader = zef_file.read(sizeFileHeader)
-        if fheader[0:4] != stringFileHeader:
-            raise BadZipfile, "Bad magic number for file header"
-
-        fheader = struct.unpack(structFileHeader, fheader)
-        fname = zef_file.read(fheader[_FH_FILENAME_LENGTH])
-        if fheader[_FH_EXTRA_FIELD_LENGTH]:
-            zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
-
-        if fname != zinfo.orig_filename:
-            raise BadZipfile, \
-                      'File name in directory "%s" and header "%s" differ.' % (
-                          zinfo.orig_filename, fname)
-
-        # check for encrypted flag & handle password
-        is_encrypted = zinfo.flag_bits & 0x1
-        zd = None
-        if is_encrypted:
-            if not pwd:
-                pwd = self.pwd
-            if not pwd:
-                raise RuntimeError, "File %s is encrypted, " \
-                      "password required for extraction" % name
-
-            zd = _ZipDecrypter(pwd)
-            # The first 12 bytes in the cypher stream is an encryption header
-            #  used to strengthen the algorithm. The first 11 bytes are
-            #  completely random, while the 12th contains the MSB of the CRC,
-            #  or the MSB of the file time depending on the header type
-            #  and is used to check the correctness of the password.
-            bytes = zef_file.read(12)
-            h = map(zd, bytes[0:12])
-            if zinfo.flag_bits & 0x8:
-                # compare against the file type from extended local headers
-                check_byte = (zinfo._raw_time >> 8) & 0xff
-            else:
-                # compare against the CRC otherwise
-                check_byte = (zinfo.CRC >> 24) & 0xff
-            if ord(h[11]) != check_byte:
-                raise RuntimeError("Bad password for file", name)
-
-        return  ZipExtFile(zef_file, mode, zinfo, zd)
+            return  ZipExtFileWithClose(zef_file, mode, zinfo, zd)
 
     def extract(self, member, path=None, pwd=None):
         """Extract a member from the archive to the current working directory,
@@ -989,7 +1002,6 @@
             if not os.path.isdir(targetpath):
                 os.mkdir(targetpath)
             return targetpath
-
         source = self.open(member, pwd=pwd)
         target = file(targetpath, "wb")
         shutil.copyfileobj(source, target)

From noreply at buildbot.pypy.org  Thu Apr 12 17:43:18 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 12 Apr 2012 17:43:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the test. The issue is not
 arg+memo. The issue is that you cannot
Message-ID: <20120412154318.104B182F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54315:1da1c1632353
Date: 2012-04-12 17:41 +0200
http://bitbucket.org/pypy/pypy/changeset/1da1c1632353/

Log:	Fix the test. The issue is not arg+memo. The issue is that you
	cannot call the specialize:arg function f with "f(i)", even if you
	just did "if i == 2" before.

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 

From noreply at buildbot.pypy.org  Thu Apr 12 22:02:18 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 22:02:18 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: enough support for
 nursery_barriers to pass one test, still xxx left
Message-ID: <20120412200218.457FB82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54316:399757c26f46
Date: 2012-04-12 22:01 +0200
http://bitbucket.org/pypy/pypy/changeset/399757c26f46/

Log:	enough support for nursery_barriers to pass one test, still xxx left

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -305,12 +305,10 @@
         # GCFLAG_HAS_SHADOW to their future location at the next
         # minor collection.
         self.nursery_objects_shadows = self.AddressDict()
-        # all pinned objects in the nursery
-        self.pinned_objects = self.AddressStack()
         # all pinned objects that were in the nursery *before* last
         # minor collect. This is a sorted stack that should be consulted when
         # considering next nursery ceiling
-        self.nursery_barriers = self.AddressStack()
+        self.nursery_barriers = self.AddressDeque()
     
         #
         # Allocate a nursery.  In case of auto_nursery_size, start by
@@ -452,6 +450,7 @@
 
     def debug_rotate_nursery(self):
         if self.debug_rotating_nurseries is not None:
+            ll_assert(self.nursery_barriers.empty(), "non empty nursery barriers with rotating nursery")
             debug_start("gc-debug")
             oldnurs = self.nursery
             llarena.arena_protect(oldnurs, self._nursery_memory_size(), True)
@@ -587,22 +586,27 @@
         and finally reserve 'totalsize' bytes at the start of the
         now-empty nursery.
         """
-        self.minor_collection()
+        if not self.nursery_top == self.nursery + self.nursery_size:
+            xxx
+        self.minor_collection(totalsize)
+        # try allocating now, otherwise we do a major collect
+        do_major_collect = False
         #
-        if self.get_total_memory_used() > self.next_major_collection_threshold:
+        if do_major_collect or (self.get_total_memory_used() > self.next_major_collection_threshold):
             self.major_collection()
             #
             # The nursery might not be empty now, because of
             # execute_finalizers().  If it is almost full again,
             # we need to fix it with another call to minor_collection().
             if self.nursery_free + totalsize > self.nursery_top:
-                self.minor_collection()
+                self.minor_collection(totalsize)
         #
         result = self.nursery_free
         self.nursery_free = result + totalsize
         ll_assert(self.nursery_free <= self.nursery_top, "nursery overflow")
         #
         if self.debug_tiny_nursery >= 0:   # for debugging
+            ll_assert(not self.nursery_barriers.non_empty(), "no support for nursery debug and pinning")
             if self.nursery_top - self.nursery_free > self.debug_tiny_nursery:
                 self.nursery_free = self.nursery_top - self.debug_tiny_nursery
         #
@@ -1245,7 +1249,7 @@
     # ----------
     # Nursery collection
 
-    def minor_collection(self):
+    def minor_collection(self, min_size=0):
         """Perform a minor collection: find the objects from the nursery
         that remain alive and move them out."""
         #
@@ -1303,9 +1307,34 @@
         #
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
-        llarena.arena_reset(self.nursery, self.nursery_size, 2)
+        # self.nursery_barriers are *not* freed
+        # XXX sort the nursery_barriers
+        new_barriers = self.AddressDeque()
+        prev = self.nursery
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        while self.nursery_barriers.non_empty():
+            next = self.nursery_barriers.popleft()
+            llarena.arena_reset(prev, next - prev, 2)
+            # clean the visited flag
+            obj = next + size_gc_header
+            self.header(obj).tid &= ~GCFLAG_VISITED
+            prev = prev + (next - prev) + (size_gc_header +
+                           self.get_size(obj))
+            new_barriers.append(next)
+        llarena.arena_reset(prev, self.nursery_top - prev, 2)
+        self.nursery_barriers.delete()
+        self.nursery_barriers = new_barriers
         self.debug_rotate_nursery()
         self.nursery_free = self.nursery
+        self.nursery_barriers.append(self.nursery + self.nursery_size)
+        self.nursery_top = self.nursery_barriers.popleft()
+        while self.nursery_barriers.non_empty() and self.nursery_free + min_size > self.nursery_top:
+            cur_obj_size = size_gc_header + self.get_size(self.nursery_free +
+                                                          size_gc_header)
+            self.nursery_free = self.nursery_free + cur_obj_size
+            self.nursery_top = self.nursery_barriers.popleft()
+        if self.nursery_free + min_size > self.nursery_top:
+            ll_assert(False, "too many pinned objects")
         #
         debug_print("minor collect, total memory used:",
                     self.get_total_memory_used())
@@ -1442,7 +1471,16 @@
             return
         #
         size_gc_header = self.gcheaderbuilder.size_gc_header
-        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
+        if self.header(obj).tid & GCFLAG_PINNED:
+            hdr = self.header(obj)
+            if hdr.tid & GCFLAG_VISITED:
+                return
+            hdr.tid |= GCFLAG_VISITED
+            ll_assert(not self.header(obj).tid & GCFLAG_HAS_SHADOW, "support shadow with pinning")
+            ll_assert(not self.header(obj).tid & GCFLAG_HAS_CARDS, "support cards with pinning")
+            self.nursery_barriers.append(obj - size_gc_header)
+            return
+        elif self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
             #
             # Common case: 'obj' was not already forwarded (otherwise
             # tid == -42, containing all flags), and it doesn't have the
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -6,8 +6,7 @@
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.objectmodel import compute_unique_id
+from pypy.rlib.objectmodel import we_are_translated, compute_unique_id
 from pypy.rlib import rgc
 from pypy.rlib.rstring import StringBuilder
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -938,5 +937,34 @@
     GC_CAN_MALLOC_NONMOVABLE = True
     BUT_HOW_BIG_IS_A_BIG_STRING = 11*WORD
 
+    # those tests are here because they'll be messy and useless
+    # on GCs that can't pin objects
+    
+    def test_pinning_collect(self):
+        from pypy.rpython.lltypesystem import llmemory
+
+        TP = lltype.GcStruct('x', ('x', lltype.Signed), ('y', lltype.Signed))
+        
+        def f(i):
+            e = lltype.malloc(TP)
+            e.x = 3
+            prev = llmemory.cast_ptr_to_adr(e)
+            rgc.pin(e)
+            for k in range(i):
+                lltype.malloc(TP)
+            res = int(llmemory.cast_ptr_to_adr(e) == prev)
+            for k in range(i):
+                lltype.malloc(TP)
+            res += int(llmemory.cast_ptr_to_adr(e) == prev)
+            assert e.x == 3 # noone overwrote it
+            rgc.unpin(e)
+            for k in range(i):
+                lltype.malloc(TP)
+            assert e.x == 3 # noone overwrote it
+            return res
+
+        res = self.interpret(f, [10])
+        assert res == 2
+
 class TestMiniMarkGCCardMarking(TestMiniMarkGC):
     GC_PARAMS = {'card_page_indices': 4}

From noreply at buildbot.pypy.org  Thu Apr 12 23:32:10 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 23:32:10 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: pass more of a test and
 write insert for support. crap ugly
Message-ID: <20120412213210.EC82A82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54317:aa6c3180899c
Date: 2012-04-12 23:31 +0200
http://bitbucket.org/pypy/pypy/changeset/aa6c3180899c/

Log:	pass more of a test and write insert for support. crap ugly

diff --git a/pypy/rpython/lltypesystem/llarena.py b/pypy/rpython/lltypesystem/llarena.py
--- a/pypy/rpython/lltypesystem/llarena.py
+++ b/pypy/rpython/lltypesystem/llarena.py
@@ -394,7 +394,7 @@
     arena_addr = getfakearenaaddress(arena_addr)
     assert arena_addr.offset == 0
     assert size == arena_addr.arena.nbytes
-    arena_addr.arena.set_protect(inaccessible)
+    arena_addr.arena.set_protect(inaccessible)    
 
 # ____________________________________________________________
 #
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -247,6 +247,8 @@
         self.nursery_top  = NULL
         self.debug_tiny_nursery = -1
         self.debug_rotating_nurseries = None
+        self.surviving_pinned_objects = NULL
+        self.nursery_barriers = NULL
         #
         # The ArenaCollection() handles the nonmovable objects allocation.
         if ArenaCollectionClass is None:
@@ -306,7 +308,7 @@
         # minor collection.
         self.nursery_objects_shadows = self.AddressDict()
         # all pinned objects that were in the nursery *before* last
-        # minor collect. This is a sorted stack that should be consulted when
+        # minor collect. This is a sorted deque that should be consulted when
         # considering next nursery ceiling
         self.nursery_barriers = self.AddressDeque()
     
@@ -1257,6 +1259,8 @@
         #
         # Before everything else, remove from 'old_objects_pointing_to_young'
         # the young arrays.
+        self.nursery_barriers.delete()
+        self.surviving_pinned_objects = self.AddressStack()
         if self.young_rawmalloced_objects:
             self.remove_young_arrays_from_old_objects_pointing_to_young()
         #
@@ -1308,22 +1312,24 @@
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
         # self.nursery_barriers are *not* freed
-        # XXX sort the nursery_barriers
-        new_barriers = self.AddressDeque()
+        nursery_barriers = self.AddressDeque()
         prev = self.nursery
         size_gc_header = self.gcheaderbuilder.size_gc_header
-        while self.nursery_barriers.non_empty():
-            next = self.nursery_barriers.popleft()
-            llarena.arena_reset(prev, next - prev, 2)
+        while self.surviving_pinned_objects.non_empty():
+            next = self.surviving_pinned_objects.pop()
+            assert next >= prev
+            size = llarena.getfakearenaaddress(next) - prev
+            llarena.arena_reset(prev, size, 2)
             # clean the visited flag
             obj = next + size_gc_header
             self.header(obj).tid &= ~GCFLAG_VISITED
-            prev = prev + (next - prev) + (size_gc_header +
+            prev = prev + size + (size_gc_header +
                            self.get_size(obj))
-            new_barriers.append(next)
+            nursery_barriers.append(next)
         llarena.arena_reset(prev, self.nursery_top - prev, 2)
-        self.nursery_barriers.delete()
-        self.nursery_barriers = new_barriers
+        self.surviving_pinned_objects.delete()
+        self.surviving_pinned_objects = NULL
+        self.nursery_barriers = nursery_barriers
         self.debug_rotate_nursery()
         self.nursery_free = self.nursery
         self.nursery_barriers.append(self.nursery + self.nursery_size)
@@ -1478,7 +1484,8 @@
             hdr.tid |= GCFLAG_VISITED
             ll_assert(not self.header(obj).tid & GCFLAG_HAS_SHADOW, "support shadow with pinning")
             ll_assert(not self.header(obj).tid & GCFLAG_HAS_CARDS, "support cards with pinning")
-            self.nursery_barriers.append(obj - size_gc_header)
+            self.surviving_pinned_objects.insert(
+                llarena.getfakearenaaddress(obj - size_gc_header))
             return
         elif self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
             #
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -173,6 +173,47 @@
                 chunk.items[count] = got
                 got = next
 
+        def insert(self, addr):
+            """ Insert addr in the already sorted stack to make sure
+            the smallest one is on top
+            """
+            if self.used_in_last_chunk == 0:
+                self.append(addr)
+                return
+            got = self.pop()
+            read = self.used_in_last_chunk - 1
+            if read == -1 and got <= addr:
+                self.append(addr)
+                self.append(got)
+                return
+            read_chunk = self.chunk
+            self.append(got)
+            if got > addr:
+                self.append(addr)
+                return
+            write = self.used_in_last_chunk
+            if self.used_in_last_chunk == chunk_size:
+                self.enlarge()
+                write = 0
+            self.used_in_last_chunk += 1
+            write_chunk = self.chunk
+            while got < addr and not read_chunk is null_chunk:
+                write_chunk.items[write] = got
+                write -= 1
+                if write < 0:
+                    write_chunk = write_chunk.next
+                    write = chunk_size - 1
+                got = read_chunk.items[read]
+                read -= 1
+                if read < 0:
+                    read_chunk = read_chunk.next
+                    read = chunk_size - 1
+            if got < addr:
+                write_chunk.items[write] = got
+                write_chunk.items[0] = addr
+            else:
+                write_chunk.items[write] = addr
+
     cache[chunk_size] = AddressStack
     return AddressStack
 
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -966,5 +966,35 @@
         res = self.interpret(f, [10])
         assert res == 2
 
+    def test_pinning_collect_2(self):
+        from pypy.rpython.lltypesystem import llmemory
+
+        TP = lltype.GcStruct('x', ('x', lltype.Signed), ('y', lltype.Signed))
+        
+        def f(i):
+            e = lltype.malloc(TP)
+            e.x = 3
+            lltype.malloc(TP)
+            e2 = lltype.malloc(TP)
+            e2.x = 5
+            rgc.pin(e2)
+            rgc.pin(e)
+            prev = llmemory.cast_ptr_to_adr(e)
+            prev2 = llmemory.cast_ptr_to_adr(e2)
+            for k in range(i):
+                lltype.malloc(TP)
+            res = int(llmemory.cast_ptr_to_adr(e) == prev)
+            res += int(llmemory.cast_ptr_to_adr(e2) == prev2)
+            rgc.unpin(e)
+            for k in range(i):
+                lltype.malloc(TP)
+            rgc.unpin(e2)
+            assert e.x == 3 # noone overwrote it
+            assert e2.x == 5 # noone overwrote it
+            return res
+
+        res = self.interpret(f, [10])
+        assert res == 2
+
 class TestMiniMarkGCCardMarking(TestMiniMarkGC):
     GC_PARAMS = {'card_page_indices': 4}
diff --git a/pypy/rpython/memory/test/test_support.py b/pypy/rpython/memory/test/test_support.py
--- a/pypy/rpython/memory/test/test_support.py
+++ b/pypy/rpython/memory/test/test_support.py
@@ -3,7 +3,7 @@
 from pypy.rpython.memory.support import get_address_deque
 
 from pypy.rpython.test.test_llinterp import interpret
-from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena
 from pypy.rpython.lltypesystem.llmemory import raw_malloc, raw_free, NULL
 
 class TestAddressStack(object):
@@ -94,6 +94,26 @@
             assert a == addrs[i]
         assert not ll.non_empty()
 
+    def test_insert(self):
+        AddressStack = get_address_stack(chunk_size=5)
+        ll = AddressStack()
+        lla = llarena.arena_malloc(10, 2)
+        addrs = [lla + i for i in range(10)]
+        ll.insert(addrs[2])
+        ll.insert(addrs[1])
+        ll.insert(addrs[5])
+        ll.insert(addrs[4])
+        ll.insert(addrs[6])
+        ll.insert(addrs[9])
+        ll.insert(addrs[0])
+        ll.insert(addrs[8])
+        ll.insert(addrs[7])
+        ll.insert(addrs[3])
+        expected = range(10)
+        for i in expected:
+            a = ll.pop()
+            assert a == addrs[i]
+
 
 class TestAddressDeque:
     def test_big_access(self):

From noreply at buildbot.pypy.org  Thu Apr 12 23:37:29 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 23:37:29 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: surprisingly enough make
	the test pass
Message-ID: <20120412213729.3CEFD82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54318:825c0462f962
Date: 2012-04-12 23:37 +0200
http://bitbucket.org/pypy/pypy/changeset/825c0462f962/

Log:	surprisingly enough make the test pass

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -589,7 +589,17 @@
         now-empty nursery.
         """
         if not self.nursery_top == self.nursery + self.nursery_size:
-            xxx
+            self.nursery_top = self.nursery_barriers.popleft()
+            size_gc_header = self.gcheaderbuilder.size_gc_header
+            while self.nursery_barriers.non_empty() and self.nursery_free + totalsize > self.nursery_top:
+                cur_obj_size = size_gc_header + self.get_size(
+                    self.nursery_free + size_gc_header)
+                self.nursery_free = self.nursery_free + cur_obj_size
+                self.nursery_top = self.nursery_barriers.popleft()
+            if self.nursery_free + totalsize <= self.nursery_top:
+                llarena.arena_reserve(self.nursery_free, totalsize)
+                res = self.nursery_free
+                self.nursery_free = res + totalsize
         self.minor_collection(totalsize)
         # try allocating now, otherwise we do a major collect
         do_major_collect = False

From noreply at buildbot.pypy.org  Thu Apr 12 23:54:33 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 12 Apr 2012 23:54:33 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: ok,
 we cannot change numbers like this. It makes benchmark results hike
 wildly, 
Message-ID: <20120412215433.C6B4882F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r182:0a63cb1bbb5f
Date: 2012-04-12 23:54 +0200
http://bitbucket.org/pypy/benchmarks/changeset/0a63cb1bbb5f/

Log:	ok, we cannot change numbers like this. It makes benchmark results
	hike wildly, because of both the GC warmup and JIT warmup

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -49,7 +49,8 @@
 for name in ['expand', 'integrate', 'sum', 'str']:
     _register_new_bm('bm_sympy', 'sympy_' + name,
                      globals(), bm_env={'PYTHONPATH': relative('lib/sympy')},
-                     extra_args=['--benchmark=' + name])
+                     extra_args=['--benchmark=' + name],
+                     iteration_scaling=.1)
 
 for name in ['xml', 'text']:
     _register_new_bm('bm_genshi', 'genshi_' + name,

From noreply at buildbot.pypy.org  Fri Apr 13 05:43:23 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 13 Apr 2012 05:43:23 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: merge default into branch
Message-ID: <20120413034323.1187B82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54319:69f0499a8cb2
Date: 2012-04-11 17:41 -0700
http://bitbucket.org/pypy/pypy/changeset/69f0499a8cb2/

Log:	merge default into branch

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return pyc
+def pytest_unconfigure(config):
+    hook = config._assertstate.hook
+    if hook is not None:
+        sys.meta_path.remove(hook)
 
-def before_module_import(mod):
-    if mod.config._assertstate.mode != "on":
-        return
-    # Some deep magic: load the source, rewrite the asserts, and write a
-    # fake pyc, so that it'll be loaded when the module is imported.
-    source = mod.fspath.read()
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
-        return
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, str(mod.fspath), "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the assertion
-        # rewriting, but I don't know of a fast way to tell.
-        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
-        return
-    mod._pyc = _write_pyc(co, mod.fspath)
-    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+def pytest_collection(session):
+    # this hook is only called when test modules are collected
+    # so for example not in the master process of pytest-xdist
+    # (which does not collect test modules)
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.set_session(session)
 
-def after_module_import(mod):
-    if not hasattr(mod, "_pyc"):
-        return
-    state = mod.config._assertstate
-    try:
-        mod._pyc.remove()
-    except py.error.ENOENT:
-        state.trace("couldn't find pyc: %r" % (mod._pyc,))
-    else:
-        state.trace("removed pyc: %r" % (mod._pyc,))
+def pytest_runtest_setup(item):
+    def callbinrepr(op, left, right):
+        hook_result = item.ihook.pytest_assertrepr_compare(
+            config=item.config, op=op, left=left, right=right)
+        for new_expl in hook_result:
+            if new_expl:
+                res = '\n~'.join(new_expl)
+                if item.config.getvalue("assertmode") == "rewrite":
+                    # The result will be fed back a python % formatting
+                    # operation, which will fail if there are extraneous
+                    # '%'s in the string. Escape them here.
+                    res = res.replace("%", "%%")
+                return res
+    util._reprcompare = callbinrepr
 
-def warn_about_missing_assertion():
+def pytest_runtest_teardown(item):
+    util._reprcompare = None
+
+def pytest_sessionfinish(session):
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.session = None
+
+def _load_modules(mode):
+    """Lazily import assertion related code."""
+    global rewrite, reinterpret
+    from _pytest.assertion import reinterpret
+    if mode == "rewrite":
+        from _pytest.assertion import rewrite
+
+def warn_about_missing_assertion(mode):
     try:
         assert False
     except AssertionError:
         pass
     else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
+        if mode == "rewrite":
+            specifically = ("assertions which are not in test modules "
+                            "will be ignored")
+        else:
+            specifically = "failing tests may report as passing"
+
+        sys.stderr.write("WARNING: " + specifically +
+                        " because assert statements are not executed "
+                        "by the underlying Python interpreter "
+                        "(are you using python -O?)\n")
 
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -53,7 +53,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
+                "compute assert expression before the assert or use --assert=plain)")
 
 def run(offending_line, frame=None):
     if frame is None:
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -482,7 +482,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --nomagic)")
+                "compute assert expression before the assert or use --assert=plain)")
     else:
         return None
 
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,14 +1,258 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
+import errno
 import itertools
+import imp
+import marshal
+import os
+import struct
 import sys
+import types
 
 import py
 from _pytest.assertion import util
 
 
+# Windows gives ENOENT in places *nix gives ENOTDIR.
+if sys.platform.startswith("win"):
+    PATH_COMPONENT_NOT_DIR = errno.ENOENT
+else:
+    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
+
+# py.test caches rewritten pycs in __pycache__.
+if hasattr(imp, "get_tag"):
+    PYTEST_TAG = imp.get_tag() + "-PYTEST"
+else:
+    if hasattr(sys, "pypy_version_info"):
+        impl = "pypy"
+    elif sys.platform == "java":
+        impl = "jython"
+    else:
+        impl = "cpython"
+    ver = sys.version_info
+    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
+    del ver, impl
+
+PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
+
+REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+
+class AssertionRewritingHook(object):
+    """Import hook which rewrites asserts."""
+
+    def __init__(self):
+        self.session = None
+        self.modules = {}
+
+    def set_session(self, session):
+        self.fnpats = session.config.getini("python_files")
+        self.session = session
+
+    def find_module(self, name, path=None):
+        if self.session is None:
+            return None
+        sess = self.session
+        state = sess.config._assertstate
+        state.trace("find_module called for: %s" % name)
+        names = name.rsplit(".", 1)
+        lastname = names[-1]
+        pth = None
+        if path is not None and len(path) == 1:
+            pth = path[0]
+        if pth is None:
+            try:
+                fd, fn, desc = imp.find_module(lastname, path)
+            except ImportError:
+                return None
+            if fd is not None:
+                fd.close()
+            tp = desc[2]
+            if tp == imp.PY_COMPILED:
+                if hasattr(imp, "source_from_cache"):
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
+            elif tp != imp.PY_SOURCE:
+                # Don't know what this is.
+                return None
+        else:
+            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
+        fn_pypath = py.path.local(fn)
+        # Is this a test file?
+        if not sess.isinitpath(fn):
+            # We have to be very careful here because imports in this code can
+            # trigger a cycle.
+            self.session = None
+            try:
+                for pat in self.fnpats:
+                    if fn_pypath.fnmatch(pat):
+                        state.trace("matched test file %r" % (fn,))
+                        break
+                else:
+                    return None
+            finally:
+                self.session = sess
+        else:
+            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+        # The requested module looks like a test file, so rewrite it. This is
+        # the most magical part of the process: load the source, rewrite the
+        # asserts, and load the rewritten source. We also cache the rewritten
+        # module code in a special pyc. We must be aware of the possibility of
+        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # tricky race conditions, we maintain the following invariant: The
+        # cached pyc is always a complete, valid pyc. Operations on it must be
+        # atomic. POSIX's atomic rename comes in handy.
+        write = not sys.dont_write_bytecode
+        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
+        if write:
+            try:
+                os.mkdir(cache_dir)
+            except OSError:
+                e = sys.exc_info()[1].errno
+                if e == errno.EEXIST:
+                    # Either the __pycache__ directory already exists (the
+                    # common case) or it's blocked by a non-dir node. In the
+                    # latter case, we'll ignore it in _write_pyc.
+                    pass
+                elif e == PATH_COMPONENT_NOT_DIR:
+                    # One of the path components was not a directory, likely
+                    # because we're in a zip file.
+                    write = False
+                elif e == errno.EACCES:
+                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    write = False
+                else:
+                    raise
+        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
+        pyc = os.path.join(cache_dir, cache_name)
+        # Notice that even if we're in a read-only directory, I'm going to check
+        # for a cached pyc. This may not be optimal...
+        co = _read_pyc(fn_pypath, pyc)
+        if co is None:
+            state.trace("rewriting %r" % (fn,))
+            co = _rewrite_test(state, fn_pypath)
+            if co is None:
+                # Probably a SyntaxError in the test.
+                return None
+            if write:
+                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+        else:
+            state.trace("found cached rewritten pyc for %r" % (fn,))
+        self.modules[name] = co, pyc
+        return self
+
+    def load_module(self, name):
+        co, pyc = self.modules.pop(name)
+        # I wish I could just call imp.load_compiled here, but __file__ has to
+        # be set properly. In Python 3.2+, this all would be handled correctly
+        # by load_compiled.
+        mod = sys.modules[name] = imp.new_module(name)
+        try:
+            mod.__file__ = co.co_filename
+            # Normally, this attribute is 3.2+.
+            mod.__cached__ = pyc
+            py.builtin.exec_(co, mod.__dict__)
+        except:
+            del sys.modules[name]
+            raise
+        return sys.modules[name]
+
+def _write_pyc(co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as (C)Python, since
+    # these "pycs" should never be seen by builtin import. However, there's
+    # little reason deviate, and I hope sometime to be able to use
+    # imp.load_compiled to load them. (See the comment in load_module above.)
+    mtime = int(source_path.mtime())
+    try:
+        fp = open(pyc, "wb")
+    except IOError:
+        err = sys.exc_info()[1].errno
+        if err == PATH_COMPONENT_NOT_DIR:
+            # This happens when we get a EEXIST in find_module creating the
+            # __pycache__ directory and __pycache__ is by some non-dir node.
+            return False
+        raise
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return True
+
+RN = "\r\n".encode("utf-8")
+N = "\n".encode("utf-8")
+
+def _rewrite_test(state, fn):
+    """Try to read and rewrite *fn* and return the code object."""
+    try:
+        source = fn.read("rb")
+    except EnvironmentError:
+        return None
+    # On Python versions which are not 2.7 and less than or equal to 3.1, the
+    # parser expects *nix newlines.
+    if REWRITE_NEWLINES:
+        source = source.replace(RN, N) + N
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        state.trace("failed to parse: %r" % (fn,))
+        return None
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, fn.strpath, "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the
+        # assertion rewriting, but I don't know of a fast way to tell.
+        state.trace("failed to compile: %r" % (fn,))
+        return None
+    return co
+
+def _make_rewritten_pyc(state, fn, pyc, co):
+    """Try to dump rewritten code to *pyc*."""
+    if sys.platform.startswith("win"):
+        # Windows grants exclusive access to open files and doesn't have atomic
+        # rename, so just write into the final file.
+        _write_pyc(co, fn, pyc)
+    else:
+        # When not on windows, assume rename is atomic. Dump the code object
+        # into a file specific to this process and atomically replace it.
+        proc_pyc = pyc + "." + str(os.getpid())
+        if _write_pyc(co, fn, proc_pyc):
+            os.rename(proc_pyc, pyc)
+
+def _read_pyc(source, pyc):
+    """Possibly read a py.test pyc containing rewritten code.
+
+    Return rewritten code if successful or None if not.
+    """
+    try:
+        fp = open(pyc, "rb")
+    except IOError:
+        return None
+    try:
+        try:
+            mtime = int(source.mtime())
+            data = fp.read(8)
+        except EnvironmentError:
+            return None
+        # Check for invalid or out of date pyc file.
+        if (len(data) != 8 or
+            data[:4] != imp.get_magic() or
+            struct.unpack("<l", data[4:])[0] != mtime):
+            return None
+        co = marshal.load(fp)
+        if not isinstance(co, types.CodeType):
+            # That's interesting....
+            return None
+        return co
+    finally:
+        fp.close()
+
+
 def rewrite_asserts(mod):
     """Rewrite the assert statements in mod."""
     AssertionRewriter().run(mod)
@@ -17,13 +261,8 @@
 _saferepr = py.io.saferepr
 from _pytest.assertion.util import format_explanation as _format_explanation
 
-def _format_boolop(operands, explanations, is_or):
-    show_explanations = []
-    for operand, expl in zip(operands, explanations):
-        show_explanations.append(expl)
-        if operand == is_or:
-            break
-    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+def _format_boolop(explanations, is_or):
+    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -109,8 +348,8 @@
                     return
                 lineno += len(doc) - 1
                 expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +357,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +382,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +437,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +460,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +480,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +545,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,21 +25,39 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +86,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("</testcase>")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="xfail-marked test passes unexpectedly"/>')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('<failure message="test failure">%s</failure>',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<failure message="collection failure">%s</failure>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<skipped message="collection skipped">%s</skipped>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('<error message="test setup failure">%s</error>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="expected test failure">%s</skipped>',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('<skipped type="pytest.skip" '
-                           'message="%s">%s</skipped>',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +185,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n<testcase classname="pytest" name="internal">'
-            '    <error message="internal error">'
-            '%s</error></testcase>' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +203,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write('<testsuite ')
-        logfile.write('name="" ')
-        logfile.write('errors="%i" ' % self.errors)
-        logfile.write('failures="%i" ' % self.failed)
-        logfile.write('skips="%i" ' % self.skipped)
-        logfile.write('tests="%i" ' % numtests)
-        logfile.write('time="%.3f"' % suite_time_delta)
-        logfile.write(' >')
-        logfile.writelines(self.test_logs)
-        logfile.write('</testsuite>')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -399,6 +410,7 @@
         self._notfound = []
         self._initialpaths = set()
         self._initialparts = []
+        self.items = items = []
         for arg in args:
             parts = self._parsearg(arg)
             self._initialparts.append(parts)
@@ -414,7 +426,6 @@
         if not genitems:
             return rep.result
         else:
-            self.items = items = []
             if rep.passed:
                 for node in rep.result:
                     self.items.extend(self.genitems(node))
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "<CallSpec id=%r param=%r funcargs=%r>" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,71 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        if not indirect:
+            #XXX should we also check for the opposite case?
+            for arg in argnames:
+                if arg not in self.funcargnames:
+                    raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +678,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +769,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +781,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +830,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +956,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         <ExceptionInfo ...>
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         <ExceptionInfo ...>
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,60 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
-
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
+def pytest_sessionfinish(session):
+    session._setupstate.teardown_all()
 
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +76,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +93,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +114,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +143,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +166,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +188,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +208,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "<TestReport %r when=%r outcome=%r>" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +234,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +259,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +319,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -117,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
@@ -169,21 +192,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -289,10 +282,18 @@
         # we take care to leave out Instances aka ()
         # because later versions are going to get rid of them anyway
         if self.config.option.verbose < 0:
-            for item in items:
-                nodeid = item.nodeid
-                nodeid = nodeid.replace("::()::", "::")
-                self._tw.line(nodeid)
+            if self.config.option.verbose < -1:
+                counts = {}
+                for item in items:
+                    name = item.nodeid.split('::', 1)[0]
+                    counts[name] = counts.get(name, 0) + 1
+                for name, count in sorted(counts.items()):
+                    self._tw.line("%s: %d" % (name, count))
+            else:
+                for item in items:
+                    nodeid = item.nodeid
+                    nodeid = nodeid.replace("::()::", "::")
+                    self._tw.line(nodeid)
             return
         stack = []
         indent = ""
@@ -318,12 +319,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +394,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +412,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +431,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +445,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -83,28 +91,37 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +137,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.4.dev1'
+__version__ = '1.4.7'
 
 from py import _apipkg
 
@@ -70,6 +70,11 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
+        '_AssertionError'   : '._code.assertion:AssertionError',
+        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
+        '_reinterpret'      : '._code.assertion:reinterpret',
+        '_reprcompare'      : '._code.assertion:_reprcompare',
+        '_format_explanation' : '._code.assertion:_format_explanation',
     },
 
     # backports and additions of builtins
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -113,9 +113,12 @@
 
     # some backward compatibility helpers
     _basestring = str
-    def _totext(obj, encoding=None):
+    def _totext(obj, encoding=None, errors=None):
         if isinstance(obj, bytes):
-            obj = obj.decode(encoding)
+            if errors is None:
+                obj = obj.decode(encoding)
+            else:
+                obj = obj.decode(encoding, errors)
         elif not isinstance(obj, str):
             obj = str(obj)
         return obj
@@ -142,7 +145,7 @@
             del back
         elif locs is None:
             locs = globs
-        fp = open(fn, "rb")
+        fp = open(fn, "r")
         try:
             source = fp.read()
         finally:
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionnew.py
@@ -0,0 +1,339 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace _assertionold.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from py._code.assertion import _format_explanation, BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(failure):
+    explanation = _format_explanation(failure.explanation)
+    value = failure.cause[1]
+    if str(value):
+        lines = explanation.splitlines()
+        if not lines:
+            lines.append("")
+        lines[0] += " << %s" % (value,)
+        explanation = "\n".join(lines)
+    text = "%s: %s" % (failure.cause[0].__name__, explanation)
+    if text.startswith("AssertionError: assert "):
+        text = text[16:]
+    return text
+
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = False
+        if not local:
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not result:
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        rcomp = py.code._reprcompare
+        if rcomp:
+            res = rcomp(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = True
+        if from_instance:
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        if test_explanation.startswith("False\n{False =") and \
+                test_explanation.endswith("\n"):
+            test_explanation = test_explanation[15:-2]
+        explanation = "assert %s" % (test_explanation,)
+        if not test_result:
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionold.py
@@ -0,0 +1,555 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from py._code.assertion import BuiltinAssertionError, _format_explanation
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = '<eval>'
+            self.__obj__.filename = '<eval>'
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = '<run>'
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return _format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # simplify 'assert False where False = ...'
+        if (test.explanation.startswith('False\n{False = ') and
+            test.explanation.endswith('\n}')):
+            test.explanation = test.explanation[15:-2]
+        # print the result as  'assert <explanation>'
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = '<run>'
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
new file mode 100644
--- /dev/null
+++ b/py/_code/assertion.py
@@ -0,0 +1,94 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+_reprcompare = None # if set, will be called by assert reinterp for comparison ops
+
+def _format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "<run>".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = "<could not determine information>"
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from py._code._assertionold import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from py._code._assertionnew import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,6 +145,17 @@
         return self.frame.f_locals
     locals = property(getlocals, None, None, "locals of underlaying frame")
 
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        if self.exprinfo is None:
+            source = str(self.statement).strip()
+            x = py.code._reinterpret(source, self.frame, should_fail=True)
+            if not isinstance(x, str):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
     def getfirstlinesource(self):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
@@ -158,13 +169,12 @@
         end = self.lineno
         try:
             _, end = source.getstatementrange(end)
-        except IndexError:
+        except (IndexError, ValueError):
             end = self.lineno + 1
         # heuristic to stop displaying source on e.g.
         #   if something:  # assume this causes a NameError
         #      # _this_ lines and the one
                #        below we don't want from entry.getsource()
-        end = min(end, len(source))
         for i in range(self.lineno, end):
             if source[i].rstrip().endswith(':'):
                 end = i + 1
@@ -273,7 +283,11 @@
         """
         cache = {}
         for i, entry in enumerate(self):
-            key = entry.frame.code.path, entry.lineno
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
             #print "checking for recursion at", key
             l = cache.setdefault(key, [])
             if l:
@@ -308,7 +322,7 @@
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
         self.type, self.value, tb = self._excinfo
-        self.typename = getattr(self.type, "__name__", "???")
+        self.typename = self.type.__name__
         self.traceback = py.code.Traceback(tb)
 
     def __repr__(self):
@@ -347,14 +361,16 @@
             showlocals: show locals per traceback entry
             style: long|short|no|native traceback style
             tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
         """
         if style == 'native':
-            import traceback
-            return ''.join(traceback.format_exception(
-                self.type,
-                self.value,
-                self.traceback[0]._rawentry,
-                ))
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
 
         fmt = FormattedExcinfo(showlocals=showlocals, style=style,
             abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
@@ -452,7 +468,7 @@
     def repr_locals(self, locals):
         if self.showlocals:
             lines = []
-            keys = list(locals)
+            keys = [loc for loc in locals if loc[0] != "@"]
             keys.sort()
             for name in keys:
                 value = locals[name]
@@ -506,7 +522,10 @@
 
     def _makepath(self, path):
         if not self.abspath:
-            np = py.path.local().bestrelpath(path)
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
             if len(np) < len(str(path)):
                 path = np
         return path
@@ -595,6 +614,19 @@
         if self.extraline:
             tw.line(self.extraline)
 
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
 class ReprEntry(TerminalRepr):
     localssep = "_ "
 
@@ -680,19 +712,26 @@
 
 oldbuiltins = {}
 
-def patch_builtins(compile=True):
-    """ put compile builtins to Python's builtins. """
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from py._code import assertion
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = assertion.AssertionError
     if compile:
         l = oldbuiltins.setdefault('compile', [])
         l.append(py.builtin.builtins.compile)
         py.builtin.builtins.compile = py.code.compile
 
-def unpatch_builtins(compile=True):
+def unpatch_builtins(assertion=True, compile=True):
     """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
     if compile:
         py.builtin.builtins.compile = oldbuiltins['compile'].pop()
 
-def getrawcode(obj):
+def getrawcode(obj, trycall=True):
     """ return code object for given function. """
     try:
         return obj.__code__
@@ -701,5 +740,10 @@
         obj = getattr(obj, 'func_code', obj)
         obj = getattr(obj, 'f_code', obj)
         obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
         return obj
 
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,6 +108,7 @@
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
+            raise an IndexError if no such statementrange can be found.
         """
         # XXX there must be a better than these heuristic ways ...
         # XXX there may even be better heuristics :-)
@@ -116,6 +117,7 @@
 
         # 1. find the start of the statement
         from codeop import compile_command
+        end = None
         for start in range(lineno, -1, -1):
             if assertion:
                 line = self.lines[start]
@@ -139,7 +141,9 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, len(self)
+        if end is None:
+            raise IndexError("no valid source range around line %d " % (lineno,))
+        return start, end
 
     def getblockend(self, lineno):
         # XXX
@@ -257,23 +261,29 @@
 
 
 def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
     try:
         code = py.code.Code(obj)
     except TypeError:
-        # fallback to
-        fn = (py.std.inspect.getsourcefile(obj) or
-              py.std.inspect.getfile(obj))
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
         fspath = fn and py.path.local(fn) or None
+        lineno = -1
         if fspath:
             try:
                 _, lineno = findsource(obj)
             except IOError:
-                lineno = None
-        else:
-            lineno = None
+                pass
     else:
         fspath = code.path
         lineno = code.firstlineno
+    assert isinstance(lineno, int)
     return fspath, lineno
 
 #
@@ -286,7 +296,7 @@
     except py.builtin._sysex:
         raise
     except:
-        return None, None
+        return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
diff --git a/py/_error.py b/py/_error.py
--- a/py/_error.py
+++ b/py/_error.py
@@ -23,6 +23,7 @@
     2: errno.ENOENT,
     3: errno.ENOENT,
     17: errno.EEXIST,
+    13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -103,6 +103,7 @@
     def _parseline(self, line, lineno):
         # comments
         line = line.split('#')[0].rstrip()
+        line = line.split(';')[0].rstrip()
         # blank lines
         if not line:
             return None, None
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -12,7 +12,7 @@
     class TextIO(StringIO):
         def write(self, data):
             if not isinstance(data, unicode):
-                data = unicode(data, getattr(self, '_encoding', 'UTF-8'))
+                data = unicode(data, getattr(self, '_encoding', 'UTF-8'), 'replace')
             StringIO.write(self, data)
 else:
     TextIO = StringIO
@@ -258,6 +258,9 @@
                 f = getattr(self, name).tmpfile
                 f.seek(0)
                 res = f.read()
+                enc = getattr(f, 'encoding', None)
+                if enc:
+                    res = py.builtin._totext(res, enc, 'replace')
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -105,6 +105,8 @@
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
+    _newline = None   # the last line printed
+
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -112,11 +114,9 @@
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-                if hasattr(file, 'encoding'):
-                    encoding = file.encoding
         elif hasattr(file, '__call__'):
             file = WriteFile(file, encoding=encoding)
-        self.encoding = encoding
+        self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
@@ -182,8 +182,31 @@
         return s
 
     def line(self, s='', **kw):
+        if self._newline == False:
+            self.write("\n")
         self.write(s, **kw)
         self.write('\n')
+        self._newline = True
+
+    def reline(self, line, **opts):
+        if not self.hasmarkup:
+            raise ValueError("cannot use rewrite-line without terminal")
+        if not self._newline:
+            self.write("\r")
+        self.write(line, **opts)
+        # see if we need to fill up some spaces at the end
+        # xxx have a more exact lastlinelen working from self.write?
+        lenline = len(line)
+        try:
+            lastlen = self._lastlinelen
+        except AttributeError:
+            pass
+        else:
+            if lenline < lastlen:
+                self.write(" " * (lastlen - lenline + 1))
+        self._lastlinelen = lenline
+        self._newline = False
+
 
 class Win32ConsoleWriter(TerminalWriter):
     def write(self, s, **kw):
@@ -280,10 +303,10 @@
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
     SetConsoleTextAttribute.argtypes = [wintypes.HANDLE, wintypes.WORD]
     SetConsoleTextAttribute.restype = wintypes.BOOL
-        
+
     _GetConsoleScreenBufferInfo = \
         ctypes.windll.kernel32.GetConsoleScreenBufferInfo
-    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, 
+    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE,
                                 ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
     _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
     def GetConsoleInfo(handle):
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -64,7 +64,10 @@
                 else:
                     if bool(value) ^ bool(meth()) ^ invert:
                         return False
-            except (py.error.ENOENT, py.error.ENOTDIR):
+            except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
+                # EBUSY feels not entirely correct,
+                # but its kind of necessary since ENOMEDIUM
+                # is not accessible in python     
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
@@ -368,6 +371,5 @@
         else:
             name = str(path) # path.strpath # XXX svn?
             pattern = '*' + path.sep + pattern
-        from fnmatch import fnmatch
-        return fnmatch(name, pattern)
+        return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -157,14 +157,16 @@
         return str(self) < str(other)
 
     def samefile(self, other):
-        """ return True if 'other' references the same file as 'self'. """
-        if not iswin32:
-            return py.error.checked_call(
-                    os.path.samefile, str(self), str(other))
+        """ return True if 'other' references the same file as 'self'.
+        """
+        if not isinstance(other, py.path.local):
+            other = os.path.abspath(str(other))
         if self == other:
             return True
-        other = os.path.abspath(str(other))
-        return self == other
+        if iswin32:
+            return False # ther is no samefile
+        return py.error.checked_call(
+                os.path.samefile, str(self), str(other))
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -539,7 +541,11 @@
                 if self.basename != "__init__.py":
                     modfile = modfile[:-12]
 
-            if not self.samefile(modfile):
+            try:
+                issame = self.samefile(modfile)
+            except py.error.ENOENT:
+                issame = False
+            if not issame:
                 raise self.ImportMismatchError(modname, modfile, self)
             return mod
         else:
diff --git a/py/_path/svnurl.py b/py/_path/svnurl.py
--- a/py/_path/svnurl.py
+++ b/py/_path/svnurl.py
@@ -233,6 +233,8 @@
                 e = sys.exc_info()[1]
                 if e.err.find('non-existent in that revision') != -1:
                     raise py.error.ENOENT(self, e.err)
+                elif e.err.find("E200009:") != -1:
+                    raise py.error.ENOENT(self, e.err)
                 elif e.err.find('File not found') != -1:
                     raise py.error.ENOENT(self, e.err)
                 elif e.err.find('not part of a repository')!=-1:
diff --git a/py/_path/svnwc.py b/py/_path/svnwc.py
--- a/py/_path/svnwc.py
+++ b/py/_path/svnwc.py
@@ -482,10 +482,13 @@
         except py.process.cmdexec.Error:
             e = sys.exc_info()[1]
             strerr = e.err.lower()
-            if strerr.find('file not found') != -1:
+            if strerr.find('not found') != -1:
+                raise py.error.ENOENT(self)
+            elif strerr.find("E200009:") != -1:
                 raise py.error.ENOENT(self)
             if (strerr.find('file exists') != -1 or
                 strerr.find('file already exists') != -1 or
+                strerr.find('w150002:') != -1 or
                 strerr.find("can't create directory") != -1):
                 raise py.error.EEXIST(self)
             raise
@@ -593,7 +596,7 @@
         out = self._authsvn('lock').strip()
         if not out:
             # warning or error, raise exception
-            raise Exception(out[4:])
+            raise ValueError("unknown error in svn lock command")
 
     def unlock(self):
         """ unset a previously set lock """
@@ -1066,6 +1069,8 @@
                 modrev = '?'
                 author = '?'
                 date = ''
+            elif itemstatus == "replaced":
+                pass
             else:
                 #print entryel.toxml()
                 commitel = entryel.getElementsByTagName('commit')[0]
@@ -1148,7 +1153,11 @@
             raise  ValueError("Not a versioned resource")
             #raise ValueError, "Not a versioned resource %r" % path
         self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind']
-        self.rev = int(d['revision'])
+        try:
+            self.rev = int(d['revision'])
+        except KeyError:
+            self.rev = None
+
         self.path = py.path.local(d['path'])
         self.size = self.path.size()
         if 'lastchangedrev' in d:
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -52,7 +52,7 @@
     def unicode(self, indent=2):
         l = []
         SimpleUnicodeVisitor(l.append, indent).visit(self)
-        return "".join(l)
+        return u("").join(l)
 
     def __repr__(self):
         name = self.__class__.__name__
@@ -122,11 +122,13 @@
                 if visitmethod is not None:
                     break
             else:
-                visitmethod = self.object
+                visitmethod = self.__object
             self.cache[cls] = visitmethod
         visitmethod(node)
 
-    def object(self, obj):
+    # the default fallback handler is marked private
+    # to avoid clashes with the tag name object
+    def __object(self, obj):
         #self.write(obj)
         self.write(escape(unicode(obj)))
 
@@ -136,7 +138,8 @@
     def list(self, obj):
         assert id(obj) not in self.visited
         self.visited[id(obj)] = 1
-        map(self.visit, obj)
+        for elem in obj:
+            self.visit(elem)
 
     def Tag(self, tag):
         assert id(tag) not in self.visited
@@ -181,7 +184,11 @@
             value = getattr(attrs, name)
             if name.endswith('_'):
                 name = name[:-1]
-            return ' %s="%s"' % (name, escape(unicode(value)))
+            if isinstance(value, raw):
+                insert = value.uniobj
+            else:
+                insert = escape(unicode(value))
+            return ' %s="%s"' % (name, insert)
 
     def getstyle(self, tag):
         """ return attribute list suitable for styling. """
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -483,7 +483,7 @@
                 return a_str.strip(' ')
             elif n == 1:
                 return a_str.rstrip(' ')
-            else: 
+            else:
                 return a_str.lstrip(' ')
         s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
         assert s.no_nul
@@ -3737,6 +3737,25 @@
         s = a.build_types(f, [int])
         assert s.listdef.listitem.range_step == 0
 
+    def test_specialize_arg_memo(self):
+        @objectmodel.specialize.memo()
+        def g(n):
+            return n
+        @objectmodel.specialize.arg(0)
+        def f(i):
+            return g(i)
+        def main(i):
+            if i == 2:
+                return f(i)
+            elif i == 3:
+                return f(i)
+            else:
+                raise NotImplementedError
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(main, [int])
+        assert isinstance(s, annmodel.SomeInteger)
+
 
 def g(n):
     return [0,1,2,n]
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
new file mode 100644
--- /dev/null
+++ b/pypy/doc/tool/makecontributor.py
@@ -0,0 +1,133 @@
+import py
+import sys
+from collections import defaultdict
+import operator
+import re
+import mercurial.localrepo
+import mercurial.ui
+
+ROOT = py.path.local(__file__).join('..', '..', '..', '..')
+author_re = re.compile('(.*) <.*>')
+pair_programming_re = re.compile(r'^\((.*?)\)')
+excluded = set(["pypy", "convert-repo"])
+
+alias = {
+    'Anders Chrigstrom': ['arre'],
+    'Antonio Cuni': ['antocuni', 'anto'],
+    'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
+    'Maciej Fijalkowski': ['fijal'],
+    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Michael Hudson': ['mwh'],
+    'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
+    "Amaury Forgeot d'Arc": ['afa'],
+    'Alex Gaynor': ['alex', 'agaynor'],
+    'David Schneider': ['bivab', 'david'],
+    'Christian Tismer': ['chris', 'christian', 'tismer',
+                         'tismer at christia-wjtqxl.localdomain'],
+    'Benjamin Peterson': ['benjamin'],
+    'Hakan Ardo': ['hakan', 'hakanardo'],
+    'Niklaus Haldimann': ['nik'],
+    'Alexander Schremmer': ['xoraxax'],
+    'Anders Hammarquist': ['iko'],
+    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'Niko Matsakis': ['niko'],
+    'Jakub Gustak': ['jlg'],
+    'Guido Wesdorp': ['guido'],
+    'Michael Foord': ['mfoord'],
+    'Mark Pearse': ['mwp'],
+    'Toon Verwaest': ['tverwaes'],
+    'Eric van Riet Paap': ['ericvrp'],
+    'Jacob Hallen': ['jacob', 'jakob'],
+    'Anders Lehmann': ['ale', 'anders'],
+    'Bert Freudenberg': ['bert'],
+    'Boris Feigin': ['boris', 'boria'],
+    'Valentino Volonghi': ['valentino', 'dialtone'],
+    'Aurelien Campeas': ['aurelien', 'aureliene'],
+    'Adrien Di Mascio': ['adim'],
+    'Jacek Generowicz': ['Jacek', 'jacek'],
+    'Jim Hunziker': ['landtuna at gmail.com'],
+    'Kristjan Valur Jonsson': ['kristjan at kristjan-lp.ccp.ad.local'],
+    'Laura Creighton': ['lac'],
+    'Aaron Iles': ['aliles'],
+    'Ludovic Aubry': ['ludal', 'ludovic'],
+    'Lukas Diekmann': ['l.diekmann', 'ldiekmann'],
+    'Matti Picus': ['Matti Picus matti.picus at gmail.com',
+                    'matthp', 'mattip', 'mattip>'],
+    'Michael Cheng': ['mikefc'],
+    'Richard Emslie': ['rxe'],
+    'Roberto De Ioris': ['roberto at goyle'],
+    'Roberto De Ioris': ['roberto at mrspurr'],
+    'Sven Hager': ['hager'],
+    'Tomo Cocoa': ['cocoatomo'],
+    }
+
+alias_map = {}
+for name, nicks in alias.iteritems():
+    for nick in nicks:
+        alias_map[nick] = name
+
+def get_canonical_author(name):
+    match = author_re.match(name)
+    if match:
+        name = match.group(1)
+    return alias_map.get(name, name)
+
+ignored_nicknames = defaultdict(int)
+
+def get_more_authors(log):
+    match = pair_programming_re.match(log)
+    if not match:
+        return set()
+    ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
+                    'in-progress', 'bits of', 'even a little', 'floating',]
+    sep_words = ['and', ';', '+', '/', 'with special  by']
+    nicknames = match.group(1)
+    for word in ignore_words:
+        nicknames = nicknames.replace(word, '')
+    for word in sep_words:
+        nicknames = nicknames.replace(word, ',')
+    nicknames = [nick.strip().lower() for nick in nicknames.split(',')]
+    authors = set()
+    for nickname in nicknames:
+        author = alias_map.get(nickname)
+        if not author:
+            ignored_nicknames[nickname] += 1
+        else:
+            authors.add(author)
+    return authors
+
+def main(show_numbers):
+    ui = mercurial.ui.ui()
+    repo = mercurial.localrepo.localrepository(ui, str(ROOT))
+    authors_count = defaultdict(int)
+    for i in repo:
+        ctx = repo[i]
+        authors = set()
+        authors.add(get_canonical_author(ctx.user()))
+        authors.update(get_more_authors(ctx.description()))
+        for author in authors:
+            if author not in excluded:
+                authors_count[author] += 1
+
+    # uncomment the next lines to get the list of nicknamed which could not be
+    # parsed from commit logs
+    ## items = ignored_nicknames.items()
+    ## items.sort(key=operator.itemgetter(1), reverse=True)
+    ## for name, n in items:
+    ##     if show_numbers:
+    ##         print '%5d %s' % (n, name)
+    ##     else:
+    ##         print name
+                
+    items = authors_count.items()
+    items.sort(key=operator.itemgetter(1), reverse=True)
+    for name, n in items:
+        if show_numbers:
+            print '%5d %s' % (n, name)
+        else:
+            print name
+
+if __name__ == '__main__':
+    show_numbers = '-n' in sys.argv
+    main(show_numbers)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/module/cppyy/test/Makefile b/pypy/module/cppyy/test/Makefile
--- a/pypy/module/cppyy/test/Makefile
+++ b/pypy/module/cppyy/test/Makefile
@@ -46,9 +46,9 @@
 
 ifeq ($(CINT),)
 # TODO: methptrgetter causes these tests to crash, so don't use it for now
-stltypesDict.so: stltypes.cxx stltypes.h stltypes.xml
-	$(genreflex) stltypes.h --selection=stltypes.xml
-	g++ -o $@ stltypes_rflx.cpp stltypes.cxx -shared -lReflex $(cppflags) $(cppflags2)
+#stltypesDict.so: stltypes.cxx stltypes.h stltypes.xml
+#	$(genreflex) stltypes.h --selection=stltypes.xml
+#	g++ -o $@ stltypes_rflx.cpp stltypes.cxx -shared -lReflex $(cppflags) $(cppflags2)
 
 std_streamsDict.so: std_streams.cxx std_streams.h std_streams.xml
 	$(genreflex) std_streams.h --selection=std_streams.xml
diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py
--- a/pypy/module/cppyy/test/test_stltypes.py
+++ b/pypy/module/cppyy/test/test_stltypes.py
@@ -65,15 +65,20 @@
         #-----
         v = tv1()
         for i in range(self.N):
+            print i, 'before'
             v.push_back(i)
+            print i, 'after'
             assert v.size() == i+1
             assert v.at(i) == i
             assert v[i] == i
 
+        return
+
         assert v.size() == self.N
         assert len(v) == self.N
         v.destruct()
 
+
     def test02_user_type_vector_type(self):
         """Test access to an std::vector<just_a_class>"""
 
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2253,24 +2253,6 @@
     """Concat two strings giving a new Unicode string."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
-def PyUnicode_Split(space, s, sep, maxsplit):
-    """Split a string giving a list of Unicode strings.  If sep is NULL, splitting
-    will be done at all whitespace substrings.  Otherwise, splits occur at the given
-    separator.  At most maxsplit splits will be done.  If negative, no limit is
-    set.  Separators are not included in the resulting list.
-
-    This function used an int type for maxsplit. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyUnicode_Splitlines(space, s, keepend):
-    """Split a Unicode string at line breaks, returning a list of Unicode strings.
-    CRLF is considered to be one line break.  If keepend is 0, the Line break
-    characters are not included in the resulting strings."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.CCHARP], PyObject)
 def PyUnicode_Translate(space, str, table, errors):
     """Translate a string by applying a character mapping table to it and return the
@@ -2287,29 +2269,6 @@
     use the default error handling."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t, error=-2)
-def PyUnicode_Find(space, str, substr, start, end, direction):
-    """Return the first position of substr in str*[*start:end] using the given
-    direction (direction == 1 means to do a forward search, direction == -1 a
-    backward search).  The return value is the index of the first match; a value of
-    -1 indicates that no match was found, and -2 indicates that an error
-    occurred and an exception has been set.
-
-    This function used an int type for start and end. This
-    might require changes in your code for properly supporting 64-bit
-    systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
-def PyUnicode_Count(space, str, substr, start, end):
-    """Return the number of non-overlapping occurrences of substr in
-    str[start:end].  Return -1 if an error occurred.
-
-    This function returned an int type and used an int
-    type for start and end. This might require changes in your code for
-    properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
 def PyUnicode_RichCompare(space, left, right, op):
     """Rich compare two unicode strings and return one of the following:
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -457,3 +457,31 @@
         assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+
+    def test_count(self, space, api):
+        w_str = space.wrap(u"abcabdab")
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, -1) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, 2) == 1
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), -5, 30) == 2
+
+    def test_find(self, space, api):
+        w_str = space.wrap(u"abcabcd")
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, 1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, 1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
+
+    def test_split(self, space, api):
+        w_str = space.wrap(u"a\nb\nc\nd")
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
+        assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 0)))
+        assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 1)))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -598,3 +598,46 @@
     else:
         return stringtype.stringendswith(str, substr, start, end)
 
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
+def PyUnicode_Count(space, w_str, w_substr, start, end):
+    """Return the number of non-overlapping occurrences of substr in
+    str[start:end].  Return -1 if an error occurred."""
+    w_count = space.call_method(w_str, "count", w_substr,
+                                space.wrap(start), space.wrap(end))
+    return space.int_w(w_count)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
+             Py_ssize_t, error=-2)
+def PyUnicode_Find(space, w_str, w_substr, start, end, direction):
+    """Return the first position of substr in str*[*start:end] using
+    the given direction (direction == 1 means to do a forward search,
+    direction == -1 a backward search).  The return value is the index
+    of the first match; a value of -1 indicates that no match was
+    found, and -2 indicates that an error occurred and an exception
+    has been set."""
+    if rffi.cast(lltype.Signed, direction) > 0:
+        w_pos = space.call_method(w_str, "find", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    else:
+        w_pos = space.call_method(w_str, "rfind", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    return space.int_w(w_pos)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
+def PyUnicode_Split(space, w_str, w_sep, maxsplit):
+    """Split a string giving a list of Unicode strings.  If sep is
+    NULL, splitting will be done at all whitespace substrings.
+    Otherwise, splits occur at the given separator.  At most maxsplit
+    splits will be done.  If negative, no limit is set.  Separators
+    are not included in the resulting list."""
+    if w_sep is None:
+        w_sep = space.w_None
+    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyUnicode_Splitlines(space, w_str, keepend):
+    """Split a Unicode string at line breaks, returning a list of
+    Unicode strings.  CRLF is considered to be one line break.  If
+    keepend is 0, the Line break characters are not included in the
+    resulting strings."""
+    return space.call_method(w_str, "splitlines", space.wrap(keepend))
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -105,6 +105,7 @@
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -132,6 +133,8 @@
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
         ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -546,6 +546,7 @@
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
+            ("trunc", "trunc", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
             ("exp2", "exp2", 1, {"promote_to_float": True}),
             ("expm1", "expm1", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -253,24 +253,17 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floorceil(self):
-        from _numpypy import array, floor, ceil
+    def test_floorceiltrunc(self):
+        from _numpypy import array, floor, ceil, trunc
         import math
-        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = floor(a)
-        for i in range(5):
-            assert b[i] == reference[i]
-        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = ceil(a)
-        assert (reference == b).all()
-        inf = float("inf")
-        data = [1.5, 2.9999, -1.999, inf]
-        results = [math.floor(x) for x in data]
-        assert (floor(data) == results).all()
-        results = [math.ceil(x) for x in data]
-        assert (ceil(data) == results).all()
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == floor(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
+        assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -597,6 +590,12 @@
         assert (bitwise_not(a) == ~a).all()
         assert (invert(a) == ~a).all()
 
+    def test_shift(self):
+        from _numpypy import left_shift, right_shift
+
+        assert (left_shift([5, 1], [2, 13]) == [20, 2**13]).all()
+        assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
+
     def test_comparisons(self):
         import operator
         from _numpypy import equal, not_equal, less, less_equal, greater, greater_equal
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -668,6 +668,13 @@
         return math.ceil(v)
 
     @simple_unary_op
+    def trunc(self, v):
+        if v < 0:
+            return math.ceil(v)
+        else:
+            return math.floor(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
diff --git a/pypy/module/pyexpat/test/__init__.py b/pypy/module/pyexpat/test/__init__.py
new file mode 100644
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 2, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 9, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -434,6 +434,13 @@
         self.lastblock = block
         self.pushvalue(w_result)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # This opcode was added with pypy-1.8.  Here is a simpler
+        # version, enough for annotation.
+        last_val = self.popvalue()
+        self.pushvalue(self.space.newlist([]))
+        self.pushvalue(last_val)
+
     # XXX Unimplemented 2.7 opcodes ----------------
 
     # Set literals, set comprehensions
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,8 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, classofinstance=None,
-                                   strdict=False):
+                                   instance=False, strdict=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -563,10 +562,7 @@
     def listview_int(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
 
-    def w_keys(self, w_dict):
-        # XXX there is no space.newlist_int yet
-        space = self.space
-        return space.call_function(space.w_list, w_dict)
+    # XXX there is no space.newlist_int yet to implement w_keys more efficiently
 
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -1,5 +1,5 @@
 ## ----------------------------------------------------------------------------
-## dict strategy (see dict_multiobject.py)
+## dict strategy (see dictmultiobject.py)
 
 from pypy.rlib import rerased
 from pypy.rlib.debug import mark_dict_non_null
@@ -80,8 +80,8 @@
     def iter(self, w_dict):
         return IdentityDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class IdentityDictIteratorImplementation(_UnwrappedIteratorMixin, IteratorImplementation):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,10 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                classofinstance=classofinstance,
                 strdict=strdict)
 
     def newset(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -885,10 +885,9 @@
     def newtuple(self, l):
         return tuple(l)
 
-    def newdict(self, module=False, instance=False, classofinstance=None):
+    def newdict(self, module=False, instance=False):
         return W_DictMultiObject.allocate_and_init_instance(
-                self, module=module, instance=instance,
-                classofinstance=classofinstance)
+                self, module=module, instance=instance)
 
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict
@@ -968,6 +967,20 @@
         assert type(self.impl.strategy) is self.StrategyClass
         #assert self.impl.r_dict_content is None
 
+    def test_popitem(self):
+        self.fill_impl()
+        assert self.impl.length() == 2
+        a, b = self.impl.popitem()
+        assert self.impl.length() == 1
+        if a == self.string:
+            assert b == 1000
+            assert self.impl.getitem(self.string2) == 2000
+        else:
+            assert a == self.string2
+            assert b == 2000
+            assert self.impl.getitem_str(self.string) == 1000
+        self.check_not_devolved()
+
     def test_setitem(self):
         self.impl.setitem(self.string, 1000)
         assert self.impl.length() == 1
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old -rf
+addopts = --assert=plain -rf
diff --git a/pypy/rlib/rsre/rsre_re.py b/pypy/rlib/rsre/rsre_re.py
--- a/pypy/rlib/rsre/rsre_re.py
+++ b/pypy/rlib/rsre/rsre_re.py
@@ -172,8 +172,9 @@
         self._ctx = ctx
 
     def span(self, groupnum=0):
-        if not isinstance(groupnum, (int, long)):
-            groupnum = self.re.groupindex[groupnum]
+#        if not isinstance(groupnum, (int, long)):
+#            groupnum = self.re.groupindex[groupnum]
+ 
         return self._ctx.span(groupnum)
 
     def start(self, groupnum=0):
@@ -182,19 +183,25 @@
     def end(self, groupnum=0):
         return self.span(groupnum)[1]
 
-    def group(self, *groups):
-        groups = groups or (0,)
-        result = []
-        for group in groups:
-            frm, to = self.span(group)
-            if 0 <= frm <= to:
-                result.append(self._ctx._string[frm:to])
-            else:
-                result.append(None)
-        if len(result) > 1:
-            return tuple(result)
+    def group(self, group=0):
+        frm, to = self.span(group)
+        if 0 <= frm <= to:
+            return self._ctx._string[frm:to]
         else:
-            return result[0]
+            return None
+
+#    def group(self, *groups):
+#        groups = groups or (0,)
+#        result = []
+#        for group in groups:
+#            frm, to = self.span(group)
+#            if 0 <= frm <= to:
+#                result.append(self._ctx._string[frm:to])
+#            else:
+#                result.append(None)
+#        if len(result) > 1:
+#            return tuple(result)
+
 
     def groups(self, default=None):
         fmarks = self._ctx.flatten_marks()
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -204,7 +204,7 @@
         assert re.match('(a)', 'a').groups() == ('a',)
         assert re.match(r'(a)', 'a').group(0) == 'a'
         assert re.match(r'(a)', 'a').group(1) == 'a'
-        assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
+        #assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
 
         pat = re.compile('((a)|(b))(c)?')
         assert pat.match('a').groups() == ('a', 'a', None, None)
@@ -218,13 +218,13 @@
         assert m.group(0) == 'a'
         assert m.group(0) == 'a'
         assert m.group(1) == 'a'
-        assert m.group(1, 1) == ('a', 'a')
+        #assert m.group(1, 1) == ('a', 'a')
 
         pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?')
-        assert pat.match('a').group(1, 2, 3) == ('a', None, None)
-        assert pat.match('b').group('a1', 'b2', 'c3') == (
-                         (None, 'b', None))
-        assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
+        #assert pat.match('a').group(1, 2, 3) == ('a', None, None)
+        #assert pat.match('b').group('a1', 'b2', 'c3') == (
+        #                 (None, 'b', None))
+        #assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
     def test_bug_923(self):
         # Issue923: grouping inside optional lookahead problem
diff --git a/pypy/rlib/rsre/test/test_zinterp.py b/pypy/rlib/rsre/test/test_zinterp.py
--- a/pypy/rlib/rsre/test/test_zinterp.py
+++ b/pypy/rlib/rsre/test/test_zinterp.py
@@ -1,7 +1,8 @@
 # minimal test: just checks that (parts of) rsre can be translated
 
-from pypy.rpython.test.test_llinterp import gengraph
+from pypy.rpython.test.test_llinterp import gengraph, interpret
 from pypy.rlib.rsre import rsre_core
+from pypy.rlib.rsre.rsre_re import compile
 
 def main(n):
     assert n >= 0
@@ -19,3 +20,18 @@
 
 def test_gengraph():
     t, typer, graph = gengraph(main, [int])
+
+m = compile("(a|b)aaaaa")
+
+def test_match():
+    def f(i):
+        if i:
+            s = "aaaaaa"
+        else:
+            s = "caaaaa"
+        g = m.match(s)
+        if g is None:
+            return 3
+        return int("aaaaaa" == g.group(0))
+    assert interpret(f, [3]) == 1
+    assert interpret(f, [0]) == 3
diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py
--- a/pypy/rpython/annlowlevel.py
+++ b/pypy/rpython/annlowlevel.py
@@ -543,11 +543,11 @@
         else:
             assert False
 
+        hop.exception_cannot_occur()
         if isinstance(hop.args_r[1], rpbc.NoneFrozenPBCRepr):
             return hop.inputconst(PTR, null)
         v_arg = hop.inputarg(hop.args_r[1], arg=1)
         assert isinstance(v_arg.concretetype, T)
-        hop.exception_cannot_occur()
         return hop.genop(opname, [v_arg], resulttype = PTR)
 
 
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1426,23 +1426,25 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
-        # If 'obj' was already forwarded, change it to its forwarding address.
-        if self.is_forwarded(obj):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
+            #
+            # Common case: 'obj' was not already forwarded (otherwise
+            # tid == -42, containing all flags), and it doesn't have the
+            # HAS_SHADOW flag either.  We must move it out of the nursery,
+            # into a new nonmovable location.
+            totalsize = size_gc_header + self.get_size(obj)
+            newhdr = self._malloc_out_of_nursery(totalsize)
+            #
+        elif self.is_forwarded(obj):
+            #
+            # 'obj' was already forwarded.  Change the original reference
+            # to point to its forwarding address, and we're done.
             root.address[0] = self.get_forwarding_address(obj)
             return
-        #
-        # First visit to 'obj': we must move it out of the nursery.
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        size = self.get_size(obj)
-        totalsize = size_gc_header + size
-        #
-        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
-            #
-            # Common case: allocate a new nonmovable location for it.
-            newhdr = self._malloc_out_of_nursery(totalsize)
             #
         else:
-            # The object has already a shadow.
+            # First visit to an object that has already a shadow.
             newobj = self.nursery_objects_shadows.get(obj)
             ll_assert(newobj != NULL, "GCFLAG_HAS_SHADOW but no shadow found")
             newhdr = newobj - size_gc_header
@@ -1450,6 +1452,8 @@
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
             # copied to the shadow itself.
             self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            #
+            totalsize = size_gc_header + self.get_size(obj)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -455,6 +455,6 @@
     return intmask(time), intmask(nsec)
 
 def time_t_to_FILE_TIME(time, filetime):
-    ft = (rffi.r_longlong(time) + secs_between_epochs) * 10000000
+    ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
     filetime.c_dwLowDateTime = rffi.r_uint(ft)    # masking off high bits
diff --git a/pypy/test_all.py b/pypy/test_all.py
old mode 100755
new mode 100644
--- a/pypy/test_all.py
+++ b/pypy/test_all.py
@@ -11,11 +11,12 @@
 """
 import sys, os
 
-if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
-    print >> sys.stderr, __doc__
-    sys.exit(2)
 
 if __name__ == '__main__':
+    if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
+        print >> sys.stderr, __doc__
+        sys.exit(2)
+
     import tool.autopath
     import pytest
     import pytest_cov
diff --git a/pypy/tool/jitlogparser/test/__init__.py b/pypy/tool/jitlogparser/test/__init__.py
new file mode 100644
diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -165,7 +165,10 @@
     def test_one(self): exec 'blow'
     """)
 
-    ev, = sorter.getreports("pytest_runtest_logreport")
+    reports = sorter.getreports("pytest_runtest_logreport")
+    setup, ev, teardown = reports
     assert ev.failed
+    assert setup.passed
+    assert teardown.passed
     assert 'NameError' in ev.longrepr.reprcrash.message
     assert 'blow' in ev.longrepr.reprcrash.message
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -847,6 +847,10 @@
                 if sources:
                     target, = sources
 
+        if target.endswith('@PLT'):
+            # In -fPIC mode, all functions calls have this suffix
+            target = target[:-4]
+
         if target in self.FUNCTIONS_NOT_RETURNING:
             return [InsnStop(target)]
         if self.format == 'mingw32' and target == '__alloca':
@@ -1137,7 +1141,7 @@
     r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$")
 
     r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$")
-    r_jmptable_item = re.compile(r"\t.quad\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$")
+    r_jmptable_item = re.compile(r"\t.(?:quad|long)\t"+LABEL+"(-\"[A-Za-z0-9$]+\"|-"+LABEL+")?\s*$")
     r_jmptable_end  = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL)
 
     r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/")
diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -110,7 +110,10 @@
                  do_dry_run=False, timeout=None,
                  _win32=(sys.platform=='win32')):
     args = interp + test_driver
-    args += ['-p', 'resultlog', '--resultlog=%s' % logfname, test]
+    args += ['-p', 'resultlog',
+             '--resultlog=%s' % logfname,
+             '--junitxml=%s.junit' % logfname,
+             test]
 
     args = map(str, args)
     interp0 = args[0]
diff --git a/testrunner/scratchbox_runner.py b/testrunner/scratchbox_runner.py
--- a/testrunner/scratchbox_runner.py
+++ b/testrunner/scratchbox_runner.py
@@ -14,14 +14,14 @@
 def dry_run_scratchbox(args, cwd, out, timeout=None):
     return dry_run(args_for_scratchbox(cwd, args), cwd, out, timeout)
 
-import runner
-# XXX hack hack hack
-dry_run = runner.dry_run
-run = runner.run
+if __name__ == '__main__':
+    import runner
+    # XXX hack hack hack
+    dry_run = runner.dry_run
+    run = runner.run
 
-runner.dry_run = dry_run_scratchbox
-runner.run = run_scratchbox
+    runner.dry_run = dry_run_scratchbox
+    runner.run = run_scratchbox
 
-if __name__ == '__main__':
     import sys
     runner.main(sys.argv)
diff --git a/testrunner/test/conftest.py b/testrunner/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/testrunner/test/conftest.py
@@ -0,0 +1,6 @@
+
+def pytest_runtest_makereport(__multicall__, item):
+    report = __multicall__.execute()
+    if 'out' in item.funcargs:
+        report.sections.append(('out', item.funcargs['out'].read()))
+    return report
diff --git a/testrunner/test/test_runner.py b/testrunner/test/test_runner.py
--- a/testrunner/test/test_runner.py
+++ b/testrunner/test/test_runner.py
@@ -53,49 +53,44 @@
     assert not should_report_failure("F Def\n. Ghi\n. Jkl\n")
 
 
+
 class TestRunHelper(object):
+    def pytest_funcarg__out(self, request):
+        tmpdir = request.getfuncargvalue('tmpdir')
+        return tmpdir.ensure('out')
 
-    def setup_method(self, meth):
-        h, self.fn = tempfile.mkstemp()
-        os.close(h)
+    def test_run(self, out):
+        res = runner.run([sys.executable, "-c", "print 42"], '.', out)
+        assert res == 0
+        assert out.read() == "42\n"
 
-    def teardown_method(self, meth):
-        os.unlink(self.fn)
-
-    def test_run(self):
-        res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn))
-        assert res == 0
-        out = py.path.local(self.fn).read('r')
-        assert out == "42\n"
-
-    def test_error(self):
-        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', py.path.local(self.fn))
+    def test_error(self, out):
+        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', out)
         assert res == 3
 
-    def test_signal(self):
+    def test_signal(self, out):
         if sys.platform == 'win32':
             py.test.skip("no death by signal on windows")
-        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', py.path.local(self.fn))
+        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', out)
         assert res == -9
 
-    def test_timeout(self):
-        res = runner.run([sys.executable, "-c", "while True: pass"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout(self, out):
+        res = runner.run([sys.executable, "-c", "while True: pass"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_lock(self):
-        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_lock(self, out):
+        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_syscall(self):
-        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_syscall(self, out):
+        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', out, timeout=3)
         assert res == -999        
 
-    def test_timeout_success(self):
+    def test_timeout_success(self, out):
         res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn), timeout=2)
+                         out, timeout=2)
         assert res == 0
-        out = py.path.local(self.fn).read('r')
+        out = out.read()
         assert out == "42\n"        
 
 
@@ -122,7 +117,10 @@
 
         expected = ['INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
+
                     'test_one']
 
         assert self.called == (expected, '/wd', 'out', 'secs')        
@@ -138,9 +136,11 @@
 
         expected = ['/wd' + os.sep + './INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
                     'test_one']
-
+        assert self.called[0] == expected
         assert self.called == (expected, '/wd', 'out', 'secs')        
         assert res == 0
 
@@ -251,7 +251,7 @@
         assert '\n' in log        
         log_lines = log.splitlines()
 
-        assert log_lines[0] == ". test_normal/test_example.py:test_one"
+        assert ". test_normal/test_example.py::test_one" in log_lines
         nfailures = 0
         noutcomes = 0
         for line in log_lines:

From noreply at buildbot.pypy.org  Fri Apr 13 05:43:24 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 13 Apr 2012 05:43:24 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: remove debugging printout
Message-ID: <20120413034324.576B782F4E@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54320:b1f5b7f6d85b
Date: 2012-04-12 11:29 -0700
http://bitbucket.org/pypy/pypy/changeset/b1f5b7f6d85b/

Log:	remove debugging printout

diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py
--- a/pypy/module/cppyy/test/test_stltypes.py
+++ b/pypy/module/cppyy/test/test_stltypes.py
@@ -65,9 +65,7 @@
         #-----
         v = tv1()
         for i in range(self.N):
-            print i, 'before'
             v.push_back(i)
-            print i, 'after'
             assert v.size() == i+1
             assert v.at(i) == i
             assert v[i] == i

From noreply at buildbot.pypy.org  Fri Apr 13 05:43:25 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 13 Apr 2012 05:43:25 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: unlike for stubs (for which a
 value type is used), libffi requires a pointer type for const builtin&
Message-ID: <20120413034325.9A05282F4E@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54321:86397c9e6d24
Date: 2012-04-12 11:52 -0700
http://bitbucket.org/pypy/pypy/changeset/86397c9e6d24/

Log:	unlike for stubs (for which a value type is used), libffi requires a
	pointer type for const builtin&

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -275,6 +275,10 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(rffi.SHORT, space.int_w(w_obj))
 
+class ConstShortRefConverter(ShortConverter):
+    _immutable_ = True
+    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+
 class UnsignedShortConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.sshort
@@ -286,6 +290,10 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(rffi.USHORT, space.int_w(w_obj))
 
+class ConstUnsignedShortRefConverter(UnsignedShortConverter):
+    _immutable_ = True
+    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+
 class IntConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.sint
@@ -297,6 +305,10 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(rffi.INT, space.c_int_w(w_obj))
 
+class ConstIntRefConverter(IntConverter):
+    _immutable_ = True
+    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+
 class UnsignedIntConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.uint
@@ -308,6 +320,10 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(rffi.UINT, space.uint_w(w_obj))
 
+class ConstUnsignedIntRefConverter(UnsignedIntConverter):
+    _immutable_ = True
+    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+
 class LongConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.slong
@@ -319,6 +335,10 @@
     def _unwrap_object(self, space, w_obj):
         return space.int_w(w_obj)
 
+class ConstLongRefConverter(LongConverter):
+    _immutable_ = True
+    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+
 class UnsignedLongConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.ulong
@@ -330,6 +350,9 @@
     def _unwrap_object(self, space, w_obj):
         return space.uint_w(w_obj)
 
+class ConstUnsignedLongRefConverter(UnsignedLongConverter):
+    _immutable_ = True
+    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
 
 class FloatConverter(FloatTypeConverterMixin, TypeConverter):
     _immutable_ = True
@@ -352,6 +375,10 @@
         rffiptr = rffi.cast(self.rffiptype, address)
         return space.wrap(float(rffiptr[0]))
 
+class ConstFloatRefConverter(FloatConverter):
+    _immutable_ = True
+    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+
 class DoubleConverter(FloatTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.double
@@ -367,6 +394,10 @@
     def _unwrap_object(self, space, w_obj):
         return space.float_w(w_obj)
 
+class ConstDoubleRefConverter(DoubleConverter):
+    _immutable_ = True
+    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+
 
 class CStringConverter(TypeConverter):
     _immutable_ = True
@@ -593,10 +624,10 @@
     #   1) full, exact match
     #       1a) const-removed match
     #   2) match of decorated, unqualified type
-    #   3) accept const ref as by value
-    #   4) accept ref as pointer
-    #   5) generalized cases (covers basically all user classes)
-    #   6) void converter, which fails on use
+    #   3) accept ref as pointer (for the stubs, const& can be
+    #       by value, but that does not work for the ffi path)
+    #   4) generalized cases (covers basically all user classes)
+    #   5) void converter, which fails on use
 
     name = capi.c_resolve_name(name)
 
@@ -622,14 +653,9 @@
     except KeyError:
         pass
 
-    #   3) accept const ref as by value
-    if compound and compound[len(compound)-1] == "&":
-        try:
-            return _converters[clean_name](space, default)
-        except KeyError:
-            pass
+    #   3) TODO: accept ref as pointer
 
-    #   5) generalized cases (covers basically all user classes)
+    #   4) generalized cases (covers basically all user classes)
     from pypy.module.cppyy import interp_cppyy
     cppclass = interp_cppyy.scope_byname(space, clean_name)
     if cppclass:
@@ -643,7 +669,7 @@
     elif capi.c_is_enum(clean_name):
         return UnsignedIntConverter(space, default)
     
-    #   6) void converter, which fails on use
+    #   5) void converter, which fails on use
     #
     # return a void converter here, so that the class can be build even
     # when some types are unknown; this overload will simply fail on use
@@ -654,17 +680,29 @@
 _converters["char"]                     = CharConverter
 _converters["unsigned char"]            = CharConverter
 _converters["short int"]                = ShortConverter
+_converters["const short int&"]         = ConstIntRefConverter
 _converters["short"]                    = _converters["short int"]
+_converters["const short&"]             = _converters["const short int&"]
 _converters["unsigned short int"]       = UnsignedShortConverter
+_converters["const unsigned short int&"] = ConstUnsignedShortRefConverter
 _converters["unsigned short"]           = _converters["unsigned short int"]
+_converters["const unsigned short&"]    = _converters["const unsigned short int&"]
 _converters["int"]                      = IntConverter
+_converters["const int&"]               = ConstIntRefConverter
 _converters["unsigned int"]             = UnsignedIntConverter
+_converters["const unsigned int&"]      = ConstUnsignedIntRefConverter
 _converters["long int"]                 = LongConverter
+_converters["const long int&"]          = ConstLongRefConverter
 _converters["long"]                     = _converters["long int"]
+_converters["const long&"]              = _converters["const long int&"]
 _converters["unsigned long int"]        = UnsignedLongConverter
+_converters["const unsigned long int&"] = ConstUnsignedLongRefConverter
 _converters["unsigned long"]            = _converters["unsigned long int"]
+_converters["const unsigned long&"]     = _converters["const unsigned long int&"]
 _converters["float"]                    = FloatConverter
+_converters["const float&"]             = ConstFloatRefConverter
 _converters["double"]                   = DoubleConverter
+_converters["const double&"]            = ConstDoubleRefConverter
 _converters["const char*"]              = CStringConverter
 _converters["char*"]                    = CStringConverter
 _converters["void*"]                    = VoidPtrConverter
diff --git a/pypy/module/cppyy/genreflex-methptrgetter.patch b/pypy/module/cppyy/genreflex-methptrgetter.patch
--- a/pypy/module/cppyy/genreflex-methptrgetter.patch
+++ b/pypy/module/cppyy/genreflex-methptrgetter.patch
@@ -1,6 +1,6 @@
 Index: cint/reflex/python/genreflex/gendict.py
 ===================================================================
---- cint/reflex/python/genreflex/gendict.py	(revision 40448)
+--- cint/reflex/python/genreflex/gendict.py	(revision 43705)
 +++ cint/reflex/python/genreflex/gendict.py	(working copy)
 @@ -52,6 +52,7 @@
      self.typedefs_for_usr = []
@@ -10,7 +10,7 @@
      # The next is to avoid a known problem with gccxml that it generates a
      # references to id equal '_0' which is not defined anywhere
      self.xref['_0'] = {'elem':'Unknown', 'attrs':{'id':'_0','name':''}, 'subelems':[]}
-@@ -1281,6 +1282,8 @@
+@@ -1306,6 +1307,8 @@
      bases = self.getBases( attrs['id'] )
      if inner and attrs.has_key('demangled') and self.isUnnamedType(attrs['demangled']) :
        cls = attrs['demangled']
@@ -19,7 +19,7 @@
        clt = ''
      else:
        cls = self.genTypeName(attrs['id'],const=True,colon=True)
-@@ -1318,7 +1321,7 @@
+@@ -1343,7 +1346,7 @@
        # Inner class/struct/union/enum.
        for m in memList :
          member = self.xref[m]
@@ -28,7 +28,7 @@
             and member['attrs'].get('access') in ('private','protected') \
             and not self.isUnnamedType(member['attrs'].get('demangled')):
            cmem = self.genTypeName(member['attrs']['id'],const=True,colon=True)
-@@ -1956,8 +1959,15 @@
+@@ -1981,8 +1984,15 @@
      else    : params  = '0'
      s = '  .AddFunctionMember(%s, Reflex::Literal("%s"), %s%s, 0, %s, %s)' % (self.genTypeID(id), name, type, id, params, mod)
      s += self.genCommentProperty(attrs)
@@ -44,7 +44,7 @@
    def genMCODef(self, type, name, attrs, args):
      id       = attrs['id']
      cl       = self.genTypeName(attrs['context'],colon=True)
-@@ -2024,8 +2034,44 @@
+@@ -2049,8 +2059,44 @@
            if returns == 'void' : body += '  }\n'
            else :                 body += '  }\n'
      body += '}\n'
@@ -92,7 +92,7 @@
      for a in args :
 Index: cint/reflex/python/genreflex/genreflex.py
 ===================================================================
---- cint/reflex/python/genreflex/genreflex.py	(revision 40448)
+--- cint/reflex/python/genreflex/genreflex.py	(revision 43705)
 +++ cint/reflex/python/genreflex/genreflex.py	(working copy)
 @@ -108,6 +108,10 @@
           Print extra debug information while processing. Keep intermediate files\n
diff --git a/pypy/module/cppyy/test/Makefile b/pypy/module/cppyy/test/Makefile
--- a/pypy/module/cppyy/test/Makefile
+++ b/pypy/module/cppyy/test/Makefile
@@ -46,17 +46,9 @@
 
 ifeq ($(CINT),)
 # TODO: methptrgetter causes these tests to crash, so don't use it for now
-#stltypesDict.so: stltypes.cxx stltypes.h stltypes.xml
-#	$(genreflex) stltypes.h --selection=stltypes.xml
-#	g++ -o $@ stltypes_rflx.cpp stltypes.cxx -shared -lReflex $(cppflags) $(cppflags2)
-
 std_streamsDict.so: std_streams.cxx std_streams.h std_streams.xml
 	$(genreflex) std_streams.h --selection=std_streams.xml
 	g++ -o $@ std_streams_rflx.cpp std_streams.cxx -shared -lReflex $(cppflags) $(cppflags2)
-
-operatorsDict.so: operators.cxx operators.h operators.xml
-	$(genreflex) operators.h --selection=operators.xml
-	g++ -o $@ operators_rflx.cpp operators.cxx -shared -lReflex $(cppflags) $(cppflags2)
 endif
 
 .PHONY: clean
diff --git a/pypy/module/cppyy/test/datatypes.cxx b/pypy/module/cppyy/test/datatypes.cxx
--- a/pypy/module/cppyy/test/datatypes.cxx
+++ b/pypy/module/cppyy/test/datatypes.cxx
@@ -114,18 +114,26 @@
 cppyy_test_pod*& cppyy_test_data::get_pod_ptrref() { return m_ppod; }
 
 //- setters -----------------------------------------------------------------
-void cppyy_test_data::set_bool(bool b)              { m_bool   = b; }
-void cppyy_test_data::set_char(char c)              { m_char   = c; }
-void cppyy_test_data::set_uchar(unsigned char uc)   { m_uchar  = uc; }
-void cppyy_test_data::set_short(short s)            { m_short  = s; }
-void cppyy_test_data::set_ushort(unsigned short us) { m_ushort = us; }
-void cppyy_test_data::set_int(int i)                { m_int    = i; }
-void cppyy_test_data::set_uint(unsigned int ui)     { m_uint   = ui; }
-void cppyy_test_data::set_long(long l)              { m_long   = l; }
-void cppyy_test_data::set_ulong(unsigned long ul)   { m_ulong  = ul; }
-void cppyy_test_data::set_float(float f)            { m_float  = f; }
-void cppyy_test_data::set_double(double d)          { m_double = d; }
-void cppyy_test_data::set_enum(what w)              { m_enum   = w; }
+void cppyy_test_data::set_bool(bool b)                       { m_bool   = b; }
+void cppyy_test_data::set_char(char c)                       { m_char   = c; }
+void cppyy_test_data::set_uchar(unsigned char uc)            { m_uchar  = uc; }
+void cppyy_test_data::set_short(short s)                     { m_short  = s; }
+void cppyy_test_data::set_short_c(const short& s)            { m_short  = s; }
+void cppyy_test_data::set_ushort(unsigned short us)          { m_ushort = us; }
+void cppyy_test_data::set_ushort_c(const unsigned short& us) { m_ushort = us; }
+void cppyy_test_data::set_int(int i)                         { m_int    = i; }
+void cppyy_test_data::set_int_c(const int& i)                { m_int    = i; }
+void cppyy_test_data::set_uint(unsigned int ui)              { m_uint   = ui; }
+void cppyy_test_data::set_uint_c(const unsigned int& ui)     { m_uint   = ui; }
+void cppyy_test_data::set_long(long l)                       { m_long   = l; }
+void cppyy_test_data::set_long_c(const long& l)              { m_long   = l; }
+void cppyy_test_data::set_ulong(unsigned long ul)            { m_ulong  = ul; }
+void cppyy_test_data::set_ulong_c(const unsigned long& ul)   { m_ulong  = ul; }
+void cppyy_test_data::set_float(float f)                     { m_float  = f; }
+void cppyy_test_data::set_float_c(const float& f)            { m_float  = f; }
+void cppyy_test_data::set_double(double d)                   { m_double = d; }
+void cppyy_test_data::set_double_c(const double& d)          { m_double = d; }
+void cppyy_test_data::set_enum(what w)                       { m_enum   = w; }
 
 char           cppyy_test_data::s_char   = 's';
 unsigned char  cppyy_test_data::s_uchar  = 'u';
diff --git a/pypy/module/cppyy/test/datatypes.h b/pypy/module/cppyy/test/datatypes.h
--- a/pypy/module/cppyy/test/datatypes.h
+++ b/pypy/module/cppyy/test/datatypes.h
@@ -62,13 +62,21 @@
     void set_char(char c);
     void set_uchar(unsigned char uc);
     void set_short(short s);
+    void set_short_c(const short& s);
     void set_ushort(unsigned short us);
+    void set_ushort_c(const unsigned short& us);
     void set_int(int i);
+    void set_int_c(const int& i);
     void set_uint(unsigned int ui);
+    void set_uint_c(const unsigned int& ui);
     void set_long(long l);
+    void set_long_c(const long& l);
     void set_ulong(unsigned long ul);
+    void set_ulong_c(const unsigned long& ul);
     void set_float(float f);
+    void set_float_c(const float& f);
     void set_double(double d);
+    void set_double_c(const double& d);
     void set_enum(what w);
 
 public:
diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py
--- a/pypy/module/cppyy/test/test_datatypes.py
+++ b/pypy/module/cppyy/test/test_datatypes.py
@@ -152,8 +152,12 @@
             assert eval('c.get_%s()' % names[i]) == i
 
         for i in range(len(names)):
-            exec 'c.set_%s = %d' % (names[i],2*i)
-            assert eval('c.m_%s' % names[i]) == i
+            exec 'c.set_%s(%d)' % (names[i],2*i)
+            assert eval('c.m_%s' % names[i]) == 2*i
+
+        for i in range(len(names)):
+            exec 'c.set_%s_c(%d)' % (names[i],3*i)
+            assert eval('c.m_%s' % names[i]) == 3*i
 
         # float types through functions
         c.set_float( 0.123 );  assert round(c.get_float()  - 0.123, 5) == 0
@@ -161,9 +165,11 @@
 
         # float types through data members
         c.m_float = 0.123;     assert round(c.get_float()  - 0.123, 5) == 0
-        c.set_float( 0.234 );  assert round(c.m_float      - 0.234, 5) == 0
-        c.m_double = 0.456;    assert round(c.get_double() - 0.456, 8) == 0
-        c.set_double( 0.567 ); assert round(c.m_double     - 0.567, 8) == 0
+        c.set_float(0.234);    assert round(c.m_float      - 0.234, 5) == 0
+        c.set_float_c(0.456);  assert round(c.m_float      - 0.456, 5) == 0
+        c.m_double = 0.678;    assert round(c.get_double() - 0.678, 8) == 0
+        c.set_double(0.890);   assert round(c.m_double     - 0.890, 8) == 0
+        c.set_double_c(0.012); assert round(c.m_double     - 0.012, 8) == 0
 
         # arrays; there will be pointer copies, so destroy the current ones
         c.destroy_arrays()

From noreply at buildbot.pypy.org  Fri Apr 13 05:43:26 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 13 Apr 2012 05:43:26 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: const builtin& for CINT backend
Message-ID: <20120413034326.E57FA82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54322:6c02358185a1
Date: 2012-04-12 12:41 -0700
http://bitbucket.org/pypy/pypy/changeset/6c02358185a1/

Log:	const builtin& for CINT backend

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -338,6 +338,13 @@
 class ConstLongRefConverter(LongConverter):
     _immutable_ = True
     libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    typecode = 'r'
+
+    def convert_argument(self, space, w_obj, address):
+        x = rffi.cast(self.rffiptype, address)
+        x[0] = self._unwrap_object(space, w_obj)
+        ba = rffi.cast(rffi.CCHARP, address)
+        ba[capi.c_function_arg_typeoffset()] = self.typecode
 
 class UnsignedLongConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
@@ -378,6 +385,7 @@
 class ConstFloatRefConverter(FloatConverter):
     _immutable_ = True
     libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    typecode = 'F'
 
 class DoubleConverter(FloatTypeConverterMixin, TypeConverter):
     _immutable_ = True
@@ -397,6 +405,7 @@
 class ConstDoubleRefConverter(DoubleConverter):
     _immutable_ = True
     libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    typecode = 'D'
 
 
 class CStringConverter(TypeConverter):
diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx
--- a/pypy/module/cppyy/src/cintcwrapper.cxx
+++ b/pypy/module/cppyy/src/cintcwrapper.cxx
@@ -167,15 +167,33 @@
     for (int i = 0; i < libp->paran; ++i) {
         libp->para[i].ref = libp->para[i].obj.i;
         const char partype = libp->para[i].type;
-        if (partype == 'p')
+        switch (partype) {
+        case 'p': {
             libp->para[i].obj.i = (long)&libp->para[i].ref;
-        else if (partype == 'r')
+            break;
+        }
+        case 'r': {
             libp->para[i].ref = (long)&libp->para[i].obj.i;
-        else if (partype == 'f') {
+            break;
+        }
+        case 'f': {
             assert(sizeof(float) <= sizeof(long));
             long val = libp->para[i].obj.i;
             void* pval = (void*)&val;
             libp->para[i].obj.d = *(float*)pval;
+            break;
+        }
+        case 'F': {
+            libp->para[i].ref = (long)&libp->para[i].obj.i;
+            libp->para[i].type = 'f';
+            break;
+        }
+        case 'D': {
+            libp->para[i].ref = (long)&libp->para[i].obj.i;
+            libp->para[i].type = 'd';
+            break;
+
+        }
         }
     }
 }

From noreply at buildbot.pypy.org  Fri Apr 13 05:43:28 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 13 Apr 2012 05:43:28 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: const ref only worked b/c it
 didn't; 2nd attempt with explicit tests
Message-ID: <20120413034328.5003682F4E@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54323:1ea1ba708588
Date: 2012-04-12 20:43 -0700
http://bitbucket.org/pypy/pypy/changeset/1ea1ba708588/

Log:	const ref only worked b/c it didn't; 2nd attempt with explicit tests

diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -150,26 +150,39 @@
 
     def convert_argument_libffi(self, space, w_obj, argchain):
         argchain.arg(self._unwrap_object(space, w_obj))
+        return lltype.nullptr(rffi.VOIDP.TO)
 
     def default_argument_libffi(self, space, argchain):
         argchain.arg(self.default)
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
-        rffiptr = rffi.cast(self.rffiptype, address)
+        rffiptr = rffi.cast(self.c_ptrtype, address)
         return space.wrap(rffiptr[0])
 
     def to_memory(self, space, w_obj, w_value, offset):
         address = self._get_raw_address(space, w_obj, offset)
-        rffiptr = rffi.cast(self.rffiptype, address)
+        rffiptr = rffi.cast(self.c_ptrtype, address)
         rffiptr[0] = self._unwrap_object(space, w_value)
 
+class ConstRefNumericTypeConverterMixin(NumericTypeConverterMixin):
+    _mixin_ = True
+    _immutable_ = True
+
+    def convert_argument_libffi(self, space, w_obj, argchain):
+        obj = self._unwrap_object(space, w_obj)
+        tbuf = lltype.malloc(self.c_ptrtype.TO, rffi.sizeof(self.c_type), flavor='raw')
+        tbuf[0] = obj
+        vbuf = rffi.cast(rffi.VOIDP, tbuf)
+        argchain.arg(vbuf)
+        return vbuf
+
 class IntTypeConverterMixin(NumericTypeConverterMixin):
     _mixin_ = True
     _immutable_ = True
 
     def convert_argument(self, space, w_obj, address):
-        x = rffi.cast(self.rffiptype, address)
+        x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
 
 class FloatTypeConverterMixin(NumericTypeConverterMixin):
@@ -177,7 +190,7 @@
     _immutable_ = True
 
     def convert_argument(self, space, w_obj, address):
-        x = rffi.cast(self.rffiptype, address)
+        x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset()] = self.typecode
@@ -212,6 +225,7 @@
 
     def convert_argument_libffi(self, space, w_obj, argchain):
         argchain.arg(self._unwrap_object(space, w_obj))
+        return lltype.nullptr(rffi.VOIDP.TO)
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
@@ -254,6 +268,7 @@
 
     def convert_argument_libffi(self, space, w_obj, argchain): 
         argchain.arg(self._unwrap_object(space, w_obj))
+        return lltype.nullptr(rffi.VOIDP.TO)
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(rffi.CCHARP, self._get_raw_address(space, w_obj, offset))
@@ -267,7 +282,8 @@
 class ShortConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.sshort
-    rffiptype = rffi.SHORTP
+    c_type     = rffi.SHORT
+    c_ptrtype  = rffi.SHORTP
 
     def __init__(self, space, default):
         self.default = rffi.cast(rffi.SHORT, capi.c_strtoll(default))
@@ -275,73 +291,77 @@
     def _unwrap_object(self, space, w_obj):
         return rffi.cast(rffi.SHORT, space.int_w(w_obj))
 
-class ConstShortRefConverter(ShortConverter):
+class ConstShortRefConverter(ConstRefNumericTypeConverterMixin, ShortConverter):
     _immutable_ = True
-    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    libffitype = libffi.types.pointer
 
 class UnsignedShortConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.sshort
-    rffiptype = rffi.USHORTP
+    c_type     = rffi.USHORT
+    c_ptrtype  = rffi.USHORTP
 
     def __init__(self, space, default):
-        self.default = rffi.cast(rffi.USHORT, capi.c_strtoull(default))
+        self.default = rffi.cast(self.c_type, capi.c_strtoull(default))
 
     def _unwrap_object(self, space, w_obj):
-        return rffi.cast(rffi.USHORT, space.int_w(w_obj))
+        return rffi.cast(self.c_type, space.int_w(w_obj))
 
-class ConstUnsignedShortRefConverter(UnsignedShortConverter):
+class ConstUnsignedShortRefConverter(ConstRefNumericTypeConverterMixin, UnsignedShortConverter):
     _immutable_ = True
-    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    libffitype = libffi.types.pointer
 
 class IntConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.sint
-    rffiptype = rffi.INTP
+    c_type     = rffi.INT
+    c_ptrtype  = rffi.INTP
 
     def __init__(self, space, default):
-        self.default = rffi.cast(rffi.INT, capi.c_strtoll(default))
+        self.default = rffi.cast(self.c_type, capi.c_strtoll(default))
 
     def _unwrap_object(self, space, w_obj):
-        return rffi.cast(rffi.INT, space.c_int_w(w_obj))
+        return rffi.cast(self.c_type, space.c_int_w(w_obj))
 
-class ConstIntRefConverter(IntConverter):
+class ConstIntRefConverter(ConstRefNumericTypeConverterMixin, IntConverter):
     _immutable_ = True
-    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    libffitype = libffi.types.pointer
 
 class UnsignedIntConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.uint
-    rffiptype = rffi.UINTP
+    c_type     = rffi.UINT
+    c_ptrtype  = rffi.UINTP
 
     def __init__(self, space, default):
-        self.default = rffi.cast(rffi.UINT, capi.c_strtoull(default))
+        self.default = rffi.cast(self.c_type, capi.c_strtoull(default))
 
     def _unwrap_object(self, space, w_obj):
-        return rffi.cast(rffi.UINT, space.uint_w(w_obj))
+        return rffi.cast(self.c_type, space.uint_w(w_obj))
 
-class ConstUnsignedIntRefConverter(UnsignedIntConverter):
+class ConstUnsignedIntRefConverter(ConstRefNumericTypeConverterMixin, UnsignedIntConverter):
     _immutable_ = True
-    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    libffitype = libffi.types.pointer
 
 class LongConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.slong
-    rffiptype = rffi.LONGP
+    c_type     = rffi.LONG
+    c_ptrtype  = rffi.LONGP
 
     def __init__(self, space, default):
-        self.default = rffi.cast(rffi.LONG, capi.c_strtoll(default))
+        self.default = rffi.cast(self.c_type, capi.c_strtoll(default))
 
     def _unwrap_object(self, space, w_obj):
         return space.int_w(w_obj)
 
-class ConstLongRefConverter(LongConverter):
+class ConstLongRefConverter(ConstRefNumericTypeConverterMixin, LongConverter):
     _immutable_ = True
-    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    libffitype = libffi.types.pointer
     typecode = 'r'
 
     def convert_argument(self, space, w_obj, address):
-        x = rffi.cast(self.rffiptype, address)
+        x = rffi.cast(self.c_ptrtype, address)
         x[0] = self._unwrap_object(space, w_obj)
         ba = rffi.cast(rffi.CCHARP, address)
         ba[capi.c_function_arg_typeoffset()] = self.typecode
@@ -349,23 +369,25 @@
 class UnsignedLongConverter(IntTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.ulong
-    rffiptype = rffi.ULONGP
+    c_type     = rffi.ULONG
+    c_ptrtype  = rffi.ULONGP
 
     def __init__(self, space, default):
-        self.default = rffi.cast(rffi.ULONG, capi.c_strtoull(default))
+        self.default = rffi.cast(self.c_type, capi.c_strtoull(default))
 
     def _unwrap_object(self, space, w_obj):
         return space.uint_w(w_obj)
 
-class ConstUnsignedLongRefConverter(UnsignedLongConverter):
+class ConstUnsignedLongRefConverter(ConstRefNumericTypeConverterMixin, UnsignedLongConverter):
     _immutable_ = True
-    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    libffitype = libffi.types.pointer
 
 class FloatConverter(FloatTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.float
-    rffiptype = rffi.FLOATP
-    typecode = 'f'
+    c_type     = rffi.FLOAT
+    c_ptrtype  = rffi.FLOATP
+    typecode   = 'f'
 
     def __init__(self, space, default):
         if default:
@@ -379,32 +401,37 @@
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = self._get_raw_address(space, w_obj, offset)
-        rffiptr = rffi.cast(self.rffiptype, address)
+        rffiptr = rffi.cast(self.c_ptrtype, address)
         return space.wrap(float(rffiptr[0]))
 
 class ConstFloatRefConverter(FloatConverter):
     _immutable_ = True
-    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    libffitype = libffi.types.pointer
     typecode = 'F'
 
+    def convert_argument_libffi(self, space, w_obj, argchain):
+        from pypy.module.cppyy.interp_cppyy import FastCallNotPossible
+        raise FastCallNotPossible
+
 class DoubleConverter(FloatTypeConverterMixin, TypeConverter):
     _immutable_ = True
     libffitype = libffi.types.double
-    rffiptype = rffi.DOUBLEP
-    typecode = 'd'
+    c_type     = rffi.DOUBLE
+    c_ptrtype  = rffi.DOUBLEP
+    typecode   = 'd'
 
     def __init__(self, space, default):
         if default:
-            self.default = rffi.cast(rffi.DOUBLE, rfloat.rstring_to_float(default))
+            self.default = rffi.cast(self.c_type, rfloat.rstring_to_float(default))
         else:
-            self.default = rffi.cast(rffi.DOUBLE, 0.)
+            self.default = rffi.cast(self.c_type, 0.)
 
     def _unwrap_object(self, space, w_obj):
         return space.float_w(w_obj)
 
-class ConstDoubleRefConverter(DoubleConverter):
+class ConstDoubleRefConverter(ConstRefNumericTypeConverterMixin, DoubleConverter):
     _immutable_ = True
-    libffitype = lltype.nullptr(clibffi.FFI_TYPE_P.TO)
+    libffitype = libffi.types.pointer
     typecode = 'D'
 
 
@@ -438,6 +465,7 @@
 
     def convert_argument_libffi(self, space, w_obj, argchain):
         argchain.arg(get_rawobject(space, w_obj))
+        return lltype.nullptr(rffi.VOIDP.TO)
 
 
 class VoidPtrPtrConverter(TypeConverter):
@@ -572,6 +600,7 @@
 
     def convert_argument_libffi(self, space, w_obj, argchain):
         argchain.arg(self._unwrap_object(space, w_obj))
+        return lltype.nullptr(rffi.VOIDP.TO)
 
     def from_memory(self, space, w_obj, w_pycppclass, offset):
         address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
@@ -689,7 +718,7 @@
 _converters["char"]                     = CharConverter
 _converters["unsigned char"]            = CharConverter
 _converters["short int"]                = ShortConverter
-_converters["const short int&"]         = ConstIntRefConverter
+_converters["const short int&"]         = ConstShortRefConverter
 _converters["short"]                    = _converters["short int"]
 _converters["const short&"]             = _converters["const short int&"]
 _converters["unsigned short int"]       = UnsignedShortConverter
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -158,14 +158,21 @@
         argchain = libffi.ArgChain()
         argchain.arg(cppthis)
         i = len(self.arg_defs)
-        for i in range(len(args_w)):
-            conv = self.arg_converters[i]
-            w_arg = args_w[i]
-            conv.convert_argument_libffi(self.space, w_arg, argchain)
-        for j in range(i+1, len(self.arg_defs)):
-            conv = self.arg_converters[j]
-            conv.default_argument_libffi(self.space, argchain)
-        return self.executor.execute_libffi(self.space, self._libffifunc, argchain)
+        refbuffers = []
+        try:
+            for i in range(len(args_w)):
+                conv = self.arg_converters[i]
+                w_arg = args_w[i]
+                refbuf = conv.convert_argument_libffi(self.space, w_arg, argchain)
+                if refbuf:
+                    refbuffers.append(refbuf)
+            for j in range(i+1, len(self.arg_defs)):
+                conv = self.arg_converters[j]
+                conv.default_argument_libffi(self.space, argchain)
+            return self.executor.execute_libffi(self.space, self._libffifunc, argchain)
+        finally:
+            for refbuf in refbuffers:
+                lltype.free(refbuf, flavor='raw')
 
     def _setup(self, cppthis):
         self.arg_converters = [converter.get_converter(self.space, arg_type, arg_dflt)
diff --git a/pypy/module/cppyy/test/bench1.py b/pypy/module/cppyy/test/bench1.py
--- a/pypy/module/cppyy/test/bench1.py
+++ b/pypy/module/cppyy/test/bench1.py
@@ -72,6 +72,14 @@
             addDataToInt.call(instance, i)
         return i
 
+class CppyyInterpBench3(CppyyInterpBench1):
+    def __call__(self):
+        addDataToInt = self.cls.get_overload("addDataToIntConstRef")
+        instance = self.inst
+        for i in range(NNN):
+            addDataToInt.call(instance, i)
+        return i
+
 class CppyyPythonBench1(object):
     scale = 1
     def __init__(self):
@@ -121,6 +129,7 @@
     print "warming up ... "
     interp_bench1 = CppyyInterpBench1()
     interp_bench2 = CppyyInterpBench2()
+    interp_bench3 = CppyyInterpBench3()
     python_bench1 = CppyyPythonBench1()
     interp_bench1(); interp_bench2(); python_bench1()
 
@@ -130,6 +139,7 @@
     # test runs ...
     print_bench("cppyy interp", run_bench(interp_bench1))
     print_bench("... overload", run_bench(interp_bench2))
+    print_bench("... constref", run_bench(interp_bench3))
     print_bench("cppyy python", run_bench(python_bench1))
     stat, t_cintex = commands.getstatusoutput("python bench1.py --pycintex")
     print_bench("pycintex    ", float(t_cintex))
diff --git a/pypy/module/cppyy/test/example01.cxx b/pypy/module/cppyy/test/example01.cxx
--- a/pypy/module/cppyy/test/example01.cxx
+++ b/pypy/module/cppyy/test/example01.cxx
@@ -91,6 +91,10 @@
     return m_somedata + a;
 }
 
+int example01::addDataToIntConstRef(const int& a) {
+    return m_somedata + a;
+}
+
 int example01::overloadedAddDataToInt(int a, int b) {
    return m_somedata + a + b;
 }
diff --git a/pypy/module/cppyy/test/example01.h b/pypy/module/cppyy/test/example01.h
--- a/pypy/module/cppyy/test/example01.h
+++ b/pypy/module/cppyy/test/example01.h
@@ -39,6 +39,7 @@
 
 public:        // instance methods
     int addDataToInt(int a);
+    int addDataToIntConstRef(const int& a);
     int overloadedAddDataToInt(int a, int b);
     int overloadedAddDataToInt(int a);
     int overloadedAddDataToInt(int a, int b, int c);
diff --git a/pypy/module/cppyy/test/test_zjit.py b/pypy/module/cppyy/test/test_zjit.py
--- a/pypy/module/cppyy/test/test_zjit.py
+++ b/pypy/module/cppyy/test/test_zjit.py
@@ -1,3 +1,4 @@
+import py, os, sys
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.rlib.objectmodel import specialize, instantiate
 from pypy.rlib import rarithmetic, jit
@@ -6,6 +7,18 @@
 
 from pypy.module.cppyy import interp_cppyy, capi
 
+
+currpath = py.path.local(__file__).dirpath()
+test_dct = str(currpath.join("example01Dict.so"))
+
+def setup_module(mod):
+    if sys.platform == 'win32':
+        py.test.skip("win32 not supported so far")
+    err = os.system("cd '%s' && make example01Dict.so" % currpath)
+    if err:
+        raise OSError("'make' failed (see stderr)")
+
+
 class FakeBase(W_Root):
     typename = None
 
@@ -150,24 +163,22 @@
         return True
 
 class TestFastPathJIT(LLJitMixin):
-    def test_simple(self):
-        """Test fast path being taken for methods"""
-
+    def _run_zjit(self, method_name):
         if capi.identify() == 'CINT':   # CINT does not support fast path
             return
 
         space = FakeSpace()
-        drv = jit.JitDriver(greens=[], reds=["i", "inst", "addDataToInt"])
+        drv = jit.JitDriver(greens=[], reds=["i", "inst", "cppmethod"])
         def f():
             lib = interp_cppyy.load_dictionary(space, "./example01Dict.so")
             cls  = interp_cppyy.scope_byname(space, "example01")
             inst = cls.get_overload("example01").call(None, [FakeInt(0)])
-            addDataToInt = cls.get_overload("addDataToInt")
+            cppmethod = cls.get_overload(method_name)
             assert isinstance(inst, interp_cppyy.W_CPPInstance)
             i = 10
             while i > 0:
-                drv.jit_merge_point(inst=inst, addDataToInt=addDataToInt, i=i)
-                addDataToInt.call(inst, [FakeInt(i)])
+                drv.jit_merge_point(inst=inst, cppmethod=cppmethod, i=i)
+                cppmethod.call(inst, [FakeInt(i)])
                 i -= 1
             return 7
         f()
@@ -175,27 +186,17 @@
         result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
         self.check_jitcell_token_count(1)
 
-    def test_overload(self):
+    def test01_simple(self):
+        """Test fast path being taken for methods"""
+
+        self._run_zjit("addDataToInt")
+
+    def test02_overload(self):
         """Test fast path being taken for overloaded methods"""
 
-        if capi.identify() == 'CINT':   # CINT does not support fast path
-            return
+        self._run_zjit("overloadedAddDataToInt")
 
-        space = FakeSpace()
-        drv = jit.JitDriver(greens=[], reds=["i", "inst", "addDataToInt"])
-        def f():
-            lib = interp_cppyy.load_dictionary(space, "./example01Dict.so")
-            cls  = interp_cppyy.scope_byname(space, "example01")
-            inst = cls.get_overload("example01").call(None, [FakeInt(0)])
-            addDataToInt = cls.get_overload("overloadedAddDataToInt")
-            assert isinstance(inst, interp_cppyy.W_CPPInstance)
-            i = 10
-            while i > 0:
-                drv.jit_merge_point(inst=inst, addDataToInt=addDataToInt, i=i)
-                addDataToInt.call(inst, [FakeInt(i)])
-                i -= 1
-            return 7
-        f()
-        space = FakeSpace()
-        result = self.meta_interp(f, [], listops=True, backendopt=True, listcomp=True)
-        self.check_jitcell_token_count(1)
+    def test03_const_ref(self):
+        """Test fast path being taken for methods with const ref arguments"""
+
+        self._run_zjit("addDataToIntConstRef")

From noreply at buildbot.pypy.org  Fri Apr 13 06:36:32 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 13 Apr 2012 06:36:32 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: merge default into branch
Message-ID: <20120413043632.9239082F4E@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54324:2fc2d0b8a090
Date: 2012-04-12 21:36 -0700
http://bitbucket.org/pypy/pypy/changeset/2fc2d0b8a090/

Log:	merge default into branch

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -158,6 +158,12 @@
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
+Note that this difference might show up indirectly in some cases.  For
+example, a generator left pending in the middle is --- again ---
+garbage-collected later in PyPy than in CPython.  You can see the
+difference if the ``yield`` keyword it is suspended at is itself
+enclosed in a ``try:`` or a ``with:`` block.
+
 Using the default GC called ``minimark``, the built-in function ``id()``
 works like it does in CPython.  With other GCs it returns numbers that
 are not real addresses (because an object can move around several times)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -169,9 +169,11 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +188,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +226,26 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
+        if self.keywords is None:
+            self.keywords = keywords[:] # copy to make non-resizable
+            self.keywords_w = keywords_w[:]
+        else:
+            # looks quadratic, but the JIT should remove all of it nicely.
+            # Also, all the lists should be small
+            for key in keywords:
+                for otherkey in self.keywords:
+                    if otherkey == key:
+                        raise operationerrfmt(self.space.w_TypeError,
+                                              "got multiple values "
+                                              "for keyword argument "
+                                              "'%s'", key)
+            self.keywords = self.keywords + keywords
+            self.keywords_w = self.keywords_w + keywords_w
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +404,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+

From noreply at buildbot.pypy.org  Fri Apr 13 11:00:53 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 13 Apr 2012 11:00:53 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: de-tab backend
Message-ID: <20120413090053.35BFC82F4F@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54326:c2f86113b607
Date: 2012-04-13 07:58 +0000
http://bitbucket.org/pypy/pypy/changeset/c2f86113b607/

Log:	de-tab backend

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -929,7 +929,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         asm_llong_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
-	return fcond 
+        return fcond 
 
     def regalloc_emit_math(self, op, arglocs, fcond, regalloc):
         effectinfo = op.getdescr().get_extra_info()
diff --git a/pypy/jit/backend/arm/instruction_builder.py b/pypy/jit/backend/arm/instruction_builder.py
--- a/pypy/jit/backend/arm/instruction_builder.py
+++ b/pypy/jit/backend/arm/instruction_builder.py
@@ -365,10 +365,10 @@
         size = 0x3
         n |= size << 20
     def f(self, dd, dn, dm):
-	N = (dn >> 4) & 0x1
-	M = (dm >> 4) & 0x1
-	D = (dd >> 4) & 0x1
-	Q = 0 # we want doubleword regs
+        N = (dn >> 4) & 0x1
+        M = (dm >> 4) & 0x1
+        D = (dd >> 4) & 0x1
+        Q = 0 # we want doubleword regs
         instr = (n
                 | D << 22
                 | (dn & 0xf) << 16
@@ -377,7 +377,7 @@
                 | Q << 6
                 | M << 5
                 | (dm & 0xf))
-	
+        
         self.write32(instr)
     return f
 
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -1272,14 +1272,14 @@
         assert arg.is_vfp_reg()
         assert res.is_reg()
         self.mc.VCVT_float_to_int(r.vfp_ip.value, arg.value)
-	self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
+        self.mc.VMOV_rc(res.value, r.ip.value, r.vfp_ip.value)
         return fcond
 
     def emit_op_cast_int_to_float(self, op, arglocs, regalloc, fcond):
         arg, res = arglocs
         assert res.is_vfp_reg()
         assert arg.is_reg()
-	self.mc.MOV_ri(r.ip.value, 0)
+        self.mc.MOV_ri(r.ip.value, 0)
         self.mc.VMOV_cr(res.value, arg.value, r.ip.value)
         self.mc.VCVT_int_to_float(res.value, res.value)
         return fcond
@@ -1301,9 +1301,9 @@
     emit_op_convert_float_bytes_to_longlong = gen_emit_unary_float_op('float_bytes_to_longlong', 'VMOV_cc')
 
     def emit_op_read_timestamp(self, op, arglocs, regalloc, fcond):
-	tmp = arglocs[0]
+        tmp = arglocs[0]
         res = arglocs[1]
         self.mc.MRC(15, 0, tmp.value, 15, 12, 1)
-	self.mc.MOV_ri(r.ip.value, 0)
+        self.mc.MOV_ri(r.ip.value, 0)
         self.mc.VMOV_cr(res.value, tmp.value, r.ip.value)
         return fcond
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -378,7 +378,7 @@
         return self.assembler.regalloc_emit_llong(op, args, fcond, self)
     
     def perform_math(self, op, args, fcond):
-	return self.assembler.regalloc_emit_math(op, args, self, fcond)
+        return self.assembler.regalloc_emit_math(op, args, self, fcond)
 
     def force_spill_var(self, var):
         if var.type == FLOAT:
@@ -548,16 +548,16 @@
         if effectinfo is not None:
             oopspecindex = effectinfo.oopspecindex
             if oopspecindex in (EffectInfo.OS_LLONG_ADD,
-            		    EffectInfo.OS_LLONG_SUB,
-            		    EffectInfo.OS_LLONG_AND,
-            		    EffectInfo.OS_LLONG_OR,
-            		    EffectInfo.OS_LLONG_XOR):
+                            EffectInfo.OS_LLONG_SUB,
+                            EffectInfo.OS_LLONG_AND,
+                            EffectInfo.OS_LLONG_OR,
+                            EffectInfo.OS_LLONG_XOR):
                 args = self._prepare_llong_binop_xx(op, fcond)
-		self.perform_llong(op, args, fcond)
+                self.perform_llong(op, args, fcond)
                 return
             if oopspecindex == EffectInfo.OS_LLONG_TO_INT:
                 args = self._prepare_llong_to_int(op, fcond)
-		self.perform_llong(op, args, fcond)
+                self.perform_llong(op, args, fcond)
                 return
             if oopspecindex == EffectInfo.OS_MATH_SQRT:
                 args = self.prepare_op_math_sqrt(op, fcond)
@@ -587,9 +587,9 @@
         return self._prepare_call(op)
 
     def _prepare_llong_binop_xx(self, op, fcond):
-	# arg 0 is the address of the function
+        # arg 0 is the address of the function
         loc0 = self._ensure_value_is_boxed(op.getarg(1))
-	loc1 = self._ensure_value_is_boxed(op.getarg(2))
+        loc1 = self._ensure_value_is_boxed(op.getarg(2))
         self.possibly_free_vars_for_op(op)
         self.free_temp_vars()
         res = self.vfprm.force_allocate_reg(op.result)
@@ -1205,7 +1205,7 @@
         return []
 
     prepare_op_convert_float_bytes_to_longlong = prepare_float_op(base=False,
-			    name='prepare_op_convert_float_bytes_to_longlong')
+                              name='prepare_op_convert_float_bytes_to_longlong')
 
     def prepare_op_read_timestamp(self, op, fcond):
         loc = self.get_scratch_reg(INT)

From noreply at buildbot.pypy.org  Fri Apr 13 11:00:51 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 13 Apr 2012 11:00:51 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: merge default
Message-ID: <20120413090051.96AAA82F4E@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54325:308075361821
Date: 2012-04-10 13:16 +0000
http://bitbucket.org/pypy/pypy/changeset/308075361821/

Log:	merge default

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return pyc
+def pytest_unconfigure(config):
+    hook = config._assertstate.hook
+    if hook is not None:
+        sys.meta_path.remove(hook)
 
-def before_module_import(mod):
-    if mod.config._assertstate.mode != "on":
-        return
-    # Some deep magic: load the source, rewrite the asserts, and write a
-    # fake pyc, so that it'll be loaded when the module is imported.
-    source = mod.fspath.read()
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
-        return
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, str(mod.fspath), "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the assertion
-        # rewriting, but I don't know of a fast way to tell.
-        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
-        return
-    mod._pyc = _write_pyc(co, mod.fspath)
-    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+def pytest_collection(session):
+    # this hook is only called when test modules are collected
+    # so for example not in the master process of pytest-xdist
+    # (which does not collect test modules)
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.set_session(session)
 
-def after_module_import(mod):
-    if not hasattr(mod, "_pyc"):
-        return
-    state = mod.config._assertstate
-    try:
-        mod._pyc.remove()
-    except py.error.ENOENT:
-        state.trace("couldn't find pyc: %r" % (mod._pyc,))
-    else:
-        state.trace("removed pyc: %r" % (mod._pyc,))
+def pytest_runtest_setup(item):
+    def callbinrepr(op, left, right):
+        hook_result = item.ihook.pytest_assertrepr_compare(
+            config=item.config, op=op, left=left, right=right)
+        for new_expl in hook_result:
+            if new_expl:
+                res = '\n~'.join(new_expl)
+                if item.config.getvalue("assertmode") == "rewrite":
+                    # The result will be fed back a python % formatting
+                    # operation, which will fail if there are extraneous
+                    # '%'s in the string. Escape them here.
+                    res = res.replace("%", "%%")
+                return res
+    util._reprcompare = callbinrepr
 
-def warn_about_missing_assertion():
+def pytest_runtest_teardown(item):
+    util._reprcompare = None
+
+def pytest_sessionfinish(session):
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.session = None
+
+def _load_modules(mode):
+    """Lazily import assertion related code."""
+    global rewrite, reinterpret
+    from _pytest.assertion import reinterpret
+    if mode == "rewrite":
+        from _pytest.assertion import rewrite
+
+def warn_about_missing_assertion(mode):
     try:
         assert False
     except AssertionError:
         pass
     else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
+        if mode == "rewrite":
+            specifically = ("assertions which are not in test modules "
+                            "will be ignored")
+        else:
+            specifically = "failing tests may report as passing"
+
+        sys.stderr.write("WARNING: " + specifically +
+                        " because assert statements are not executed "
+                        "by the underlying Python interpreter "
+                        "(are you using python -O?)\n")
 
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -53,7 +53,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
+                "compute assert expression before the assert or use --assert=plain)")
 
 def run(offending_line, frame=None):
     if frame is None:
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -482,7 +482,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --nomagic)")
+                "compute assert expression before the assert or use --assert=plain)")
     else:
         return None
 
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,14 +1,258 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
+import errno
 import itertools
+import imp
+import marshal
+import os
+import struct
 import sys
+import types
 
 import py
 from _pytest.assertion import util
 
 
+# Windows gives ENOENT in places *nix gives ENOTDIR.
+if sys.platform.startswith("win"):
+    PATH_COMPONENT_NOT_DIR = errno.ENOENT
+else:
+    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
+
+# py.test caches rewritten pycs in __pycache__.
+if hasattr(imp, "get_tag"):
+    PYTEST_TAG = imp.get_tag() + "-PYTEST"
+else:
+    if hasattr(sys, "pypy_version_info"):
+        impl = "pypy"
+    elif sys.platform == "java":
+        impl = "jython"
+    else:
+        impl = "cpython"
+    ver = sys.version_info
+    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
+    del ver, impl
+
+PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
+
+REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+
+class AssertionRewritingHook(object):
+    """Import hook which rewrites asserts."""
+
+    def __init__(self):
+        self.session = None
+        self.modules = {}
+
+    def set_session(self, session):
+        self.fnpats = session.config.getini("python_files")
+        self.session = session
+
+    def find_module(self, name, path=None):
+        if self.session is None:
+            return None
+        sess = self.session
+        state = sess.config._assertstate
+        state.trace("find_module called for: %s" % name)
+        names = name.rsplit(".", 1)
+        lastname = names[-1]
+        pth = None
+        if path is not None and len(path) == 1:
+            pth = path[0]
+        if pth is None:
+            try:
+                fd, fn, desc = imp.find_module(lastname, path)
+            except ImportError:
+                return None
+            if fd is not None:
+                fd.close()
+            tp = desc[2]
+            if tp == imp.PY_COMPILED:
+                if hasattr(imp, "source_from_cache"):
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
+            elif tp != imp.PY_SOURCE:
+                # Don't know what this is.
+                return None
+        else:
+            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
+        fn_pypath = py.path.local(fn)
+        # Is this a test file?
+        if not sess.isinitpath(fn):
+            # We have to be very careful here because imports in this code can
+            # trigger a cycle.
+            self.session = None
+            try:
+                for pat in self.fnpats:
+                    if fn_pypath.fnmatch(pat):
+                        state.trace("matched test file %r" % (fn,))
+                        break
+                else:
+                    return None
+            finally:
+                self.session = sess
+        else:
+            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+        # The requested module looks like a test file, so rewrite it. This is
+        # the most magical part of the process: load the source, rewrite the
+        # asserts, and load the rewritten source. We also cache the rewritten
+        # module code in a special pyc. We must be aware of the possibility of
+        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # tricky race conditions, we maintain the following invariant: The
+        # cached pyc is always a complete, valid pyc. Operations on it must be
+        # atomic. POSIX's atomic rename comes in handy.
+        write = not sys.dont_write_bytecode
+        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
+        if write:
+            try:
+                os.mkdir(cache_dir)
+            except OSError:
+                e = sys.exc_info()[1].errno
+                if e == errno.EEXIST:
+                    # Either the __pycache__ directory already exists (the
+                    # common case) or it's blocked by a non-dir node. In the
+                    # latter case, we'll ignore it in _write_pyc.
+                    pass
+                elif e == PATH_COMPONENT_NOT_DIR:
+                    # One of the path components was not a directory, likely
+                    # because we're in a zip file.
+                    write = False
+                elif e == errno.EACCES:
+                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    write = False
+                else:
+                    raise
+        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
+        pyc = os.path.join(cache_dir, cache_name)
+        # Notice that even if we're in a read-only directory, I'm going to check
+        # for a cached pyc. This may not be optimal...
+        co = _read_pyc(fn_pypath, pyc)
+        if co is None:
+            state.trace("rewriting %r" % (fn,))
+            co = _rewrite_test(state, fn_pypath)
+            if co is None:
+                # Probably a SyntaxError in the test.
+                return None
+            if write:
+                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+        else:
+            state.trace("found cached rewritten pyc for %r" % (fn,))
+        self.modules[name] = co, pyc
+        return self
+
+    def load_module(self, name):
+        co, pyc = self.modules.pop(name)
+        # I wish I could just call imp.load_compiled here, but __file__ has to
+        # be set properly. In Python 3.2+, this all would be handled correctly
+        # by load_compiled.
+        mod = sys.modules[name] = imp.new_module(name)
+        try:
+            mod.__file__ = co.co_filename
+            # Normally, this attribute is 3.2+.
+            mod.__cached__ = pyc
+            py.builtin.exec_(co, mod.__dict__)
+        except:
+            del sys.modules[name]
+            raise
+        return sys.modules[name]
+
+def _write_pyc(co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as (C)Python, since
+    # these "pycs" should never be seen by builtin import. However, there's
+    # little reason deviate, and I hope sometime to be able to use
+    # imp.load_compiled to load them. (See the comment in load_module above.)
+    mtime = int(source_path.mtime())
+    try:
+        fp = open(pyc, "wb")
+    except IOError:
+        err = sys.exc_info()[1].errno
+        if err == PATH_COMPONENT_NOT_DIR:
+            # This happens when we get a EEXIST in find_module creating the
+            # __pycache__ directory and __pycache__ is by some non-dir node.
+            return False
+        raise
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return True
+
+RN = "\r\n".encode("utf-8")
+N = "\n".encode("utf-8")
+
+def _rewrite_test(state, fn):
+    """Try to read and rewrite *fn* and return the code object."""
+    try:
+        source = fn.read("rb")
+    except EnvironmentError:
+        return None
+    # On Python versions which are not 2.7 and less than or equal to 3.1, the
+    # parser expects *nix newlines.
+    if REWRITE_NEWLINES:
+        source = source.replace(RN, N) + N
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        state.trace("failed to parse: %r" % (fn,))
+        return None
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, fn.strpath, "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the
+        # assertion rewriting, but I don't know of a fast way to tell.
+        state.trace("failed to compile: %r" % (fn,))
+        return None
+    return co
+
+def _make_rewritten_pyc(state, fn, pyc, co):
+    """Try to dump rewritten code to *pyc*."""
+    if sys.platform.startswith("win"):
+        # Windows grants exclusive access to open files and doesn't have atomic
+        # rename, so just write into the final file.
+        _write_pyc(co, fn, pyc)
+    else:
+        # When not on windows, assume rename is atomic. Dump the code object
+        # into a file specific to this process and atomically replace it.
+        proc_pyc = pyc + "." + str(os.getpid())
+        if _write_pyc(co, fn, proc_pyc):
+            os.rename(proc_pyc, pyc)
+
+def _read_pyc(source, pyc):
+    """Possibly read a py.test pyc containing rewritten code.
+
+    Return rewritten code if successful or None if not.
+    """
+    try:
+        fp = open(pyc, "rb")
+    except IOError:
+        return None
+    try:
+        try:
+            mtime = int(source.mtime())
+            data = fp.read(8)
+        except EnvironmentError:
+            return None
+        # Check for invalid or out of date pyc file.
+        if (len(data) != 8 or
+            data[:4] != imp.get_magic() or
+            struct.unpack("<l", data[4:])[0] != mtime):
+            return None
+        co = marshal.load(fp)
+        if not isinstance(co, types.CodeType):
+            # That's interesting....
+            return None
+        return co
+    finally:
+        fp.close()
+
+
 def rewrite_asserts(mod):
     """Rewrite the assert statements in mod."""
     AssertionRewriter().run(mod)
@@ -17,13 +261,8 @@
 _saferepr = py.io.saferepr
 from _pytest.assertion.util import format_explanation as _format_explanation
 
-def _format_boolop(operands, explanations, is_or):
-    show_explanations = []
-    for operand, expl in zip(operands, explanations):
-        show_explanations.append(expl)
-        if operand == is_or:
-            break
-    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+def _format_boolop(explanations, is_or):
+    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -109,8 +348,8 @@
                     return
                 lineno += len(doc) - 1
                 expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +357,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +382,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +437,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +460,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +480,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +545,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,21 +25,39 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +86,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("</testcase>")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="xfail-marked test passes unexpectedly"/>')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('<failure message="test failure">%s</failure>',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<failure message="collection failure">%s</failure>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<skipped message="collection skipped">%s</skipped>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('<error message="test setup failure">%s</error>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="expected test failure">%s</skipped>',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('<skipped type="pytest.skip" '
-                           'message="%s">%s</skipped>',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +185,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n<testcase classname="pytest" name="internal">'
-            '    <error message="internal error">'
-            '%s</error></testcase>' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +203,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write('<testsuite ')
-        logfile.write('name="" ')
-        logfile.write('errors="%i" ' % self.errors)
-        logfile.write('failures="%i" ' % self.failed)
-        logfile.write('skips="%i" ' % self.skipped)
-        logfile.write('tests="%i" ' % numtests)
-        logfile.write('time="%.3f"' % suite_time_delta)
-        logfile.write(' >')
-        logfile.writelines(self.test_logs)
-        logfile.write('</testsuite>')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -399,6 +410,7 @@
         self._notfound = []
         self._initialpaths = set()
         self._initialparts = []
+        self.items = items = []
         for arg in args:
             parts = self._parsearg(arg)
             self._initialparts.append(parts)
@@ -414,7 +426,6 @@
         if not genitems:
             return rep.result
         else:
-            self.items = items = []
             if rep.passed:
                 for node in rep.result:
                     self.items.extend(self.genitems(node))
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "<CallSpec id=%r param=%r funcargs=%r>" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,71 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        if not indirect:
+            #XXX should we also check for the opposite case?
+            for arg in argnames:
+                if arg not in self.funcargnames:
+                    raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +678,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +769,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +781,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +830,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +956,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         <ExceptionInfo ...>
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         <ExceptionInfo ...>
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,60 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
-
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
+def pytest_sessionfinish(session):
+    session._setupstate.teardown_all()
 
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +76,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +93,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +114,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +143,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +166,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +188,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +208,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "<TestReport %r when=%r outcome=%r>" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +234,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +259,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +319,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -117,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
@@ -169,21 +192,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -289,10 +282,18 @@
         # we take care to leave out Instances aka ()
         # because later versions are going to get rid of them anyway
         if self.config.option.verbose < 0:
-            for item in items:
-                nodeid = item.nodeid
-                nodeid = nodeid.replace("::()::", "::")
-                self._tw.line(nodeid)
+            if self.config.option.verbose < -1:
+                counts = {}
+                for item in items:
+                    name = item.nodeid.split('::', 1)[0]
+                    counts[name] = counts.get(name, 0) + 1
+                for name, count in sorted(counts.items()):
+                    self._tw.line("%s: %d" % (name, count))
+            else:
+                for item in items:
+                    nodeid = item.nodeid
+                    nodeid = nodeid.replace("::()::", "::")
+                    self._tw.line(nodeid)
             return
         stack = []
         indent = ""
@@ -318,12 +319,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +394,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +412,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +431,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +445,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -83,28 +91,37 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +137,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -968,8 +968,7 @@
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
+        return NotImplemented    # note that this doesn't work on CPython 2.2
 
     __radd__ = __add__
 
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.4.dev1'
+__version__ = '1.4.7'
 
 from py import _apipkg
 
@@ -70,6 +70,11 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
+        '_AssertionError'   : '._code.assertion:AssertionError',
+        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
+        '_reinterpret'      : '._code.assertion:reinterpret',
+        '_reprcompare'      : '._code.assertion:_reprcompare',
+        '_format_explanation' : '._code.assertion:_format_explanation',
     },
 
     # backports and additions of builtins
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -113,9 +113,12 @@
 
     # some backward compatibility helpers
     _basestring = str
-    def _totext(obj, encoding=None):
+    def _totext(obj, encoding=None, errors=None):
         if isinstance(obj, bytes):
-            obj = obj.decode(encoding)
+            if errors is None:
+                obj = obj.decode(encoding)
+            else:
+                obj = obj.decode(encoding, errors)
         elif not isinstance(obj, str):
             obj = str(obj)
         return obj
@@ -142,7 +145,7 @@
             del back
         elif locs is None:
             locs = globs
-        fp = open(fn, "rb")
+        fp = open(fn, "r")
         try:
             source = fp.read()
         finally:
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionnew.py
@@ -0,0 +1,339 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace _assertionold.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from py._code.assertion import _format_explanation, BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(failure):
+    explanation = _format_explanation(failure.explanation)
+    value = failure.cause[1]
+    if str(value):
+        lines = explanation.splitlines()
+        if not lines:
+            lines.append("")
+        lines[0] += " << %s" % (value,)
+        explanation = "\n".join(lines)
+    text = "%s: %s" % (failure.cause[0].__name__, explanation)
+    if text.startswith("AssertionError: assert "):
+        text = text[16:]
+    return text
+
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = False
+        if not local:
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not result:
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        rcomp = py.code._reprcompare
+        if rcomp:
+            res = rcomp(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = True
+        if from_instance:
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        if test_explanation.startswith("False\n{False =") and \
+                test_explanation.endswith("\n"):
+            test_explanation = test_explanation[15:-2]
+        explanation = "assert %s" % (test_explanation,)
+        if not test_result:
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionold.py
@@ -0,0 +1,555 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from py._code.assertion import BuiltinAssertionError, _format_explanation
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = '<eval>'
+            self.__obj__.filename = '<eval>'
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = '<run>'
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return _format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # simplify 'assert False where False = ...'
+        if (test.explanation.startswith('False\n{False = ') and
+            test.explanation.endswith('\n}')):
+            test.explanation = test.explanation[15:-2]
+        # print the result as  'assert <explanation>'
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = '<run>'
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
new file mode 100644
--- /dev/null
+++ b/py/_code/assertion.py
@@ -0,0 +1,94 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+_reprcompare = None # if set, will be called by assert reinterp for comparison ops
+
+def _format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "<run>".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = "<could not determine information>"
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from py._code._assertionold import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from py._code._assertionnew import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,6 +145,17 @@
         return self.frame.f_locals
     locals = property(getlocals, None, None, "locals of underlaying frame")
 
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        if self.exprinfo is None:
+            source = str(self.statement).strip()
+            x = py.code._reinterpret(source, self.frame, should_fail=True)
+            if not isinstance(x, str):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
     def getfirstlinesource(self):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
@@ -158,13 +169,12 @@
         end = self.lineno
         try:
             _, end = source.getstatementrange(end)
-        except IndexError:
+        except (IndexError, ValueError):
             end = self.lineno + 1
         # heuristic to stop displaying source on e.g.
         #   if something:  # assume this causes a NameError
         #      # _this_ lines and the one
                #        below we don't want from entry.getsource()
-        end = min(end, len(source))
         for i in range(self.lineno, end):
             if source[i].rstrip().endswith(':'):
                 end = i + 1
@@ -273,7 +283,11 @@
         """
         cache = {}
         for i, entry in enumerate(self):
-            key = entry.frame.code.path, entry.lineno
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
             #print "checking for recursion at", key
             l = cache.setdefault(key, [])
             if l:
@@ -308,7 +322,7 @@
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
         self.type, self.value, tb = self._excinfo
-        self.typename = getattr(self.type, "__name__", "???")
+        self.typename = self.type.__name__
         self.traceback = py.code.Traceback(tb)
 
     def __repr__(self):
@@ -347,14 +361,16 @@
             showlocals: show locals per traceback entry
             style: long|short|no|native traceback style
             tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
         """
         if style == 'native':
-            import traceback
-            return ''.join(traceback.format_exception(
-                self.type,
-                self.value,
-                self.traceback[0]._rawentry,
-                ))
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
 
         fmt = FormattedExcinfo(showlocals=showlocals, style=style,
             abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
@@ -452,7 +468,7 @@
     def repr_locals(self, locals):
         if self.showlocals:
             lines = []
-            keys = list(locals)
+            keys = [loc for loc in locals if loc[0] != "@"]
             keys.sort()
             for name in keys:
                 value = locals[name]
@@ -506,7 +522,10 @@
 
     def _makepath(self, path):
         if not self.abspath:
-            np = py.path.local().bestrelpath(path)
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
             if len(np) < len(str(path)):
                 path = np
         return path
@@ -595,6 +614,19 @@
         if self.extraline:
             tw.line(self.extraline)
 
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
 class ReprEntry(TerminalRepr):
     localssep = "_ "
 
@@ -680,19 +712,26 @@
 
 oldbuiltins = {}
 
-def patch_builtins(compile=True):
-    """ put compile builtins to Python's builtins. """
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from py._code import assertion
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = assertion.AssertionError
     if compile:
         l = oldbuiltins.setdefault('compile', [])
         l.append(py.builtin.builtins.compile)
         py.builtin.builtins.compile = py.code.compile
 
-def unpatch_builtins(compile=True):
+def unpatch_builtins(assertion=True, compile=True):
     """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
     if compile:
         py.builtin.builtins.compile = oldbuiltins['compile'].pop()
 
-def getrawcode(obj):
+def getrawcode(obj, trycall=True):
     """ return code object for given function. """
     try:
         return obj.__code__
@@ -701,5 +740,10 @@
         obj = getattr(obj, 'func_code', obj)
         obj = getattr(obj, 'f_code', obj)
         obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
         return obj
 
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,6 +108,7 @@
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
+            raise an IndexError if no such statementrange can be found.
         """
         # XXX there must be a better than these heuristic ways ...
         # XXX there may even be better heuristics :-)
@@ -116,6 +117,7 @@
 
         # 1. find the start of the statement
         from codeop import compile_command
+        end = None
         for start in range(lineno, -1, -1):
             if assertion:
                 line = self.lines[start]
@@ -139,7 +141,9 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, len(self)
+        if end is None:
+            raise IndexError("no valid source range around line %d " % (lineno,))
+        return start, end
 
     def getblockend(self, lineno):
         # XXX
@@ -257,23 +261,29 @@
 
 
 def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
     try:
         code = py.code.Code(obj)
     except TypeError:
-        # fallback to
-        fn = (py.std.inspect.getsourcefile(obj) or
-              py.std.inspect.getfile(obj))
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
         fspath = fn and py.path.local(fn) or None
+        lineno = -1
         if fspath:
             try:
                 _, lineno = findsource(obj)
             except IOError:
-                lineno = None
-        else:
-            lineno = None
+                pass
     else:
         fspath = code.path
         lineno = code.firstlineno
+    assert isinstance(lineno, int)
     return fspath, lineno
 
 #
@@ -286,7 +296,7 @@
     except py.builtin._sysex:
         raise
     except:
-        return None, None
+        return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
diff --git a/py/_error.py b/py/_error.py
--- a/py/_error.py
+++ b/py/_error.py
@@ -23,6 +23,7 @@
     2: errno.ENOENT,
     3: errno.ENOENT,
     17: errno.EEXIST,
+    13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -103,6 +103,7 @@
     def _parseline(self, line, lineno):
         # comments
         line = line.split('#')[0].rstrip()
+        line = line.split(';')[0].rstrip()
         # blank lines
         if not line:
             return None, None
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -12,7 +12,7 @@
     class TextIO(StringIO):
         def write(self, data):
             if not isinstance(data, unicode):
-                data = unicode(data, getattr(self, '_encoding', 'UTF-8'))
+                data = unicode(data, getattr(self, '_encoding', 'UTF-8'), 'replace')
             StringIO.write(self, data)
 else:
     TextIO = StringIO
@@ -258,6 +258,9 @@
                 f = getattr(self, name).tmpfile
                 f.seek(0)
                 res = f.read()
+                enc = getattr(f, 'encoding', None)
+                if enc:
+                    res = py.builtin._totext(res, enc, 'replace')
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -105,6 +105,8 @@
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
+    _newline = None   # the last line printed
+
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -112,11 +114,9 @@
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-                if hasattr(file, 'encoding'):
-                    encoding = file.encoding
         elif hasattr(file, '__call__'):
             file = WriteFile(file, encoding=encoding)
-        self.encoding = encoding
+        self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
@@ -182,8 +182,31 @@
         return s
 
     def line(self, s='', **kw):
+        if self._newline == False:
+            self.write("\n")
         self.write(s, **kw)
         self.write('\n')
+        self._newline = True
+
+    def reline(self, line, **opts):
+        if not self.hasmarkup:
+            raise ValueError("cannot use rewrite-line without terminal")
+        if not self._newline:
+            self.write("\r")
+        self.write(line, **opts)
+        # see if we need to fill up some spaces at the end
+        # xxx have a more exact lastlinelen working from self.write?
+        lenline = len(line)
+        try:
+            lastlen = self._lastlinelen
+        except AttributeError:
+            pass
+        else:
+            if lenline < lastlen:
+                self.write(" " * (lastlen - lenline + 1))
+        self._lastlinelen = lenline
+        self._newline = False
+
 
 class Win32ConsoleWriter(TerminalWriter):
     def write(self, s, **kw):
@@ -280,10 +303,10 @@
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
     SetConsoleTextAttribute.argtypes = [wintypes.HANDLE, wintypes.WORD]
     SetConsoleTextAttribute.restype = wintypes.BOOL
-        
+
     _GetConsoleScreenBufferInfo = \
         ctypes.windll.kernel32.GetConsoleScreenBufferInfo
-    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, 
+    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE,
                                 ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
     _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
     def GetConsoleInfo(handle):
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -64,7 +64,10 @@
                 else:
                     if bool(value) ^ bool(meth()) ^ invert:
                         return False
-            except (py.error.ENOENT, py.error.ENOTDIR):
+            except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
+                # EBUSY feels not entirely correct,
+                # but its kind of necessary since ENOMEDIUM
+                # is not accessible in python     
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
@@ -368,6 +371,5 @@
         else:
             name = str(path) # path.strpath # XXX svn?
             pattern = '*' + path.sep + pattern
-        from fnmatch import fnmatch
-        return fnmatch(name, pattern)
+        return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -157,14 +157,16 @@
         return str(self) < str(other)
 
     def samefile(self, other):
-        """ return True if 'other' references the same file as 'self'. """
-        if not iswin32:
-            return py.error.checked_call(
-                    os.path.samefile, str(self), str(other))
+        """ return True if 'other' references the same file as 'self'.
+        """
+        if not isinstance(other, py.path.local):
+            other = os.path.abspath(str(other))
         if self == other:
             return True
-        other = os.path.abspath(str(other))
-        return self == other
+        if iswin32:
+            return False # ther is no samefile
+        return py.error.checked_call(
+                os.path.samefile, str(self), str(other))
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -539,7 +541,11 @@
                 if self.basename != "__init__.py":
                     modfile = modfile[:-12]
 
-            if not self.samefile(modfile):
+            try:
+                issame = self.samefile(modfile)
+            except py.error.ENOENT:
+                issame = False
+            if not issame:
                 raise self.ImportMismatchError(modname, modfile, self)
             return mod
         else:
diff --git a/py/_path/svnurl.py b/py/_path/svnurl.py
--- a/py/_path/svnurl.py
+++ b/py/_path/svnurl.py
@@ -233,6 +233,8 @@
                 e = sys.exc_info()[1]
                 if e.err.find('non-existent in that revision') != -1:
                     raise py.error.ENOENT(self, e.err)
+                elif e.err.find("E200009:") != -1:
+                    raise py.error.ENOENT(self, e.err)
                 elif e.err.find('File not found') != -1:
                     raise py.error.ENOENT(self, e.err)
                 elif e.err.find('not part of a repository')!=-1:
diff --git a/py/_path/svnwc.py b/py/_path/svnwc.py
--- a/py/_path/svnwc.py
+++ b/py/_path/svnwc.py
@@ -482,10 +482,13 @@
         except py.process.cmdexec.Error:
             e = sys.exc_info()[1]
             strerr = e.err.lower()
-            if strerr.find('file not found') != -1:
+            if strerr.find('not found') != -1:
+                raise py.error.ENOENT(self)
+            elif strerr.find("E200009:") != -1:
                 raise py.error.ENOENT(self)
             if (strerr.find('file exists') != -1 or
                 strerr.find('file already exists') != -1 or
+                strerr.find('w150002:') != -1 or
                 strerr.find("can't create directory") != -1):
                 raise py.error.EEXIST(self)
             raise
@@ -593,7 +596,7 @@
         out = self._authsvn('lock').strip()
         if not out:
             # warning or error, raise exception
-            raise Exception(out[4:])
+            raise ValueError("unknown error in svn lock command")
 
     def unlock(self):
         """ unset a previously set lock """
@@ -1066,6 +1069,8 @@
                 modrev = '?'
                 author = '?'
                 date = ''
+            elif itemstatus == "replaced":
+                pass
             else:
                 #print entryel.toxml()
                 commitel = entryel.getElementsByTagName('commit')[0]
@@ -1148,7 +1153,11 @@
             raise  ValueError("Not a versioned resource")
             #raise ValueError, "Not a versioned resource %r" % path
         self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind']
-        self.rev = int(d['revision'])
+        try:
+            self.rev = int(d['revision'])
+        except KeyError:
+            self.rev = None
+
         self.path = py.path.local(d['path'])
         self.size = self.path.size()
         if 'lastchangedrev' in d:
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -52,7 +52,7 @@
     def unicode(self, indent=2):
         l = []
         SimpleUnicodeVisitor(l.append, indent).visit(self)
-        return "".join(l)
+        return u("").join(l)
 
     def __repr__(self):
         name = self.__class__.__name__
@@ -122,11 +122,13 @@
                 if visitmethod is not None:
                     break
             else:
-                visitmethod = self.object
+                visitmethod = self.__object
             self.cache[cls] = visitmethod
         visitmethod(node)
 
-    def object(self, obj):
+    # the default fallback handler is marked private
+    # to avoid clashes with the tag name object
+    def __object(self, obj):
         #self.write(obj)
         self.write(escape(unicode(obj)))
 
@@ -136,7 +138,8 @@
     def list(self, obj):
         assert id(obj) not in self.visited
         self.visited[id(obj)] = 1
-        map(self.visit, obj)
+        for elem in obj:
+            self.visit(elem)
 
     def Tag(self, tag):
         assert id(tag) not in self.visited
@@ -181,7 +184,11 @@
             value = getattr(attrs, name)
             if name.endswith('_'):
                 name = name[:-1]
-            return ' %s="%s"' % (name, escape(unicode(value)))
+            if isinstance(value, raw):
+                insert = value.uniobj
+            else:
+                insert = escape(unicode(value))
+            return ' %s="%s"' % (name, insert)
 
     def getstyle(self, tag):
         """ return attribute list suitable for styling. """
diff --git a/py/bin/_findpy.py b/py/bin/_findpy.py
deleted file mode 100644
--- a/py/bin/_findpy.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-#
-# find and import a version of 'py'
-#
-import sys
-import os
-from os.path import dirname as opd, exists, join, basename, abspath
-
-def searchpy(current):
-    while 1:
-        last = current
-        initpy = join(current, '__init__.py')
-        if not exists(initpy):
-            pydir = join(current, 'py')
-            # recognize py-package and ensure it is importable
-            if exists(pydir) and exists(join(pydir, '__init__.py')):
-                #for p in sys.path:
-                #    if p == current:
-                #        return True
-                if current != sys.path[0]:  # if we are already first, then ok
-                    sys.stderr.write("inserting into sys.path: %s\n" % current)
-                    sys.path.insert(0, current)
-                return True
-        current = opd(current)
-        if last == current:
-            return False
-
-if not searchpy(abspath(os.curdir)):
-    if not searchpy(opd(abspath(sys.argv[0]))):
-        if not searchpy(opd(__file__)):
-            pass # let's hope it is just on sys.path
-
-import py
-import pytest
-
-if __name__ == '__main__':
-    print ("py lib is at %s" % py.__file__)
diff --git a/py/bin/py.test b/py/bin/py.test
deleted file mode 100755
--- a/py/bin/py.test
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env python
-from _findpy import pytest
-raise SystemExit(pytest.main())
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -483,7 +483,7 @@
                 return a_str.strip(' ')
             elif n == 1:
                 return a_str.rstrip(' ')
-            else: 
+            else:
                 return a_str.lstrip(' ')
         s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
         assert s.no_nul
@@ -3737,6 +3737,25 @@
         s = a.build_types(f, [int])
         assert s.listdef.listitem.range_step == 0
 
+    def test_specialize_arg_memo(self):
+        @objectmodel.specialize.memo()
+        def g(n):
+            return n
+        @objectmodel.specialize.arg(0)
+        def f(i):
+            return g(i)
+        def main(i):
+            if i == 2:
+                return f(i)
+            elif i == 3:
+                return f(i)
+            else:
+                raise NotImplementedError
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(main, [int])
+        assert isinstance(s, annmodel.SomeInteger)
+
 
 def g(n):
     return [0,1,2,n]
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -149,6 +149,22 @@
 exported.  This would give us a one-size-fits-all generic .so file to be
 imported by any application that wants to load .so files :-)
 
+Optimising cpyext (CPython C-API compatibility layer)
+-----------------------------------------------------
+
+A lot of work has gone into PyPy's implementation of CPython's C-API over
+the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However,
+there are still many edges and corner cases where it misbehaves, and it has
+not received any substantial optimisation so far.
+
+The objective of this project is to fix bugs in cpyext and to optimise
+several performance critical parts of it, such as the reference counting
+support and other heavily used C-API functions. The net result would be to
+have CPython extensions run much faster on PyPy than they currently do, or
+to make them work at all if they currently don't. A part of this work would
+be to get cpyext into a shape where it supports running Cython generated
+extensions.
 
 .. _`issue tracker`: http://bugs.pypy.org
 .. _`mailing list`: http://mail.python.org/mailman/listinfo/pypy-dev
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -199,17 +199,11 @@
 The following features (present in some past Stackless version of PyPy)
 are for the time being not supported any more:
 
-* Tasklets and channels (currently ``stackless.py`` seems to import,
-  but you have tasklets on top of coroutines on top of greenlets on
-  top of continulets on top of stacklets, and it's probably not too
-  hard to cut two of these levels by adapting ``stackless.py`` to
-  use directly continulets)
-
 * Coroutines (could be rewritten at app-level)
 
-* Pickling and unpickling continulets (*)
-
-* Continuing execution of a continulet in a different thread (*)
+* Continuing execution of a continulet in a different thread
+  (but if it is "simple enough", you can pickle it and unpickle it
+  in the other thread).
 
 * Automatic unlimited stack (must be emulated__ so far)
 
@@ -217,15 +211,6 @@
 
 .. __: `recursion depth limit`_
 
-(*) Pickling, as well as changing threads, could be implemented by using
-a "soft" stack switching mode again.  We would get either "hard" or
-"soft" switches, similarly to Stackless Python 3rd version: you get a
-"hard" switch (like now) when the C stack contains non-trivial C frames
-to save, and a "soft" switch (like previously) when it contains only
-simple calls from Python to Python.  Soft-switched continulets would
-also consume a bit less RAM, and the switch might be a bit faster too
-(unsure about that; what is the Stackless Python experience?).
-
 
 Recursion depth limit
 +++++++++++++++++++++
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
new file mode 100644
--- /dev/null
+++ b/pypy/doc/tool/makecontributor.py
@@ -0,0 +1,133 @@
+import py
+import sys
+from collections import defaultdict
+import operator
+import re
+import mercurial.localrepo
+import mercurial.ui
+
+ROOT = py.path.local(__file__).join('..', '..', '..', '..')
+author_re = re.compile('(.*) <.*>')
+pair_programming_re = re.compile(r'^\((.*?)\)')
+excluded = set(["pypy", "convert-repo"])
+
+alias = {
+    'Anders Chrigstrom': ['arre'],
+    'Antonio Cuni': ['antocuni', 'anto'],
+    'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
+    'Maciej Fijalkowski': ['fijal'],
+    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Michael Hudson': ['mwh'],
+    'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
+    "Amaury Forgeot d'Arc": ['afa'],
+    'Alex Gaynor': ['alex', 'agaynor'],
+    'David Schneider': ['bivab', 'david'],
+    'Christian Tismer': ['chris', 'christian', 'tismer',
+                         'tismer at christia-wjtqxl.localdomain'],
+    'Benjamin Peterson': ['benjamin'],
+    'Hakan Ardo': ['hakan', 'hakanardo'],
+    'Niklaus Haldimann': ['nik'],
+    'Alexander Schremmer': ['xoraxax'],
+    'Anders Hammarquist': ['iko'],
+    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'Niko Matsakis': ['niko'],
+    'Jakub Gustak': ['jlg'],
+    'Guido Wesdorp': ['guido'],
+    'Michael Foord': ['mfoord'],
+    'Mark Pearse': ['mwp'],
+    'Toon Verwaest': ['tverwaes'],
+    'Eric van Riet Paap': ['ericvrp'],
+    'Jacob Hallen': ['jacob', 'jakob'],
+    'Anders Lehmann': ['ale', 'anders'],
+    'Bert Freudenberg': ['bert'],
+    'Boris Feigin': ['boris', 'boria'],
+    'Valentino Volonghi': ['valentino', 'dialtone'],
+    'Aurelien Campeas': ['aurelien', 'aureliene'],
+    'Adrien Di Mascio': ['adim'],
+    'Jacek Generowicz': ['Jacek', 'jacek'],
+    'Jim Hunziker': ['landtuna at gmail.com'],
+    'Kristjan Valur Jonsson': ['kristjan at kristjan-lp.ccp.ad.local'],
+    'Laura Creighton': ['lac'],
+    'Aaron Iles': ['aliles'],
+    'Ludovic Aubry': ['ludal', 'ludovic'],
+    'Lukas Diekmann': ['l.diekmann', 'ldiekmann'],
+    'Matti Picus': ['Matti Picus matti.picus at gmail.com',
+                    'matthp', 'mattip', 'mattip>'],
+    'Michael Cheng': ['mikefc'],
+    'Richard Emslie': ['rxe'],
+    'Roberto De Ioris': ['roberto at goyle'],
+    'Roberto De Ioris': ['roberto at mrspurr'],
+    'Sven Hager': ['hager'],
+    'Tomo Cocoa': ['cocoatomo'],
+    }
+
+alias_map = {}
+for name, nicks in alias.iteritems():
+    for nick in nicks:
+        alias_map[nick] = name
+
+def get_canonical_author(name):
+    match = author_re.match(name)
+    if match:
+        name = match.group(1)
+    return alias_map.get(name, name)
+
+ignored_nicknames = defaultdict(int)
+
+def get_more_authors(log):
+    match = pair_programming_re.match(log)
+    if not match:
+        return set()
+    ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
+                    'in-progress', 'bits of', 'even a little', 'floating',]
+    sep_words = ['and', ';', '+', '/', 'with special  by']
+    nicknames = match.group(1)
+    for word in ignore_words:
+        nicknames = nicknames.replace(word, '')
+    for word in sep_words:
+        nicknames = nicknames.replace(word, ',')
+    nicknames = [nick.strip().lower() for nick in nicknames.split(',')]
+    authors = set()
+    for nickname in nicknames:
+        author = alias_map.get(nickname)
+        if not author:
+            ignored_nicknames[nickname] += 1
+        else:
+            authors.add(author)
+    return authors
+
+def main(show_numbers):
+    ui = mercurial.ui.ui()
+    repo = mercurial.localrepo.localrepository(ui, str(ROOT))
+    authors_count = defaultdict(int)
+    for i in repo:
+        ctx = repo[i]
+        authors = set()
+        authors.add(get_canonical_author(ctx.user()))
+        authors.update(get_more_authors(ctx.description()))
+        for author in authors:
+            if author not in excluded:
+                authors_count[author] += 1
+
+    # uncomment the next lines to get the list of nicknamed which could not be
+    # parsed from commit logs
+    ## items = ignored_nicknames.items()
+    ## items.sort(key=operator.itemgetter(1), reverse=True)
+    ## for name, n in items:
+    ##     if show_numbers:
+    ##         print '%5d %s' % (n, name)
+    ##     else:
+    ##         print name
+                
+    items = authors_count.items()
+    items.sort(key=operator.itemgetter(1), reverse=True)
+    for name, n in items:
+        if show_numbers:
+            print '%5d %s' % (n, name)
+        else:
+            print name
+
+if __name__ == '__main__':
+    show_numbers = '-n' in sys.argv
+    main(show_numbers)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -296,6 +296,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self.frame_trace_action = FrameTraceAction(self)
+        self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
         self.our_magic = default_magic
@@ -467,9 +468,9 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: time2 or rctime take precedence over time
+        # a bit of custom logic: rctime take precedence over time
         # XXX this could probably be done as a "requires" in the config
-        if ('time2' in modules or 'rctime' in modules) and 'time' in modules:
+        if 'rctime' in modules and 'time' in modules:
             modules.remove('time')
 
         if not self.config.objspace.nofaking:
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -154,6 +154,7 @@
         #operationerr.print_detailed_traceback(self.space)
 
     def _convert_exc(self, operr):
+        # Only for the flow object space
         return operr
 
     def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
@@ -166,6 +167,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -113,6 +113,12 @@
         from pypy.interpreter.pycode import PyCode
 
         code = self.getcode() # hook for the jit
+        #
+        if (jit.we_are_jitted() and code is self.space._code_of_sys_exc_info
+                                and nargs == 0):
+            from pypy.module.sys.vm import exc_info_direct
+            return exc_info_direct(self.space, frame)
+        #
         fast_natural_arity = code.fast_natural_arity
         if nargs == fast_natural_arity:
             if nargs == 0:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -874,6 +874,12 @@
             fn.add_to_table()
         if gateway.as_classmethod:
             fn = ClassMethod(space.wrap(fn))
+        #
+        from pypy.module.sys.vm import exc_info
+        if code._bltin is exc_info:
+            assert space._code_of_sys_exc_info is None
+            space._code_of_sys_exc_info = code
+        #
         return fn
 
 
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -2,34 +2,39 @@
 from pypy.interpreter import unicodehelper
 from pypy.rlib.rstring import StringBuilder
 
-def parsestr(space, encoding, s, unicode_literals=False):
-    # compiler.transformer.Transformer.decode_literal depends on what 
-    # might seem like minor details of this function -- changes here 
-    # must be reflected there.
+def parsestr(space, encoding, s, unicode_literal=False):
+    """Parses a string or unicode literal, and return a wrapped value.
+
+    If encoding=iso8859-1, the source string is also in this encoding.
+    If encoding=None, the source string is ascii only.
+    In other cases, the source string is in utf-8 encoding.
+
+    When a bytes string is returned, it will be encoded with the
+    original encoding.
+
+    Yes, it's very inefficient.
+    Yes, CPython has very similar code.
+    """
 
     # we use ps as "pointer to s"
     # q is the virtual last char index of the string
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = unicode_literals
 
     # string decoration handling
-    o = ord(quote)
-    isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
-    if isalpha or quote == '_':
-        if quote == 'b' or quote == 'B':
-            ps += 1
-            quote = s[ps]
-            unicode = False
-        elif quote == 'u' or quote == 'U':
-            ps += 1
-            quote = s[ps]
-            unicode = True
-        if quote == 'r' or quote == 'R':
-            ps += 1
-            quote = s[ps]
-            rawmode = True
+    if quote == 'b' or quote == 'B':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = False
+    elif quote == 'u' or quote == 'U':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = True
+    if quote == 'r' or quote == 'R':
+        ps += 1
+        quote = s[ps]
+        rawmode = True
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
                              'Internal error: parser passed unquoted literal')
@@ -46,21 +51,28 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
-    if unicode: # XXX Py_UnicodeFlag is ignored for now
+    if unicode_literal: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None or encoding == "iso-8859-1":
+            # 'unicode_escape' expects latin-1 bytes, string is ready.
             buf = s
             bufp = ps
             bufq = q
             u = None
         else:
-            # "\XX" may become "\u005c\uHHLL" (12 bytes)
+            # String is utf8-encoded, but 'unicode_escape' expects
+            # latin-1; So multibyte sequences must be escaped.
             lis = [] # using a list to assemble the value
             end = q
+            # Worst case: "\XX" may become "\u005c\uHHLL" (12 bytes)
             while ps < end:
                 if s[ps] == '\\':
                     lis.append(s[ps])
                     ps += 1
                     if ord(s[ps]) & 0x80:
+                        # A multibyte sequence will follow, it will be
+                        # escaped like \u1234. To avoid confusion with
+                        # the backslash we just wrote, we emit "\u005c"
+                        # instead.
                         lis.append("u005c")
                 if ord(s[ps]) & 0x80: # XXX inefficient
                     w, ps = decode_utf8(space, s, ps, end, "utf-16-be")
@@ -86,13 +98,11 @@
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
-    # XXX add strchr like interface to rtyper
     assert 0 <= ps <= q
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
             w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
-            #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1809,6 +1809,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1825,6 +1826,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -27,6 +27,12 @@
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
 
+def boxlonglong(ll):
+    if longlong.is_64_bit:
+        return BoxInt(ll)
+    else:
+        return BoxFloat(ll)
+
 
 class Runner(object):
 
@@ -1782,6 +1788,11 @@
                                      [boxfloat(2.5)], t).value
         assert res == longlong2float.float2longlong(2.5)
 
+        bytes = longlong2float.float2longlong(2.5)
+        res = self.execute_operation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT,
+                                     [boxlonglong(res)], 'float').value
+        assert longlong.getrealfloat(res) == 2.5
+
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -328,6 +328,15 @@
     def produce_into(self, builder, r):
         self.put(builder, [r.choice(builder.intvars)])
 
+class CastLongLongToFloatOperation(AbstractFloatOperation):
+    def produce_into(self, builder, r):
+        if longlong.is_64_bit:
+            self.put(builder, [r.choice(builder.intvars)])
+        else:
+            if not builder.floatvars:
+                raise CannotProduceOperation
+            self.put(builder, [r.choice(builder.floatvars)])
+
 class CastFloatToIntOperation(AbstractFloatOperation):
     def produce_into(self, builder, r):
         if not builder.floatvars:
@@ -450,6 +459,7 @@
 OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT))
 OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT))
 OPERATIONS.append(CastFloatToIntOperation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG))
+OPERATIONS.append(CastLongLongToFloatOperation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT))
 
 OperationBuilder.OPERATIONS = OPERATIONS
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1251,6 +1251,15 @@
         else:
             self.mov(loc0, resloc)
 
+    def genop_convert_longlong_bytes_to_float(self, op, arglocs, resloc):
+        loc0, = arglocs
+        if longlong.is_64_bit:
+            assert isinstance(resloc, RegLoc)
+            assert isinstance(loc0, RegLoc)
+            self.mc.MOVD(resloc, loc0)
+        else:
+            self.mov(loc0, resloc)
+
     def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
         guard_opnum = guard_op.getopnum()
         self.mc.CMP(arglocs[0], imm0)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -719,7 +719,20 @@
             loc0 = self.xrm.loc(arg0)
             loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
             self.Perform(op, [loc0], loc1)
-            self.xrm.possibly_free_var(op.getarg(0))
+            self.xrm.possibly_free_var(arg0)
+
+    def consider_convert_longlong_bytes_to_float(self, op):
+        if longlong.is_64_bit:
+            loc0 = self.rm.make_sure_var_in_reg(op.getarg(0))
+            loc1 = self.xrm.force_allocate_reg(op.result)
+            self.Perform(op, [loc0], loc1)
+            self.rm.possibly_free_var(op.getarg(0))
+        else:
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.make_sure_var_in_reg(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
 
     def _consider_llong_binop_xx(self, op):
         # must force both arguments into xmm registers, because we don't
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -295,6 +295,7 @@
         return op
 
     rewrite_op_convert_float_bytes_to_longlong = _noop_rewrite
+    rewrite_op_convert_longlong_bytes_to_float = _noop_rewrite
 
     # ----------
     # Various kinds of calls
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -968,20 +968,22 @@
             int_return %i2
         """, transform=True)
 
-    def test_convert_float_bytes_to_int(self):
-        from pypy.rlib.longlong2float import float2longlong
+    def test_convert_float_bytes(self):
+        from pypy.rlib.longlong2float import float2longlong, longlong2float
         def f(x):
-            return float2longlong(x)
+            ll = float2longlong(x)
+            return longlong2float(ll)
         if longlong.is_64_bit:
-            result_var = "%i0"
-            return_op = "int_return"
+            tmp_var = "%i0"
+            result_var = "%f1"
         else:
-            result_var = "%f1"
-            return_op = "float_return"
+            tmp_var = "%f1"
+            result_var = "%f2"
         self.encoding_test(f, [25.0], """
-            convert_float_bytes_to_longlong %%f0 -> %(result_var)s
-            %(return_op)s %(result_var)s
-        """ % {"result_var": result_var, "return_op": return_op})
+            convert_float_bytes_to_longlong %%f0 -> %(tmp_var)s
+            convert_longlong_bytes_to_float %(tmp_var)s -> %(result_var)s
+            float_return %(result_var)s
+        """ % {"result_var": result_var, "tmp_var": tmp_var}, transform=True)
 
 
 def check_force_cast(FROM, TO, operations, value):
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -672,6 +672,11 @@
         a = longlong.getrealfloat(a)
         return longlong2float.float2longlong(a)
 
+    @arguments(LONGLONG_TYPECODE, returns="f")
+    def bhimpl_convert_longlong_bytes_to_float(a):
+        a = longlong2float.longlong2float(a)
+        return longlong.getfloatstorage(a)
+
     # ----------
     # control flow operations
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -224,6 +224,7 @@
                     'float_neg', 'float_abs',
                     'cast_ptr_to_int', 'cast_int_to_ptr',
                     'convert_float_bytes_to_longlong',
+                    'convert_longlong_bytes_to_float',
                     ]:
         exec py.code.Source('''
             @arguments("box")
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -420,6 +420,7 @@
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     'CONVERT_FLOAT_BYTES_TO_LONGLONG/1',
+    'CONVERT_LONGLONG_BYTES_TO_FLOAT/1',
     #
     'INT_LT/2b',
     'INT_LE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,3 +1,4 @@
+import math
 import sys
 
 import py
@@ -15,7 +16,7 @@
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
     isconstant, isvirtual, promote_string, set_param, record_known_class)
-from pypy.rlib.longlong2float import float2longlong
+from pypy.rlib.longlong2float import float2longlong, longlong2float
 from pypy.rlib.rarithmetic import ovfcheck, is_valid_int
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3795,15 +3796,15 @@
         res = self.interp_operations(g, [1])
         assert res == 3
 
-    def test_float2longlong(self):
+    def test_float_bytes(self):
         def f(n):
-            return float2longlong(n)
+            ll = float2longlong(n)
+            return longlong2float(ll)
 
         for x in [2.5, float("nan"), -2.5, float("inf")]:
             # There are tests elsewhere to verify the correctness of this.
-            expected = float2longlong(x)
             res = self.interp_operations(f, [x])
-            assert longlong.getfloatstorage(res) == expected
+            assert res == x or math.isnan(x) and math.isnan(res)
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -432,7 +432,8 @@
                     raise _ssl_seterror(self.space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return self.space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return self.space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                              length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -11,7 +11,7 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.objectmodel import specialize
+from pypy.rlib.objectmodel import specialize, keepalive_until_here
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
@@ -145,18 +145,24 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(RWBuffer):
-    def __init__(self, data, bytes):
-        self.data = data
-        self.len = bytes
+    def __init__(self, array):
+        self.array = array
 
     def getlength(self):
-        return self.len
+        return self.array.len * self.array.itemsize
 
     def getitem(self, index):
-        return self.data[index]
+        array = self.array
+        data = array._charbuf_start()
+        char = data[index]
+        array._charbuf_stop()
+        return char
 
     def setitem(self, index, char):
-        self.data[index] = char
+        array = self.array
+        data = array._charbuf_start()
+        data[index] = char
+        array._charbuf_stop()
 
 
 def make_array(mytype):
@@ -278,9 +284,10 @@
             oldlen = self.len
             new = len(s) / mytype.bytes
             self.setlen(oldlen + new)
-            cbuf = self.charbuf()
+            cbuf = self._charbuf_start()
             for i in range(len(s)):
                 cbuf[oldlen * mytype.bytes + i] = s[i]
+            self._charbuf_stop()
 
         def fromlist(self, w_lst):
             s = self.len
@@ -310,8 +317,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def charbuf(self):
-            return  rffi.cast(rffi.CCHARP, self.buffer)
+        def _charbuf_start(self):
+            return rffi.cast(rffi.CCHARP, self.buffer)
+
+        def _charbuf_stop(self):
+            keepalive_until_here(self)
 
         def w_getitem(self, space, idx):
             item = self.buffer[idx]
@@ -530,8 +540,10 @@
         self.fromstring(space.str_w(w_s))
 
     def array_tostring__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
+        self._charbuf_stop()
+        return self.space.wrap(s)
 
     def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
         if not isinstance(w_f, W_File):
@@ -613,8 +625,7 @@
     # Misc methods
 
     def buffer__Array(space, self):
-        b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes)
-        return space.wrap(b)
+        return space.wrap(ArrayBuffer(self))
 
     def array_buffer_info__Array(space, self):
         w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer))
@@ -649,7 +660,7 @@
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
-        bytes = self.charbuf()
+        bytes = self._charbuf_start()
         tmp = [bytes[0]] * mytype.bytes
         for start in range(0, self.len * mytype.bytes, mytype.bytes):
             stop = start + mytype.bytes - 1
@@ -657,6 +668,7 @@
                 tmp[i] = bytes[start + i]
             for i in range(mytype.bytes):
                 bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
 
     def repr__Array(space, self):
         if self.len == 0:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -433,7 +433,25 @@
         a = self.array('h', 'Hi')
         buf = buffer(a)
         assert buf[1] == 'i'
-        #raises(TypeError, buf.__setitem__, 1, 'o')
+
+    def test_buffer_write(self):
+        a = self.array('c', 'hello')
+        buf = buffer(a)
+        print repr(buf)
+        try:
+            buf[3] = 'L'
+        except TypeError:
+            skip("buffer(array) returns a read-only buffer on CPython")
+        assert a.tostring() == 'helLo'
+
+    def test_buffer_keepalive(self):
+        buf = buffer(self.array('c', 'text'))
+        assert buf[2] == 'x'
+        #
+        a = self.array('c', 'foobarbaz')
+        buf = buffer(a)
+        a.fromstring('some extra text')
+        assert buf[:] == 'foobarbazsome extra text'
 
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,8 +2,10 @@
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
     PyObjectFields, PyObject)
-from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
+from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
+from pypy.interpreter.error import OperationError
+from pypy.module.array.interp_array import ArrayBuffer
 
 
 PyBufferObjectStruct = lltype.ForwardReference()
@@ -41,26 +43,38 @@
         py_buf.c_b_offset = w_obj.offset
         w_obj = w_obj.buffer
 
+    # If w_obj already allocated a fixed buffer, use it, and keep a
+    # reference to w_obj.
+    # Otherwise, b_base stays NULL, and we own the b_ptr.
+
     if isinstance(w_obj, StringBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.as_str()))
+        py_buf.c_b_base = lltype.nullptr(PyObject.TO)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
+        py_buf.c_b_size = w_obj.getlength()
+    elif isinstance(w_obj, ArrayBuffer):
+        w_base = w_obj.array
+        py_buf.c_b_base = make_ref(space, w_base)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start())
         py_buf.c_b_size = w_obj.getlength()
     else:
-        raise Exception("Fail fail fail fail fail")
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "buffer flavor not supported"))
 
 
 def buffer_realize(space, py_obj):
     """
     Creates the buffer in the PyPy interpreter from a cpyext representation.
     """
-    raise Exception("realize fail fail fail")
-
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "Don't know how to realize a buffer"))
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
-    Py_DecRef(space, py_buf.c_b_base)
-    rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
+    if py_buf.c_b_base:
+        Py_DecRef(space, py_buf.c_b_base)
+    else:
+        rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -315,3 +316,65 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,14 +167,16 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -183,7 +185,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -48,3 +48,17 @@
             ])
         b = module.buffer_new()
         raises(AttributeError, getattr, b, 'x')
+
+    def test_array_buffer(self):
+        module = self.import_extension('foo', [
+            ("roundtrip", "METH_O",
+             """
+                 PyBufferObject *buf = (PyBufferObject *)args;
+                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
+             """),
+            ])
+        import array
+        a = array.array('c', 'text')
+        b = buffer(a)
+        assert module.roundtrip(b) == 'text'
+        
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
@@ -414,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -425,6 +429,36 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
+
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -307,6 +307,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -624,7 +626,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
@@ -99,9 +100,12 @@
         ("exp2", "exp2"),
         ("expm1", "expm1"),
         ("fabs", "fabs"),
+        ("fmax", "fmax"),
+        ("fmin", "fmin"),
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -122,12 +126,16 @@
         ("sinh", "sinh"),
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
+        ('square', 'square'),
         ("tan", "tan"),
         ("tanh", "tanh"),
         ('bitwise_and', 'bitwise_and'),
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
+        ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
         ('isneginf', 'isneginf'),
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -673,6 +673,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -1261,6 +1265,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1386,6 +1415,7 @@
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,9 +3,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype, support, loop
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
+
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
@@ -28,7 +30,7 @@
         return self.identity
 
     def descr_call(self, space, __args__):
-	from interp_numarray import BaseArray
+        from interp_numarray import BaseArray
         args_w, kwds_w = __args__.unpack()
         # it occurs to me that we don't support any datatypes that
         # require casting, change it later when we do
@@ -179,7 +181,7 @@
                 elif out.shape != shape:
                     raise operationerrfmt(space.w_ValueError,
                         'output parameter shape mismatch, expecting [%s]' +
-                        ' , got [%s]', 
+                        ' , got [%s]',
                         ",".join([str(x) for x in shape]),
                         ",".join([str(x) for x in out.shape]),
                         )
@@ -204,7 +206,7 @@
         else:
             arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
             val = loop.compute(arr)
-        return val 
+        return val
 
     def do_axis_reduce(self, obj, dtype, axis, result):
         from pypy.module.micronumpy.interp_numarray import AxisReduce
@@ -253,7 +255,7 @@
         if isinstance(w_obj, Scalar):
             arr = self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))
             if isinstance(out,Scalar):
-                out.value=arr
+                out.value = arr
             elif isinstance(out, BaseArray):
                 out.fill(space, arr)
             else:
@@ -265,7 +267,7 @@
             if not broadcast_shape or broadcast_shape != out.shape:
                 raise operationerrfmt(space.w_ValueError,
                     'output parameter shape mismatch, could not broadcast [%s]' +
-                    ' to [%s]', 
+                    ' to [%s]',
                     ",".join([str(x) for x in w_obj.shape]),
                     ",".join([str(x) for x in out.shape]),
                     )
@@ -292,10 +294,11 @@
         self.func = func
         self.comparison_func = comparison_func
 
+    @jit.unroll_safe
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
             convert_to_array, Scalar, shape_agreement, BaseArray)
-        if len(args_w)>2:
+        if len(args_w) > 2:
             [w_lhs, w_rhs, w_out] = args_w
         else:
             [w_lhs, w_rhs] = args_w
@@ -326,7 +329,7 @@
                 w_rhs.value.convert_to(calc_dtype)
             )
             if isinstance(out,Scalar):
-                out.value=arr
+                out.value = arr
             elif isinstance(out, BaseArray):
                 out.fill(space, arr)
             else:
@@ -337,7 +340,7 @@
         if out and out.shape != shape_agreement(space, new_shape, out.shape):
             raise operationerrfmt(space.w_ValueError,
                 'output parameter shape mismatch, could not broadcast [%s]' +
-                ' to [%s]', 
+                ' to [%s]',
                 ",".join([str(x) for x in new_shape]),
                 ",".join([str(x) for x in out.shape]),
                 )
@@ -347,7 +350,6 @@
         w_lhs.add_invalidates(w_res)
         w_rhs.add_invalidates(w_res)
         if out:
-            #out.add_invalidates(w_res) #causes a recursion loop
             w_res.get_concrete()
         return w_res
 
@@ -539,14 +541,18 @@
             ("reciprocal", "reciprocal", 1),
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
+            ("fmax", "fmax", 2, {"promote_to_float": True}),
+            ("fmin", "fmin", 2, {"promote_to_float": True}),
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
+            ("trunc", "trunc", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
             ("exp2", "exp2", 1, {"promote_to_float": True}),
             ("expm1", "expm1", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
+            ('square', 'square', 1, {'promote_to_float': True}),
 
             ("sin", "sin", 1, {"promote_to_float": True}),
             ("cos", "cos", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -107,6 +107,10 @@
         arr.compute_first_step(self, f)
         return f
 
+    def debug_repr(self):
+        # should be overridden, but in case it isn't, provide a default
+        return str(self)
+
 class ConcreteSignature(Signature):
     _immutable_fields_ = ['dtype']
 
@@ -207,7 +211,7 @@
     def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        transforms = transforms + [ViewTransform(arr.chunks)]
+        transforms = [ViewTransform(arr.chunks)] + transforms 
         self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
@@ -215,6 +219,9 @@
         assert isinstance(arr, VirtualSlice)
         return self.child.eval(frame, arr.child)
 
+    def debug_repr(self):
+        return 'VirtualSlice(%s)' % self.child.debug_repr()
+
 class Call1(Signature):
     _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
 
@@ -270,7 +277,7 @@
         from pypy.module.micronumpy.interp_numarray import Call1
 
         assert isinstance(arr, Call1)
-        vtransforms = transforms + [BroadcastTransform(arr.values.shape)]
+        vtransforms = [BroadcastTransform(arr.values.shape)] + transforms
         self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
         self.res._create_iter(iterlist, arraylist, arr.res, transforms)
 
@@ -348,7 +355,7 @@
         from pypy.module.micronumpy.interp_numarray import ResultArray
 
         assert isinstance(arr, ResultArray)
-        rtransforms = transforms + [BroadcastTransform(arr.left.shape)]
+        rtransforms = [BroadcastTransform(arr.left.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -375,7 +382,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
@@ -388,7 +395,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -401,8 +408,8 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -424,7 +431,7 @@
         frame.cur_value = self.binfunc(self.calc_dtype, frame.cur_value, rval)
 
     def debug_repr(self):
-        return 'ReduceSig(%s)' % (self.name, self.right.debug_repr())
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
 
 class SliceloopSignature(Call2):
     def eval(self, frame, arr):
@@ -448,7 +455,7 @@
         from pypy.module.micronumpy.interp_numarray import SliceArray
 
         assert isinstance(arr, SliceArray)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,6 +1,7 @@
 from pypy.rlib import jit
 from pypy.interpreter.error import OperationError
 
+ at jit.look_inside_iff(lambda chunks: jit.isconstant(len(chunks)))
 def enumerate_chunks(chunks):
     result = []
     i = -1
@@ -85,9 +86,9 @@
         space.isinstance_w(w_item_or_slice, space.w_slice)):
         raise OperationError(space.w_IndexError,
                              space.wrap('unsupported iterator index'))
-            
+
     start, stop, step, lngth = space.decode_index4(w_item_or_slice, size)
-        
+
     coords = [0] * len(shape)
     i = start
     if order == 'C':
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,5 +1,9 @@
+from pypy.rlib import jit
+
+
+ at jit.unroll_safe
 def product(s):
     i = 1
     for x in s:
         i *= x
-    return i
\ No newline at end of file
+    return i
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -395,11 +395,19 @@
         assert a[3] == 0.
 
     def test_newaxis(self):
-        from _numpypy import array
+        import math
+        from _numpypy import array, cos, zeros
         from numpypy.core.numeric import newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
+        a = array(range(3))
+        b = array([1, 3])
+        expected = zeros((3, 2))
+        for x in range(3):
+            for y in range(2):
+                expected[x, y] = math.cos(a[x]) * math.cos(b[y])
+        assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):
         from _numpypy import array
@@ -1338,6 +1346,10 @@
         dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0)
         assert str(dims_disagree.value) == \
             "array dimensions must agree except for axis being concatenated"
+        a = array([1, 2, 3, 4, 5, 6])
+        a = (a + a)[::2]
+        b = concatenate((a[:3], a[-3:]))
+        assert (b == [2, 6, 10, 2, 6, 10]).all()
 
     def test_std(self):
         from _numpypy import array
@@ -1387,6 +1399,16 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat, array
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
+
 
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -135,6 +135,38 @@
         assert fabs(float('-inf')) == float('inf')
         assert isnan(fabs(float('nan')))
 
+    def test_fmax(self):
+        from _numpypy import fmax
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmax(a, [ninf]*5) == a).all()
+        assert (fmax(a, [inf]*5) == [inf]*5).all()
+        assert (fmax(a, [1]*5) == [1, 1, 1, 5, inf]).all()
+        assert math.isnan(fmax(nan, 0))
+        assert math.isnan(fmax(0, nan))
+        assert math.isnan(fmax(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+
+    def test_fmin(self):
+        from _numpypy import fmin
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmin(a, [ninf]*5) == [ninf]*5).all()
+        assert (fmin(a, [inf]*5) == a).all()
+        assert (fmin(a, [1]*5) == [ninf, -5, 0, 1, 1]).all()
+        assert math.isnan(fmin(nan, 0))
+        assert math.isnan(fmin(0, nan))
+        assert math.isnan(fmin(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+
     def test_fmod(self):
         from _numpypy import fmod
         import math
@@ -221,24 +253,17 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floorceil(self):
-        from _numpypy import array, floor, ceil
+    def test_floorceiltrunc(self):
+        from _numpypy import array, floor, ceil, trunc
         import math
-        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = floor(a)
-        for i in range(5):
-            assert b[i] == reference[i]
-        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = ceil(a)
-        assert (reference == b).all()
-        inf = float("inf")
-        data = [1.5, 2.9999, -1.999, inf]
-        results = [math.floor(x) for x in data]
-        assert (floor(data) == results).all()
-        results = [math.ceil(x) for x in data]
-        assert (ceil(data) == results).all()
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == floor(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
+        assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -455,6 +480,19 @@
         assert math.isnan(sqrt(-1))
         assert math.isnan(sqrt(nan))
 
+    def test_square(self):
+        import math
+        from _numpypy import square
+
+        nan, inf, ninf = float("nan"), float("inf"), float("-inf")
+
+        assert math.isnan(square(nan))
+        assert math.isinf(square(inf))
+        assert math.isinf(square(ninf))
+        assert square(ninf) > 0
+        assert [square(x) for x in range(-5, 5)] == [x*x for x in range(-5, 5)]
+        assert math.isinf(square(1e300))
+
     def test_radians(self):
         import math
         from _numpypy import radians, array
@@ -546,10 +584,17 @@
         raises(TypeError, 'array([1.0]) & 1')
 
     def test_unary_bitops(self):
-        from _numpypy import bitwise_not, array
+        from _numpypy import bitwise_not, invert, array
         a = array([1, 2, 3, 4])
         assert (~a == [-2, -3, -4, -5]).all()
         assert (bitwise_not(a) == ~a).all()
+        assert (invert(a) == ~a).all()
+
+    def test_shift(self):
+        from _numpypy import left_shift, right_shift
+
+        assert (left_shift([5, 1], [2, 13]) == [20, 2**13]).all()
+        assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
 
     def test_comparisons(self):
         import operator
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -631,6 +631,22 @@
         return math.fabs(v)
 
     @simple_binary_op
+    def fmax(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return max(v1, v2)
+
+    @simple_binary_op
+    def fmin(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return min(v1, v2)
+
+    @simple_binary_op
     def fmod(self, v1, v2):
         try:
             return math.fmod(v1, v2)
@@ -652,6 +668,13 @@
         return math.ceil(v)
 
     @simple_unary_op
+    def trunc(self, v):
+        if v < 0:
+            return math.ceil(v)
+        else:
+            return math.floor(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -741,6 +764,10 @@
         except ValueError:
             return rfloat.NAN
 
+    @simple_unary_op
+    def square(self, v):
+        return v*v
+
     @raw_unary_op
     def isnan(self, v):
         return rfloat.isnan(v)
diff --git a/pypy/module/pyexpat/test/__init__.py b/pypy/module/pyexpat/test/__init__.py
new file mode 100644
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -212,7 +212,7 @@
             i19 = int_add(i12, 1)
             setfield_gc(p9, i19, descr=<FieldS .*W_AbstractSeqIterObject.inst_index .*>)
             guard_nonnull_class(p17, 146982464, descr=...)
-            i21 = getfield_gc(p17, descr=<FieldS .*W_ArrayTypei.inst_len .*>)
+            i21 = getfield_gc(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
             i24 = getfield_gc(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)
@@ -351,3 +351,23 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(log.loops_by_id("eval")) == 1
+
+    def test_sys_exc_info(self):
+        def main():
+            i = 1
+            lst = [i]
+            while i < 1000:
+                try:
+                    return lst[i]
+                except:
+                    e = sys.exc_info()[1]    # ID: exc_info
+                    if not isinstance(e, IndexError):
+                        raise
+                i += 1
+            return 42
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("exc_info")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -198,3 +198,37 @@
             i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>)
             guard_value(i49, 1, descr=...)
             ''')
+
+    def test_remove_duplicate_method_calls(self):
+        def main(n):
+            lst = []
+            for i in range(n):
+                s = 'Hello %d' % i
+                t = s.lower()   # ID: callone
+                u = s.lower()   # ID: calltwo
+                lst.append(t)
+                lst.append(u)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '''
+            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr . r EF=3>)
+            guard_no_exception(descr=...)
+            ''')
+        loop.match_by_id('calltwo', '')    # nothing
+
+    def test_move_method_call_out_of_loop(self):
+        def main(n):
+            lst = []
+            s = 'Hello %d' % n
+            for i in range(n):
+                t = s.lower()   # ID: callone
+                lst.append(t)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '')    # nothing
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -595,3 +595,124 @@
         assert len(frames) == 1
         _, other_frame = frames.popitem()
         assert other_frame.f_code.co_name in ('other_thread', '?')
+
+
+class AppTestSysExcInfoDirect:
+
+    def setup_method(self, meth):
+        self.checking = not option.runappdirect
+        if self.checking:
+            self.seen = []
+            from pypy.module.sys import vm
+            def exc_info_with_tb(*args):
+                self.seen.append("n")     # not optimized
+                return self.old[0](*args)
+            def exc_info_without_tb(*args):
+                self.seen.append("y")     # optimized
+                return self.old[1](*args)
+            self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
+            vm.exc_info_with_tb = exc_info_with_tb
+            vm.exc_info_without_tb = exc_info_without_tb
+            #
+            from pypy.rlib import jit
+            self.old2 = [jit.we_are_jitted]
+            jit.we_are_jitted = lambda: True
+
+    def teardown_method(self, meth):
+        if self.checking:
+            from pypy.module.sys import vm
+            from pypy.rlib import jit
+            vm.exc_info_with_tb = self.old[0]
+            vm.exc_info_without_tb = self.old[1]
+            jit.we_are_jitted = self.old2[0]
+            #
+            assert ''.join(self.seen) == meth.expected
+
+    def test_returns_none(self):
+        import sys
+        assert sys.exc_info() == (None, None, None)
+        assert sys.exc_info()[0] is None
+        assert sys.exc_info()[1] is None
+        assert sys.exc_info()[2] is None
+        assert sys.exc_info()[:2] == (None, None)
+        assert sys.exc_info()[:3] == (None, None, None)
+        assert sys.exc_info()[0:2] == (None, None)
+        assert sys.exc_info()[2:4] == (None,)
+    test_returns_none.expected = 'nnnnnnnn'
+
+    def test_returns_subscr(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert sys.exc_info()[0] is KeyError  # y
+            assert sys.exc_info()[1] is e         # y
+            assert sys.exc_info()[2] is not None  # n
+            assert sys.exc_info()[-3] is KeyError # y
+            assert sys.exc_info()[-2] is e        # y
+            assert sys.exc_info()[-1] is not None # n
+    test_returns_subscr.expected = 'yynyyn'
+
+    def test_returns_slice_2(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                  # n
+            assert sys.exc_info()[:0] == ()       # y
+            assert sys.exc_info()[:1] == foo[:1]  # y
+            assert sys.exc_info()[:2] == foo[:2]  # y
+            assert sys.exc_info()[:3] == foo      # n
+            assert sys.exc_info()[:4] == foo      # n
+            assert sys.exc_info()[:-1] == foo[:2] # y
+            assert sys.exc_info()[:-2] == foo[:1] # y
+            assert sys.exc_info()[:-3] == ()      # y
+    test_returns_slice_2.expected = 'nyyynnyyy'
+
+    def test_returns_slice_3(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                   # n
+            assert sys.exc_info()[2:2] == ()       # y
+            assert sys.exc_info()[0:1] == foo[:1]  # y
+            assert sys.exc_info()[1:2] == foo[1:2] # y
+            assert sys.exc_info()[0:3] == foo      # n
+            assert sys.exc_info()[2:4] == foo[2:]  # n
+            assert sys.exc_info()[0:-1] == foo[:2] # y
+            assert sys.exc_info()[0:-2] == foo[:1] # y
+            assert sys.exc_info()[5:-3] == ()      # y
+    test_returns_slice_3.expected = 'nyyynnyyy'
+
+    def test_strange_invocation(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            a = []; k = {}
+            assert sys.exc_info(*a)[:0] == ()
+            assert sys.exc_info(**k)[:0] == ()
+    test_strange_invocation.expected = 'nn'
+
+    def test_call_in_subfunction(self):
+        import sys
+        def g():
+            # this case is not optimized, because we need to search the
+            # frame chain.  it's probably not worth the complications
+            return sys.exc_info()[1]
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert g() is e
+    test_call_in_subfunction.expected = 'n'
+
+
+class AppTestSysExcInfoDirectCallMethod(AppTestSysExcInfoDirect):
+    def setup_class(cls):
+        cls.space = gettestobjspace(**{"objspace.opcodes.CALL_METHOD": True})
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 2, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 9, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -89,6 +89,9 @@
     """Return the (type, value, traceback) of the most recent exception
 caught by an except clause in the current stack frame or in an older stack
 frame."""
+    return exc_info_with_tb(space)    # indirection for the tests
+
+def exc_info_with_tb(space):
     operror = space.getexecutioncontext().sys_exc_info()
     if operror is None:
         return space.newtuple([space.w_None,space.w_None,space.w_None])
@@ -96,6 +99,59 @@
         return space.newtuple([operror.w_type, operror.get_w_value(space),
                                space.wrap(operror.get_traceback())])
 
+def exc_info_without_tb(space, frame):
+    operror = frame.last_exception
+    return space.newtuple([operror.w_type, operror.get_w_value(space),
+                           space.w_None])
+
+def exc_info_direct(space, frame):
+    from pypy.tool import stdlib_opcode
+    # In order to make the JIT happy, we try to return (exc, val, None)
+    # instead of (exc, val, tb).  We can do that only if we recognize
+    # the following pattern in the bytecode:
+    #       CALL_FUNCTION/CALL_METHOD         <-- invoking me
+    #       LOAD_CONST 0, 1, -2 or -3
+    #       BINARY_SUBSCR
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST <=2
+    #       SLICE_2
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST any integer
+    #       LOAD_CONST <=2
+    #       SLICE_3
+    need_all_three_args = True
+    co = frame.getcode().co_code
+    p = frame.last_instr
+    if (ord(co[p]) == stdlib_opcode.CALL_FUNCTION or
+        ord(co[p]) == stdlib_opcode.CALL_METHOD):
+        if ord(co[p+3]) == stdlib_opcode.LOAD_CONST:
+            lo = ord(co[p+4])
+            hi = ord(co[p+5])
+            w_constant = frame.getconstant_w((hi * 256) | lo)
+            if space.isinstance_w(w_constant, space.w_int):
+                constant = space.int_w(w_constant)
+                if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+                    if -3 <= constant <= 1 and constant != -1:
+                        need_all_three_args = False
+                elif ord(co[p+6]) == stdlib_opcode.SLICE+2:
+                    if constant <= 2:
+                        need_all_three_args = False
+                elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
+                      ord(co[p+9]) == stdlib_opcode.SLICE+3):
+                    lo = ord(co[p+7])
+                    hi = ord(co[p+8])
+                    w_constant = frame.getconstant_w((hi * 256) | lo)
+                    if space.isinstance_w(w_constant, space.w_int):
+                        if space.int_w(w_constant) <= 2:
+                            need_all_three_args = False
+    #
+    if need_all_three_args or frame.last_exception is None or frame.hide():
+        return exc_info_with_tb(space)
+    else:
+        return exc_info_without_tb(space, frame)
+
 def exc_clear(space):
     """Clear global information on the current exception.  Subsequent calls
 to exc_info() will return (None,None,None) until another exception is
diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py
--- a/pypy/module/test_lib_pypy/test_datetime.py
+++ b/pypy/module/test_lib_pypy/test_datetime.py
@@ -44,3 +44,9 @@
     assert type(dt.microsecond) is int
 
     copy.copy(dt)
+
+def test_radd():
+    class X(object):
+        def __radd__(self, other):
+            return "radd"
+    assert datetime.date(10, 10, 10) + X() == "radd"
diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py
--- a/pypy/module/thread/test/test_ll_thread.py
+++ b/pypy/module/thread/test/test_ll_thread.py
@@ -66,7 +66,6 @@
     def test_gc_locking(self):
         import time
         from pypy.rlib.objectmodel import invoke_around_extcall
-        from pypy.rlib.objectmodel import we_are_translated
         from pypy.rlib.debug import ll_assert
 
         class State:
@@ -129,8 +128,6 @@
             state.finished = 0
             # the next line installs before_extcall() and after_extcall()
             # to be called automatically around external function calls.
-            # When not translated it does not work around time.sleep(),
-            # so we have to call them manually for this test.
             invoke_around_extcall(before_extcall, after_extcall)
 
             g(10, 1)
@@ -142,13 +139,9 @@
                 willing_to_wait_more -= 1
                 done = len(state.answers) == expected
 
-                if not we_are_translated(): before_extcall()
                 time.sleep(0.01)
-                if not we_are_translated(): after_extcall()
 
-            if not we_are_translated(): before_extcall()
             time.sleep(0.1)
-            if not we_are_translated(): after_extcall()
 
             return len(state.answers)
 
@@ -160,12 +153,11 @@
         answers = fn()
         assert answers == expected
 
-class TestRunDirectly(AbstractThreadTests):
-    def getcompiled(self, f, argtypes):
-        return f
-
-    def test_start_new_thread(self):
-        py.test.skip("deadlocks occasionally -- why???")
+#class TestRunDirectly(AbstractThreadTests):
+#    def getcompiled(self, f, argtypes):
+#        return f
+# These are disabled because they crash occasionally for bad reasons
+# related to the fact that ll2ctypes is not at all thread-safe
 
 class TestUsingBoehm(AbstractThreadTests):
     gcpolicy = 'boehm'
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -43,6 +43,13 @@
     return w_eq
 type_eq._annspecialcase_ = 'specialize:memo'
 
+def list_iter(space):
+    "Utility that returns the app-level descriptor list.__iter__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_list,
+                                               '__iter__')
+    return w_iter
+list_iter._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     w_type = space.type(w_obj)
     typename = w_type.getname(space)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -86,6 +86,7 @@
         return s_None
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 # ____________________________________________________________
@@ -109,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -434,6 +434,13 @@
         self.lastblock = block
         self.pushvalue(w_result)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # This opcode was added with pypy-1.8.  Here is a simpler
+        # version, enough for annotation.
+        last_val = self.popvalue()
+        self.pushvalue(self.space.newlist([]))
+        self.pushvalue(last_val)
+
     # XXX Unimplemented 2.7 opcodes ----------------
 
     # Set literals, set comprehensions
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -163,7 +163,8 @@
 
 class ModuleDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         dict_w = strategy.unerase(dictimplementation.dstorage)
         self.iterator = dict_w.iteritems()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,8 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, classofinstance=None,
-                                   strdict=False):
+                                   instance=False, strdict=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -247,7 +246,7 @@
         return 0
 
     def iter(self, w_dict):
-        return EmptyIteratorImplementation(self.space, w_dict)
+        return EmptyIteratorImplementation(self.space, self, w_dict)
 
     def clear(self, w_dict):
         return
@@ -263,8 +262,9 @@
 # Iterator Implementation base classes
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.dictimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -280,7 +280,20 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.dictimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the dict.  The (key, value) pair in 'result'
+                # might be out-of-date.  We try to explicitly look up
+                # the key in the dict.
+                w_key = result[0]
+                w_value = self.dictimplementation.getitem(w_key)
+                if w_value is None:
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return (w_key, w_value)
         # no more entries
         self.dictimplementation = None
         return None, None
@@ -489,7 +502,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -503,7 +516,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -549,10 +562,7 @@
     def listview_int(self, w_dict):
         return self.unerase(w_dict.dstorage).keys()
 
-    def w_keys(self, w_dict):
-        # XXX there is no space.newlist_int yet
-        space = self.space
-        return space.call_function(space.w_list, w_dict)
+    # XXX there is no space.newlist_int yet to implement w_keys more efficiently
 
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -20,7 +20,17 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_w(w_lookup_type, space.w_str):
+        if (space.is_w(w_lookup_type, space.w_str) or  # Most common path first
+            space.abstract_issubclass_w(w_lookup_type, space.w_str)):
+            return self.getitem_str(w_dict, space.str_w(w_key))
+        elif space.abstract_issubclass_w(w_lookup_type, space.w_unicode):
+            try:
+                w_key = space.str(w_key)
+            except OperationError, e:
+                if not e.match(space, space.w_UnicodeEncodeError):
+                    raise
+                # non-ascii unicode is never equal to a byte string
+                return None
             return self.getitem_str(w_dict, space.str_w(w_key))
         else:
             return None
@@ -98,7 +108,8 @@
 
 class DictProxyIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_type = strategy.unerase(dictimplementation.dstorage)
         self.iterator = w_type.dict_w.iteritems()
 
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -1,5 +1,5 @@
 ## ----------------------------------------------------------------------------
-## dict strategy (see dict_multiobject.py)
+## dict strategy (see dictmultiobject.py)
 
 from pypy.rlib import rerased
 from pypy.rlib.debug import mark_dict_non_null
@@ -80,8 +80,8 @@
     def iter(self, w_dict):
         return IdentityDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class IdentityDictIteratorImplementation(_UnwrappedIteratorMixin, IteratorImplementation):
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -703,7 +703,8 @@
 
 class MapDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_obj = strategy.unerase(dictimplementation.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,10 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                classofinstance=classofinstance,
                 strdict=strdict)
 
     def newset(self):
@@ -439,6 +438,8 @@
             t = w_obj.getitems()
         elif isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
+        elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            t = w_obj.getitems()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
         if expected_length != -1 and len(t) != expected_length:
@@ -456,6 +457,8 @@
             return w_obj.listview_str()
         if isinstance(w_obj, W_StringObject):
             return w_obj.listview_str()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_str()
         return None
 
     def listview_int(self, w_obj):
@@ -465,8 +468,14 @@
             return w_obj.listview_int()
         if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
             return w_obj.listview_int()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_int()
         return None
 
+    def _uses_list_iter(self, w_obj):
+        from pypy.objspace.descroperation import list_iter
+        return self.lookup(w_obj, '__iter__') is list_iter(self)
+
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -359,7 +359,7 @@
         w_set.sstorage = w_other.get_storage_copy()
 
     def iter(self, w_set):
-        return EmptyIteratorImplementation(self.space, w_set)
+        return EmptyIteratorImplementation(self.space, self, w_set)
 
     def popitem(self, w_set):
         raise OperationError(self.space.w_KeyError,
@@ -784,8 +784,9 @@
             d_obj[w_item] = None
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.setimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -801,7 +802,17 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.setimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the set.  The 'result' might be out-of-date.
+                # We try to explicitly look it up in the set.
+                if not self.setimplementation.has_key(result):
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return result
         # no more entries
         self.setimplementation = None
         return None
@@ -823,7 +834,7 @@
 
 class StringIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, w_set):
-        IteratorImplementation.__init__(self, space, w_set)
+        IteratorImplementation.__init__(self, space, strategy, w_set)
         d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
@@ -835,9 +846,9 @@
 
 class IntegerIteratorImplementation(IteratorImplementation):
     #XXX same implementation in dictmultiobject on dictstrategy-branch
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
@@ -848,9 +859,9 @@
             return None
 
 class RDictIteratorImplementation(IteratorImplementation):
-    def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
-        d = strategy.unerase(dictimplementation.sstorage)
+    def __init__(self, space, strategy, w_set):
+        IteratorImplementation.__init__(self, space, strategy, w_set)
+        d = strategy.unerase(w_set.sstorage)
         self.iterator = d.iterkeys()
 
     def next_entry(self):
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -804,6 +804,33 @@
         assert "IntDictStrategy" in self.get_strategy(d)
         assert d[1L] == "hi"
 
+    def test_iter_dict_length_change(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d[7] = 8
+        # 'd' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_dict_strategy_only_change_1(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert d.get(Foo()) is None    # this changes the strategy of 'd'
+        lst = list(it)  # but iterating still works
+        assert sorted(lst) == [(1, 2), (3, 4), (5, 6)]
+
+    def test_iter_dict_strategy_only_change_2(self):
+        d = {1: 2, 3: 4, 5: 6}
+        it = d.iteritems()
+        d['foo'] = 'bar'
+        del d[1]
+        # 'd' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us (1, 2), but 1 is not in the dict any longer.
+        raises(RuntimeError, list, it)
+
 
 class FakeString(str):
     hash_count = 0
@@ -858,10 +885,9 @@
     def newtuple(self, l):
         return tuple(l)
 
-    def newdict(self, module=False, instance=False, classofinstance=None):
+    def newdict(self, module=False, instance=False):
         return W_DictMultiObject.allocate_and_init_instance(
-                self, module=module, instance=instance,
-                classofinstance=classofinstance)
+                self, module=module, instance=instance)
 
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict
@@ -941,6 +967,20 @@
         assert type(self.impl.strategy) is self.StrategyClass
         #assert self.impl.r_dict_content is None
 
+    def test_popitem(self):
+        self.fill_impl()
+        assert self.impl.length() == 2
+        a, b = self.impl.popitem()
+        assert self.impl.length() == 1
+        if a == self.string:
+            assert b == 1000
+            assert self.impl.getitem(self.string2) == 2000
+        else:
+            assert a == self.string2
+            assert b == 2000
+            assert self.impl.getitem_str(self.string) == 1000
+        self.check_not_devolved()
+
     def test_setitem(self):
         self.impl.setitem(self.string, 1000)
         assert self.impl.length() == 1
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -25,6 +25,16 @@
         key, value = NotEmpty.__dict__.popitem()
         assert (key == 'a' and value == 1) or (key == 'b' and value == 4)
 
+    def test_dictproxy_getitem(self):
+        class NotEmpty(object):
+            a = 1
+        assert 'a' in NotEmpty.__dict__
+        class substr(str): pass
+        assert substr('a') in NotEmpty.__dict__
+        assert u'a' in NotEmpty.__dict__
+        assert NotEmpty.__dict__[u'a'] == 1
+        assert u'\xe9' not in NotEmpty.__dict__
+
     def test_dictproxyeq(self):
         class a(object):
             pass
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1186,14 +1186,23 @@
         # of dicts, because the OrderedDict in the stdlib relies on this.
         # we extend the use case to lists and sets, i.e. all types that have
         # strategies, to avoid surprizes depending on the strategy.
-        for base, arg in [(list, []), (list, [5]), (list, ['x']),
-                          (set, []),  (set,  [5]), (set,  ['x']),
-                          (dict, []), (dict, [(5,6)]), (dict, [('x',7)])]:
+        class X: pass
+        for base, arg in [
+                (list, []), (list, [5]), (list, ['x']), (list, [X]),
+                (set, []),  (set,  [5]), (set,  ['x']), (set, [X]),
+                (dict, []), (dict, [(5,6)]), (dict, [('x',7)]), (dict, [(X,8)]),
+                ]:
             print base, arg
             class SubClass(base):
                 def __iter__(self):
                     return iter("foobar")
             assert list(SubClass(arg)) == ['f', 'o', 'o', 'b', 'a', 'r']
+            class Sub2(base):
+                pass
+            assert list(Sub2(arg)) == list(base(arg))
+            s = set()
+            s.update(Sub2(arg))
+            assert s == set(base(arg))
 
 class AppTestForRangeLists(AppTestW_ListObject):
 
diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -907,3 +907,30 @@
                 return [5, 3, 4][i]
         s = set([10,3,2]).intersection(Obj())
         assert list(s) == [3]
+
+    def test_iter_set_length_change(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add(7)
+        # 's' is now length 4
+        raises(RuntimeError, it.next)
+
+    def test_iter_set_strategy_only_change_1(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        class Foo(object):
+            def __eq__(self, other):
+                return False
+        assert Foo() not in s      # this changes the strategy of 'd'
+        lst = list(s)  # but iterating still works
+        assert sorted(lst) == [1, 3, 5]
+
+    def test_iter_set_strategy_only_change_2(self):
+        s = set([1, 3, 5])
+        it = iter(s)
+        s.add('foo')
+        s.remove(1)
+        # 's' is still length 3, but its strategy changed.  we are
+        # getting a RuntimeError because iterating over the old storage
+        # gives us 1, but 1 is not in the set any longer.
+        raises(RuntimeError, list, it)
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old -rf
+addopts = --assert=plain -rf
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -382,7 +382,7 @@
         pass
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 vref_None = non_virtual_ref(None)
 
diff --git a/pypy/rlib/jit_hooks.py b/pypy/rlib/jit_hooks.py
--- a/pypy/rlib/jit_hooks.py
+++ b/pypy/rlib/jit_hooks.py
@@ -22,6 +22,7 @@
                 c_name = hop.inputconst(lltype.Void, 'access_helper')
                 args_v = [hop.inputarg(arg, arg=i)
                           for i, arg in enumerate(hop.args_r)]
+                hop.exception_cannot_occur()
                 return hop.genop('jit_marker', [c_name, c_func] + args_v,
                                  resulttype=hop.r_result)
         return helper
diff --git a/pypy/rlib/longlong2float.py b/pypy/rlib/longlong2float.py
--- a/pypy/rlib/longlong2float.py
+++ b/pypy/rlib/longlong2float.py
@@ -21,7 +21,7 @@
 FLOAT_ARRAY_PTR = lltype.Ptr(lltype.Array(rffi.FLOAT))
 
 # these definitions are used only in tests, when not translated
-def longlong2float_emulator(llval):
+def longlong2float(llval):
     with lltype.scoped_alloc(DOUBLE_ARRAY_PTR.TO, 1) as d_array:
         ll_array = rffi.cast(LONGLONG_ARRAY_PTR, d_array)
         ll_array[0] = llval
@@ -51,12 +51,6 @@
 
 eci = ExternalCompilationInfo(includes=['string.h', 'assert.h'],
                               post_include_bits=["""
-static double pypy__longlong2float(long long x) {
-    double dd;
-    assert(sizeof(double) == 8 && sizeof(long long) == 8);
-    memcpy(&dd, &x, 8);
-    return dd;
-}
 static float pypy__uint2singlefloat(unsigned int x) {
     float ff;
     assert(sizeof(float) == 4 && sizeof(unsigned int) == 4);
@@ -71,12 +65,6 @@
 }
 """])
 
-longlong2float = rffi.llexternal(
-    "pypy__longlong2float", [rffi.LONGLONG], rffi.DOUBLE,
-    _callable=longlong2float_emulator, compilation_info=eci,
-    _nowrapper=True, elidable_function=True, sandboxsafe=True,
-    oo_primitive="pypy__longlong2float")
-
 uint2singlefloat = rffi.llexternal(
     "pypy__uint2singlefloat", [rffi.UINT], rffi.FLOAT,
     _callable=uint2singlefloat_emulator, compilation_info=eci,
@@ -99,4 +87,17 @@
 
     def specialize_call(self, hop):
         [v_float] = hop.inputargs(lltype.Float)
-        return hop.genop("convert_float_bytes_to_longlong", [v_float], resulttype=hop.r_result)
+        hop.exception_cannot_occur()
+        return hop.genop("convert_float_bytes_to_longlong", [v_float], resulttype=lltype.SignedLongLong)
+
+class LongLong2FloatEntry(ExtRegistryEntry):
+    _about_ = longlong2float
+
+    def compute_result_annotation(self, s_longlong):
+        assert annmodel.SomeInteger(knowntype=r_int64).contains(s_longlong)
+        return annmodel.SomeFloat()
+
+    def specialize_call(self, hop):
+        [v_longlong] = hop.inputargs(lltype.SignedLongLong)
+        hop.exception_cannot_occur()
+        return hop.genop("convert_longlong_bytes_to_float", [v_longlong], resulttype=lltype.Float)
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -215,6 +215,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
@@ -397,6 +398,7 @@
         r_obj, = hop.args_r
         v_obj, = hop.inputargs(r_obj)
         ll_fn = r_obj.get_ll_hash_function()
+        hop.exception_is_here()
         return hop.gendirectcall(ll_fn, v_obj)
 
 class Entry(ExtRegistryEntry):
@@ -419,6 +421,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_identity_hash() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_identityhash', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -441,6 +444,7 @@
             from pypy.rpython.error import TyperError
             raise TyperError("compute_unique_id() cannot be applied to"
                              " %r" % (vobj.concretetype,))
+        hop.exception_cannot_occur()
         return hop.genop('gc_id', [vobj], resulttype=lltype.Signed)
 
 class Entry(ExtRegistryEntry):
@@ -452,6 +456,7 @@
 
     def specialize_call(self, hop):
         vobj, = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         if hop.rtyper.type_system.name == 'lltypesystem':
             from pypy.rpython.lltypesystem import lltype
             if isinstance(vobj.concretetype, lltype.Ptr):
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -85,7 +85,7 @@
         s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
         assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
 
 class rbigint(object):
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -100,6 +100,7 @@
         def specialize_call(self, hop):
             bk = hop.rtyper.annotator.bookkeeper
             s_obj = identity.get_input_annotation(bk)
+            hop.exception_cannot_occur()
             return hop.r_result.rtype_erase(hop, s_obj)
 
     class Entry(ExtRegistryEntry):
@@ -110,6 +111,7 @@
             return identity.leave_tunnel(self.bookkeeper)
 
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             if hop.r_result.lowleveltype is lltype.Void:
                 return hop.inputconst(lltype.Void, None)
             [v] = hop.inputargs(hop.args_r[0])
@@ -214,6 +216,7 @@
         return hop.genop('cast_opaque_ptr', [v], resulttype=hop.r_result)
 
     def rtype_unerase_int(self, hop, v):
+        hop.exception_cannot_occur()
         return hop.gendirectcall(ll_unerase_int, v)
 
     def rtype_erase_int(self, hop):
@@ -264,6 +267,7 @@
 
     def rtype_unerase_int(self, hop, v):
         c_one = hop.inputconst(lltype.Signed, 1)
+        hop.exception_cannot_occur()
         v2 = hop.genop('oounbox_int', [v], resulttype=hop.r_result)
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -382,6 +382,7 @@
     def compute_result_annotation(self):
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_roots', [], resulttype = hop.r_result)
 
 class Entry(ExtRegistryEntry):
@@ -392,6 +393,7 @@
         return s_list_of_gcrefs()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_referents', vlist,
                          resulttype = hop.r_result)
 
@@ -402,6 +404,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_memory_usage', vlist,
                          resulttype = hop.r_result)
 
@@ -412,6 +415,7 @@
         return annmodel.SomeInteger()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
@@ -430,6 +434,7 @@
         return annmodel.SomeBool()
     def specialize_call(self, hop):
         vlist = hop.inputargs(hop.args_r[0])
+        hop.exception_cannot_occur()
         return hop.genop('gc_is_rpy_instance', vlist,
                          resulttype = hop.r_result)
 
@@ -449,6 +454,7 @@
         classrepr = getclassrepr(hop.rtyper, classdef)
         vtable = classrepr.getvtable()
         assert lltype.typeOf(vtable) == rclass.CLASSTYPE
+        hop.exception_cannot_occur()
         return Constant(vtable, concretetype=rclass.CLASSTYPE)
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rlib/rsre/rsre_re.py b/pypy/rlib/rsre/rsre_re.py
--- a/pypy/rlib/rsre/rsre_re.py
+++ b/pypy/rlib/rsre/rsre_re.py
@@ -172,8 +172,9 @@
         self._ctx = ctx
 
     def span(self, groupnum=0):
-        if not isinstance(groupnum, (int, long)):
-            groupnum = self.re.groupindex[groupnum]
+#        if not isinstance(groupnum, (int, long)):
+#            groupnum = self.re.groupindex[groupnum]
+ 
         return self._ctx.span(groupnum)
 
     def start(self, groupnum=0):
@@ -182,19 +183,25 @@
     def end(self, groupnum=0):
         return self.span(groupnum)[1]
 
-    def group(self, *groups):
-        groups = groups or (0,)
-        result = []
-        for group in groups:
-            frm, to = self.span(group)
-            if 0 <= frm <= to:
-                result.append(self._ctx._string[frm:to])
-            else:
-                result.append(None)
-        if len(result) > 1:
-            return tuple(result)
+    def group(self, group=0):
+        frm, to = self.span(group)
+        if 0 <= frm <= to:
+            return self._ctx._string[frm:to]
         else:
-            return result[0]
+            return None
+
+#    def group(self, *groups):
+#        groups = groups or (0,)
+#        result = []
+#        for group in groups:
+#            frm, to = self.span(group)
+#            if 0 <= frm <= to:
+#                result.append(self._ctx._string[frm:to])
+#            else:
+#                result.append(None)
+#        if len(result) > 1:
+#            return tuple(result)
+
 
     def groups(self, default=None):
         fmarks = self._ctx.flatten_marks()
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -204,7 +204,7 @@
         assert re.match('(a)', 'a').groups() == ('a',)
         assert re.match(r'(a)', 'a').group(0) == 'a'
         assert re.match(r'(a)', 'a').group(1) == 'a'
-        assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
+        #assert re.match(r'(a)', 'a').group(1, 1) == ('a', 'a')
 
         pat = re.compile('((a)|(b))(c)?')
         assert pat.match('a').groups() == ('a', 'a', None, None)
@@ -218,13 +218,13 @@
         assert m.group(0) == 'a'
         assert m.group(0) == 'a'
         assert m.group(1) == 'a'
-        assert m.group(1, 1) == ('a', 'a')
+        #assert m.group(1, 1) == ('a', 'a')
 
         pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?')
-        assert pat.match('a').group(1, 2, 3) == ('a', None, None)
-        assert pat.match('b').group('a1', 'b2', 'c3') == (
-                         (None, 'b', None))
-        assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
+        #assert pat.match('a').group(1, 2, 3) == ('a', None, None)
+        #assert pat.match('b').group('a1', 'b2', 'c3') == (
+        #                 (None, 'b', None))
+        #assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
     def test_bug_923(self):
         # Issue923: grouping inside optional lookahead problem
diff --git a/pypy/rlib/rsre/test/test_zinterp.py b/pypy/rlib/rsre/test/test_zinterp.py
--- a/pypy/rlib/rsre/test/test_zinterp.py
+++ b/pypy/rlib/rsre/test/test_zinterp.py
@@ -1,7 +1,8 @@
 # minimal test: just checks that (parts of) rsre can be translated
 
-from pypy.rpython.test.test_llinterp import gengraph
+from pypy.rpython.test.test_llinterp import gengraph, interpret
 from pypy.rlib.rsre import rsre_core
+from pypy.rlib.rsre.rsre_re import compile
 
 def main(n):
     assert n >= 0
@@ -19,3 +20,18 @@
 
 def test_gengraph():
     t, typer, graph = gengraph(main, [int])
+
+m = compile("(a|b)aaaaa")
+
+def test_match():
+    def f(i):
+        if i:
+            s = "aaaaaa"
+        else:
+            s = "caaaaa"
+        g = m.match(s)
+        if g is None:
+            return 3
+        return int("aaaaaa" == g.group(0))
+    assert interpret(f, [3]) == 1
+    assert interpret(f, [0]) == 3
diff --git a/pypy/rlib/rstring.py b/pypy/rlib/rstring.py
--- a/pypy/rlib/rstring.py
+++ b/pypy/rlib/rstring.py
@@ -245,5 +245,5 @@
             raise ValueError("Value is not no_nul")
 
     def specialize_call(self, hop):
-        pass
+        hop.exception_cannot_occur()
 
diff --git a/pypy/rlib/test/test_longlong2float.py b/pypy/rlib/test/test_longlong2float.py
--- a/pypy/rlib/test/test_longlong2float.py
+++ b/pypy/rlib/test/test_longlong2float.py
@@ -2,6 +2,7 @@
 from pypy.rlib.longlong2float import longlong2float, float2longlong
 from pypy.rlib.longlong2float import uint2singlefloat, singlefloat2uint
 from pypy.rlib.rarithmetic import r_singlefloat
+from pypy.rpython.test.test_llinterp import interpret
 
 
 def fn(f1):
@@ -31,6 +32,18 @@
         res = fn2(x)
         assert repr(res) == repr(x)
 
+def test_interpreted():
+    def f(f1):
+        try:
+            ll = float2longlong(f1)
+            return longlong2float(ll)
+        except Exception:
+            return 500
+
+    for x in enum_floats():
+        res = interpret(f, [x])
+        assert repr(res) == repr(x)
+
 # ____________________________________________________________
 
 def fnsingle(f1):
diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py
--- a/pypy/rpython/annlowlevel.py
+++ b/pypy/rpython/annlowlevel.py
@@ -543,11 +543,11 @@
         else:
             assert False
 
+        hop.exception_cannot_occur()
         if isinstance(hop.args_r[1], rpbc.NoneFrozenPBCRepr):
             return hop.inputconst(PTR, null)
         v_arg = hop.inputarg(hop.args_r[1], arg=1)
         assert isinstance(v_arg.concretetype, T)
-        hop.exception_cannot_occur()
         return hop.genop(opname, [v_arg], resulttype = PTR)
 
 
diff --git a/pypy/rpython/controllerentry.py b/pypy/rpython/controllerentry.py
--- a/pypy/rpython/controllerentry.py
+++ b/pypy/rpython/controllerentry.py
@@ -201,6 +201,7 @@
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
         assert hop.s_result.is_constant()
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Bool, hop.s_result.const)
 
 # ____________________________________________________________
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -130,6 +130,7 @@
 
     def specialize_call(self, hop):
         from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 def enum_ops_without_sideeffects(raising_is_ok=False):
@@ -350,6 +351,7 @@
     'truncate_longlong_to_int':LLOp(canfold=True),
     'force_cast':           LLOp(sideeffects=False),    # only for rffi.cast()
     'convert_float_bytes_to_longlong': LLOp(canfold=True),
+    'convert_longlong_bytes_to_float': LLOp(canfold=True),
 
     # __________ pointer operations __________
 
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -431,6 +431,10 @@
     from pypy.rlib.longlong2float import float2longlong
     return float2longlong(a)
 
+def op_convert_longlong_bytes_to_float(a):
+    from pypy.rlib.longlong2float import longlong2float
+    return longlong2float(a)
+
 
 def op_unichar_eq(x, y):
     assert isinstance(x, unicode) and len(x) == 1
diff --git a/pypy/rpython/lltypesystem/rbuiltin.py b/pypy/rpython/lltypesystem/rbuiltin.py
--- a/pypy/rpython/lltypesystem/rbuiltin.py
+++ b/pypy/rpython/lltypesystem/rbuiltin.py
@@ -9,6 +9,7 @@
 from pypy.rpython.rbool import bool_repr
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr or hop.args_r[1] == pyobj_repr:
@@ -33,6 +34,7 @@
     return my_instantiate()
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     s_class = hop.args_s[0]
     assert isinstance(s_class, annmodel.SomePBC)
     if len(s_class.descriptions) != 1:
@@ -46,6 +48,7 @@
     return rclass.rtype_new_instance(hop.rtyper, classdef, hop.llops)
 
 def rtype_builtin_hasattr(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(lltype.Bool, hop.s_result.const)
     if hop.args_r[0] == pyobj_repr:
@@ -56,6 +59,7 @@
     raise TyperError("hasattr is only suported on a constant or on PyObject")
 
 def rtype_builtin___import__(hop):
+    xxx    # should not be used any more
     args_v = hop.inputargs(*[pyobj_repr for ign in hop.args_r])
     c = hop.inputconst(pyobj_repr, __import__)
     return hop.genop('simple_call', [c] + args_v, resulttype = pyobj_repr)
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -746,4 +746,5 @@
         assert isinstance(TYPE, GcStruct)
         assert lltype._castdepth(TYPE, OBJECT) > 0
         hop.rtyper.set_type_for_typeptr(vtable, TYPE)
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -765,7 +765,11 @@
     def _ll_stringslice(s1, start, stop):
         lgt = stop - start
         assert start >= 0
-        assert lgt >= 0
+        # If start > stop, return a empty string. This can happen if the start
+        # is greater than the length of the string. Use < instead of <= to avoid
+        # creating another path for the JIT when start == stop.
+        if lgt < 0:
+            return s1.empty()
         newstr = s1.malloc(lgt)
         s1.copy_contents(s1, newstr, start, 0, lgt)
         return newstr
diff --git a/pypy/rpython/lltypesystem/rtuple.py b/pypy/rpython/lltypesystem/rtuple.py
--- a/pypy/rpython/lltypesystem/rtuple.py
+++ b/pypy/rpython/lltypesystem/rtuple.py
@@ -55,6 +55,7 @@
         vtup = hop.inputarg(self, 0)
         LIST = hop.r_result.lowleveltype.TO
         cno = inputconst(Signed, nitems)
+        hop.exception_is_here()
         vlist = hop.gendirectcall(LIST.ll_newlist, cno)
         v_func = hop.inputconst(Void, rlist.dum_nocheck)
         for index in range(nitems):
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1426,23 +1426,25 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
-        # If 'obj' was already forwarded, change it to its forwarding address.
-        if self.is_forwarded(obj):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
+            #
+            # Common case: 'obj' was not already forwarded (otherwise
+            # tid == -42, containing all flags), and it doesn't have the
+            # HAS_SHADOW flag either.  We must move it out of the nursery,
+            # into a new nonmovable location.
+            totalsize = size_gc_header + self.get_size(obj)
+            newhdr = self._malloc_out_of_nursery(totalsize)
+            #
+        elif self.is_forwarded(obj):
+            #
+            # 'obj' was already forwarded.  Change the original reference
+            # to point to its forwarding address, and we're done.
             root.address[0] = self.get_forwarding_address(obj)
             return
-        #
-        # First visit to 'obj': we must move it out of the nursery.
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        size = self.get_size(obj)
-        totalsize = size_gc_header + size
-        #
-        if self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
-            #
-            # Common case: allocate a new nonmovable location for it.
-            newhdr = self._malloc_out_of_nursery(totalsize)
             #
         else:
-            # The object has already a shadow.
+            # First visit to an object that has already a shadow.
             newobj = self.nursery_objects_shadows.get(obj)
             ll_assert(newobj != NULL, "GCFLAG_HAS_SHADOW but no shadow found")
             newhdr = newobj - size_gc_header
@@ -1450,6 +1452,8 @@
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
             # copied to the shadow itself.
             self.header(obj).tid &= ~GCFLAG_HAS_SHADOW
+            #
+            totalsize = size_gc_header + self.get_size(obj)
         #
         # Copy it.  Note that references to other objects in the
         # nursery are kept unchanged in this step.
diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -455,6 +455,6 @@
     return intmask(time), intmask(nsec)
 
 def time_t_to_FILE_TIME(time, filetime):
-    ft = (rffi.r_longlong(time) + secs_between_epochs) * 10000000
+    ft = rffi.r_longlong((time + secs_between_epochs) * 10000000)
     filetime.c_dwHighDateTime = rffi.r_uint(ft >> 32)
     filetime.c_dwLowDateTime = rffi.r_uint(ft)    # masking off high bits
diff --git a/pypy/rpython/module/r_os_stat.py b/pypy/rpython/module/r_os_stat.py
--- a/pypy/rpython/module/r_os_stat.py
+++ b/pypy/rpython/module/r_os_stat.py
@@ -65,4 +65,5 @@
     r_StatResult = hop.rtyper.getrepr(ll_os_stat.s_StatResult)
     [v_result] = hop.inputargs(r_StatResult.r_tuple)
     # no-op conversion from r_StatResult.r_tuple to r_StatResult
+    hop.exception_cannot_occur()
     return v_result
diff --git a/pypy/rpython/ootypesystem/ooregistry.py b/pypy/rpython/ootypesystem/ooregistry.py
--- a/pypy/rpython/ootypesystem/ooregistry.py
+++ b/pypy/rpython/ootypesystem/ooregistry.py
@@ -22,6 +22,7 @@
                                          annmodel.SomeOOInstance,
                                          annmodel.SomeString))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oostring', vlist, resulttype = ootype.String)
 
 class Entry_oounicode(ExtRegistryEntry):
@@ -38,6 +39,7 @@
         assert isinstance(hop.args_s[0], (annmodel.SomeUnicodeCodePoint,
                                           annmodel.SomeOOInstance))
         vlist = hop.inputargs(hop.args_r[0], ootype.Signed)
+        hop.exception_cannot_occur()
         return hop.genop('oounicode', vlist, resulttype = ootype.Unicode)
     
 
diff --git a/pypy/rpython/ootypesystem/rbuiltin.py b/pypy/rpython/ootypesystem/rbuiltin.py
--- a/pypy/rpython/ootypesystem/rbuiltin.py
+++ b/pypy/rpython/ootypesystem/rbuiltin.py
@@ -7,12 +7,14 @@
 from pypy.rpython.error import TyperError
 
 def rtype_new(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputargs(ootype.Void)
     return hop.genop('new', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_oonewarray(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     vlist = hop.inputarg(ootype.Void, arg=0)
     vlength = hop.inputarg(ootype.Signed, arg=1)
@@ -20,23 +22,27 @@
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_null(hop):
+    hop.exception_cannot_occur()
     assert hop.args_s[0].is_constant()
     TYPE = hop.args_s[0].const
     nullvalue = ootype.null(TYPE)
     return hop.inputconst(TYPE, nullvalue)
 
 def rtype_classof(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
     vlist = hop.inputargs(hop.args_r[0])
     return hop.genop('classof', vlist,
                      resulttype = ootype.Class)
 
 def rtype_subclassof(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr, rootype.ooclass_repr)
     return hop.genop('subclassof', vlist,
                      resulttype = ootype.Bool)
 
 def rtype_instanceof(hop):
+    hop.exception_cannot_occur()
     INSTANCE = hop.args_v[1].value
     v_inst = hop.inputarg(hop.args_r[0], arg=0)
     c_cls = hop.inputconst(ootype.Void, INSTANCE)
@@ -44,23 +50,27 @@
                      resulttype=ootype.Bool)
 
 def rtype_runtimenew(hop):
+    hop.exception_cannot_occur()
     vlist = hop.inputargs(rootype.ooclass_repr)
     return hop.genop('runtimenew', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
 def rtype_ooupcast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_oodowncast(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.Instance)
     assert isinstance(hop.args_s[1], annmodel.SomeOOInstance)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_to_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0], annmodel.SomeOOStaticMeth) or \
            isinstance(hop.args_s[0], annmodel.SomeOOClass) or \
            isinstance(hop.args_s[0].ootype, ootype.OOType)
@@ -68,12 +78,14 @@
     return hop.genop('cast_to_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_cast_from_object(hop):
+    hop.exception_cannot_occur()
     assert isinstance(hop.args_s[0].const, ootype.OOType)
     assert isinstance(hop.args_s[1], annmodel.SomeOOObject)
     v_inst = hop.inputarg(hop.args_r[1], arg=1)
     return hop.genop('cast_from_object', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 def rtype_builtin_isinstance(hop):
+    hop.exception_cannot_occur()
     if hop.s_result.is_constant():
         return hop.inputconst(ootype.Bool, hop.s_result.const)
 
@@ -99,6 +111,7 @@
     return ootype.subclassof(c1, class_)
 
 def rtype_instantiate(hop):
+    hop.exception_cannot_occur()
     if hop.args_s[0].is_constant():
 ##        INSTANCE = hop.s_result.rtyper_makerepr(hop.rtyper).lowleveltype
 ##        v_instance = hop.inputconst(ootype.Void, INSTANCE)
diff --git a/pypy/rpython/ootypesystem/rstr.py b/pypy/rpython/ootypesystem/rstr.py
--- a/pypy/rpython/ootypesystem/rstr.py
+++ b/pypy/rpython/ootypesystem/rstr.py
@@ -222,6 +222,10 @@
         length = s.ll_strlen()
         if stop > length:
             stop = length
+        # If start > stop, return a empty string. This can happen if the start
+        # is greater than the length of the string.
+        if start > stop:
+            start = stop
         return s.ll_substring(start, stop-start)
 
     def ll_stringslice_minusone(s):
diff --git a/pypy/rpython/ootypesystem/rtuple.py b/pypy/rpython/ootypesystem/rtuple.py
--- a/pypy/rpython/ootypesystem/rtuple.py
+++ b/pypy/rpython/ootypesystem/rtuple.py
@@ -39,6 +39,7 @@
         RESULT = hop.r_result.lowleveltype
         c_resulttype = inputconst(ootype.Void, RESULT)
         c_length = inputconst(ootype.Signed, len(self.items_r))
+        hop.exception_is_here()
         if isinstance(RESULT, ootype.Array):
             v_list = hop.genop('oonewarray', [c_resulttype, c_length], resulttype=RESULT)
         else:
diff --git a/pypy/rpython/rbool.py b/pypy/rpython/rbool.py
--- a/pypy/rpython/rbool.py
+++ b/pypy/rpython/rbool.py
@@ -34,6 +34,7 @@
 
     def rtype_float(_, hop):
         vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
         return vlist[0]
 
 #
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -111,25 +111,32 @@
         raise TyperError("don't know about built-in function %r" % (
             self.builtinfunc,))
 
+    def _call(self, hop2, **kwds_i):
+        bltintyper = self.findbltintyper(hop2.rtyper)
+        hop2.llops._called_exception_is_here_or_cannot_occur = False
+        v_result = bltintyper(hop2, **kwds_i)
+        if not hop2.llops._called_exception_is_here_or_cannot_occur:
+            raise TyperError("missing hop.exception_cannot_occur() or "
+                             "hop.exception_is_here() in %s" % bltintyper)
+        return v_result
+
     def rtype_simple_call(self, hop):
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
-        return bltintyper(hop2)
+        return self._call(hop2)
 
     def rtype_call_args(self, hop):
         # calling a built-in function with keyword arguments:
         # mostly for rpython.objectmodel.hint()
         hop, kwds_i = call_args_expand(hop)
 
-        bltintyper = self.findbltintyper(hop.rtyper)
         hop2 = hop.copy()
         hop2.r_s_popfirstarg()
         hop2.r_s_popfirstarg()
         # the RPython-level keyword args are passed with an 'i_' prefix and
         # the corresponding value is an *index* in the hop2 arguments,
         # to be used with hop.inputarg(arg=..)
-        return bltintyper(hop2, **kwds_i)
+        return self._call(hop2, **kwds_i)
 
 
 class BuiltinMethodRepr(Repr):
@@ -198,6 +205,7 @@
 # ____________________________________________________________
 
 def rtype_builtin_bool(hop):
+    # not called any more?
     assert hop.nb_args == 1
     return hop.args_r[0].rtype_is_true(hop)
 
@@ -241,6 +249,7 @@
 
 def rtype_builtin_min(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_min, v1, v2)
 
 def ll_min(i1, i2):
@@ -250,6 +259,7 @@
 
 def rtype_builtin_max(hop):
     v1, v2 = hop.inputargs(hop.r_result, hop.r_result)
+    hop.exception_cannot_occur()
     return hop.gendirectcall(ll_max, v1, v2)
 
 def ll_max(i1, i2):
@@ -264,6 +274,7 @@
     pass
 
 def rtype_OSError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("OSError() should not be called with "
                          "a single argument")
@@ -274,6 +285,7 @@
         r_self.setfield(v_self, 'errno', v_errno, hop.llops)
 
 def rtype_WindowsError__init__(hop):
+    hop.exception_cannot_occur()
     if hop.nb_args == 2:
         raise TyperError("WindowsError() should not be called with "
                          "a single argument")
@@ -442,6 +454,7 @@
     assert hop.args_s[0].is_constant()
     TGT = hop.args_s[0].const
     v_type, v_value = hop.inputargs(lltype.Void, hop.args_r[1])
+    hop.exception_cannot_occur()
     return gen_cast(hop.llops, TGT, v_value)
 
 _cast_to_Signed = {
@@ -523,11 +536,13 @@
 
 def rtype_identity_hash(hop):
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('gc_identityhash', vlist, resulttype=lltype.Signed)
 
 def rtype_runtime_type_info(hop):
     assert isinstance(hop.args_r[0], rptr.PtrRepr)
     vlist = hop.inputargs(hop.args_r[0])
+    hop.exception_cannot_occur()
     return hop.genop('runtime_type_info', vlist,
                      resulttype = hop.r_result.lowleveltype)
 
@@ -558,6 +573,7 @@
 
 def rtype_raw_malloc(hop):
     v_size, = hop.inputargs(lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_malloc', [v_size], resulttype=llmemory.Address)
 
 def rtype_raw_malloc_usage(hop):
@@ -586,6 +602,7 @@
     if s_addr.is_null_address():
         raise TyperError("raw_memclear(x, n) where x is the constant NULL")
     v_list = hop.inputargs(llmemory.Address, lltype.Signed)
+    hop.exception_cannot_occur()
     return hop.genop('raw_memclear', v_list)
 
 BUILTIN_TYPER[llmemory.raw_malloc] = rtype_raw_malloc
@@ -596,6 +613,7 @@
 
 def rtype_offsetof(hop):
     TYPE, field = hop.inputargs(lltype.Void, lltype.Void)
+    hop.exception_cannot_occur()
     return hop.inputconst(lltype.Signed,
                           llmemory.offsetof(TYPE.value, field.value))
 
@@ -605,6 +623,7 @@
 # non-gc objects
 
 def rtype_free_non_gc_object(hop):
+    hop.exception_cannot_occur()
     vinst, = hop.inputargs(hop.args_r[0])
     flavor = hop.args_r[0].gcflavor
     assert flavor != 'gc'
@@ -617,6 +636,7 @@
 # keepalive_until_here
 
 def rtype_keepalive_until_here(hop):
+    hop.exception_cannot_occur()
     for v in hop.args_v:
         hop.genop('keepalive', [v], resulttype=lltype.Void)
     return hop.inputconst(lltype.Void, None)
diff --git a/pypy/rpython/rfloat.py b/pypy/rpython/rfloat.py
--- a/pypy/rpython/rfloat.py
+++ b/pypy/rpython/rfloat.py
@@ -136,7 +136,10 @@
         hop.exception_cannot_occur()
         return hop.genop('cast_float_to_int', vlist, resulttype=Signed)
 
-    rtype_float = rtype_pos
+    def rtype_float(_, hop):
+        vlist = hop.inputargs(Float)
+        hop.exception_cannot_occur()
+        return vlist[0]
 
     # version picked by specialisation based on which
     # type system rtyping is using, from <type_system>.ll_str module
diff --git a/pypy/rpython/rint.py b/pypy/rpython/rint.py
--- a/pypy/rpython/rint.py
+++ b/pypy/rpython/rint.py
@@ -310,6 +310,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_chr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_char', vlist, resulttype=Char)
 
     def rtype_unichr(_, hop):
@@ -317,6 +319,8 @@
         if hop.has_implicit_exception(ValueError):
             hop.exception_is_here()
             hop.gendirectcall(ll_check_unichr, vlist[0])
+        else:
+            hop.exception_cannot_occur()
         return hop.genop('cast_int_to_unichar', vlist, resulttype=UniChar)
 
     def rtype_is_true(self, hop):
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -115,6 +115,7 @@
     def rtype_bltn_list(self, hop):
         v_lst = hop.inputarg(self, 0)
         cRESLIST = hop.inputconst(Void, hop.r_result.LIST)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_copy, cRESLIST, v_lst)
 
     def rtype_len(self, hop):
diff --git a/pypy/rpython/rrange.py b/pypy/rpython/rrange.py
--- a/pypy/rpython/rrange.py
+++ b/pypy/rpython/rrange.py
@@ -107,8 +107,10 @@
     if isinstance(hop.r_result, AbstractRangeRepr):
         if hop.r_result.step != 0:
             c_rng = hop.inputconst(Void, hop.r_result.RANGE)
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrange, c_rng, vstart, vstop)
         else:
+            hop.exception_is_here()
             return hop.gendirectcall(hop.r_result.ll_newrangest, vstart, vstop, vstep)
     else:
         # cannot build a RANGE object, needs a real list
@@ -117,6 +119,7 @@
         if isinstance(ITEMTYPE, Ptr):
             ITEMTYPE = ITEMTYPE.TO
         cLIST = hop.inputconst(Void, ITEMTYPE)
+        hop.exception_is_here()
         return hop.gendirectcall(ll_range2list, cLIST, vstart, vstop, vstep)
 
 rtype_builtin_xrange = rtype_builtin_range
@@ -212,4 +215,5 @@
                                      [v_index, v_item])
 
 def rtype_builtin_enumerate(hop):
+    hop.exception_cannot_occur()
     return hop.r_result.r_baseiter.newiter(hop)
diff --git a/pypy/rpython/rstr.py b/pypy/rpython/rstr.py
--- a/pypy/rpython/rstr.py
+++ b/pypy/rpython/rstr.py
@@ -288,6 +288,8 @@
 
     def rtype_unicode(self, hop):
         if hop.args_s[0].is_constant():
+            # convertion errors occur during annotation, so cannot any more:
+            hop.exception_cannot_occur()
             return hop.inputconst(hop.r_result, hop.s_result.const)
         repr = hop.args_r[0].repr
         v_str = hop.inputarg(repr, 0)
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -846,6 +846,7 @@
         return result
 
     def exception_is_here(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
@@ -861,6 +862,7 @@
         self.llops.llop_raising_exceptions = len(self.llops)
 
     def exception_cannot_occur(self):
+        self.llops._called_exception_is_here_or_cannot_occur = True
         if self.llops.llop_raising_exceptions is not None:
             raise TyperError("cannot catch an exception at more than one llop")
         if not self.exceptionlinks:
diff --git a/pypy/rpython/test/test_extregistry.py b/pypy/rpython/test/test_extregistry.py
--- a/pypy/rpython/test/test_extregistry.py
+++ b/pypy/rpython/test/test_extregistry.py
@@ -114,6 +114,7 @@
         _about_ = dummy_func
         s_result_annotation = annmodel.SomeInteger()
         def specialize_call(self, hop):
+            hop.exception_cannot_occur()
             return hop.inputconst(lltype.Signed, 42)
 
     def func():
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1085,6 +1085,7 @@
                 return annmodel.SomeInteger()
             def specialize_call(self, hop):
                 [v_instance] = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.gendirectcall(ll_my_gethash, v_instance)
 
         def f(n):
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -477,7 +477,11 @@
             s1 = s[:3]
             s2 = s[3:]
             s3 = s[3:10]
-            return s1+s2 == s and s2+s1 == const('lohel') and s1+s3 == s
+            s4 = s[42:44]
+            return (s1+s2 == s and
+                    s2+s1 == const('lohel') and
+                    s1+s3 == s and
+                    s4 == const(''))
         res = self.interpret(fn, [0])
         assert res
 
diff --git a/pypy/test_all.py b/pypy/test_all.py
old mode 100755
new mode 100644
--- a/pypy/test_all.py
+++ b/pypy/test_all.py
@@ -11,11 +11,12 @@
 """
 import sys, os
 
-if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
-    print >> sys.stderr, __doc__
-    sys.exit(2)
 
 if __name__ == '__main__':
+    if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
+        print >> sys.stderr, __doc__
+        sys.exit(2)
+
     import tool.autopath
     import pytest
     import pytest_cov
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -38,7 +38,7 @@
 closed_heads.reverse()
 
 for head, branch in closed_heads:
-    print '\t', branch
+    print '\t', head, '\t', branch
 print
 print 'The branches listed above will be merged to "closed-branches".'
 print 'You need to run this script in a clean working copy where you'
diff --git a/pypy/tool/jitlogparser/test/__init__.py b/pypy/tool/jitlogparser/test/__init__.py
new file mode 100644
diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -165,7 +165,10 @@
     def test_one(self): exec 'blow'
     """)
 
-    ev, = sorter.getreports("pytest_runtest_logreport")
+    reports = sorter.getreports("pytest_runtest_logreport")
+    setup, ev, teardown = reports
     assert ev.failed
+    assert setup.passed
+    assert teardown.passed
     assert 'NameError' in ev.longrepr.reprcrash.message
     assert 'blow' in ev.longrepr.reprcrash.message
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -7,10 +7,17 @@
 from pypy import conftest
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.translator.platform import platform as compiler
+from pypy.rlib.rarithmetic import is_emulated_long
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib.entrypoint import entrypoint, secondary_entrypoints
 from pypy.rpython.lltypesystem.lloperation import llop
 
+_MSVC = compiler.name == "msvc"
+_MINGW = compiler.name == "mingw32"
+_WIN32 = _MSVC or _MINGW
+_WIN64 = _WIN32 and is_emulated_long
+# XXX get rid of 'is_emulated_long' and have a real config here.
+
 class AbstractTestAsmGCRoot:
     # the asmgcroot gc transformer doesn't generate gc_reload_possibly_moved
     # instructions:
@@ -18,8 +25,8 @@
 
     @classmethod
     def make_config(cls):
-        if compiler.name == "msvc":
-            py.test.skip("all asmgcroot tests disabled for MSVC")
+        if _MSVC and _WIN64:
+            py.test.skip("all asmgcroot tests disabled for MSVC X64")
         from pypy.config.pypyoption import get_pypy_config
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -847,6 +847,10 @@
                 if sources:
                     target, = sources
 
+        if target.endswith('@PLT'):
+            # In -fPIC mode, all functions calls have this suffix
+            target = target[:-4]
+
         if target in self.FUNCTIONS_NOT_RETURNING:
             return [InsnStop(target)]
         if self.format == 'mingw32' and target == '__alloca':
@@ -1137,7 +1141,7 @@
     r_jump_rel_label = re.compile(r"\tj\w+\s+"+"(\d+)f"+"\s*$")
 
     r_unaryinsn_star= re.compile(r"\t[a-z]\w*\s+[*]("+OPERAND+")\s*$")
-    r_jmptable_item = re.compile(r"\t.quad\t"+LABEL+"(-\"[A-Za-z0-9$]+\")?\s*$")
+    r_jmptable_item = re.compile(r"\t.(?:quad|long)\t"+LABEL+"(-\"[A-Za-z0-9$]+\"|-"+LABEL+")?\s*$")
     r_jmptable_end  = re.compile(r"\t.text|\t.section\s+.text|\t\.align|"+LABEL)
 
     r_gcroot_marker = re.compile(r"\t/[*] GCROOT ("+LOCALVARFP+") [*]/")
diff --git a/pypy/translator/c/src/float.h b/pypy/translator/c/src/float.h
--- a/pypy/translator/c/src/float.h
+++ b/pypy/translator/c/src/float.h
@@ -43,5 +43,6 @@
 #define OP_CAST_FLOAT_TO_LONGLONG(x,r) r = (long long)(x)
 #define OP_CAST_FLOAT_TO_ULONGLONG(x,r) r = (unsigned long long)(x)
 #define OP_CONVERT_FLOAT_BYTES_TO_LONGLONG(x,r) memcpy(&r, &x, sizeof(double))
+#define OP_CONVERT_LONGLONG_BYTES_TO_FLOAT(x,r) memcpy(&r, &x, sizeof(long long))
 #endif
 
diff --git a/pypy/translator/c/test/test_extfunc.py b/pypy/translator/c/test/test_extfunc.py
--- a/pypy/translator/c/test/test_extfunc.py
+++ b/pypy/translator/c/test/test_extfunc.py
@@ -919,4 +919,5 @@
             t, cbuilder = self.compile(does_stuff)
             data = cbuilder.cmdexec('')
             res = os.nice(0) + 3
+            if res > 19: res = 19    # xxx Linux specific, probably
             assert data.startswith('os.nice returned %d\n' % res)
diff --git a/pypy/translator/cli/dotnet.py b/pypy/translator/cli/dotnet.py
--- a/pypy/translator/cli/dotnet.py
+++ b/pypy/translator/cli/dotnet.py
@@ -459,6 +459,7 @@
             
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         assert hop.args_s[1].is_constant()
         TYPE = hop.args_s[1].const
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
@@ -507,6 +508,7 @@
 
     def specialize_call(self, hop):
         v_obj, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('same_as', [v_obj], hop.r_result.lowleveltype)
 
 def new_array(type, length):
@@ -608,6 +610,7 @@
 
     def specialize_call(self, hop):
         v_type, = hop.inputargs(*hop.args_r)
+        hop.exception_cannot_occur()
         return hop.genop('cli_typeof', [v_type], hop.r_result.lowleveltype)
 
 
@@ -626,6 +629,7 @@
         v_obj, = hop.inputargs(*hop.args_r)
         methodname = hop.args_r[0].methodname
         c_methodname = hop.inputconst(ootype.Void, methodname)
+        hop.exception_cannot_occur()
         return hop.genop('cli_eventhandler', [v_obj, c_methodname], hop.r_result.lowleveltype)
 
 
@@ -647,6 +651,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -668,6 +673,7 @@
     def specialize_call(self, hop):
         assert isinstance(hop.args_s[0], annmodel.SomeOOInstance)
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('ooupcast', [v_inst], resulttype = hop.r_result.lowleveltype)
 
 
@@ -701,6 +707,7 @@
 
     def specialize_call(self, hop):
         v_obj = hop.inputarg(hop.args_r[0], arg=0)
+        hop.exception_cannot_occur()
         return hop.genop('oodowncast', [v_obj], hop.r_result.lowleveltype)
 
 
diff --git a/pypy/translator/jvm/opcodes.py b/pypy/translator/jvm/opcodes.py
--- a/pypy/translator/jvm/opcodes.py
+++ b/pypy/translator/jvm/opcodes.py
@@ -243,4 +243,5 @@
     'force_cast':               [PushAllArgs, CastPrimitive, StoreResult],
 
     'convert_float_bytes_to_longlong': jvm.PYPYDOUBLEBYTESTOLONG,
+    'convert_longlong_bytes_to_float': jvm.PYPYLONGBYTESTODOUBLE,
 })
diff --git a/pypy/translator/jvm/typesystem.py b/pypy/translator/jvm/typesystem.py
--- a/pypy/translator/jvm/typesystem.py
+++ b/pypy/translator/jvm/typesystem.py
@@ -942,6 +942,7 @@
 PYPYULONGTODOUBLE =     Method.s(jPyPy, 'ulong_to_double', (jLong,), jDouble)
 PYPYLONGBITWISENEGATE = Method.v(jPyPy, 'long_bitwise_negate', (jLong,), jLong)
 PYPYDOUBLEBYTESTOLONG = Method.v(jPyPy, 'pypy__float2longlong', (jDouble,), jLong)
+PYPYLONGBYTESTODOUBLE = Method.v(jPyPy, 'pypy__longlong2float', (jLong,), jDouble)
 PYPYSTRTOINT =          Method.v(jPyPy, 'str_to_int', (jString,), jInt)
 PYPYSTRTOUINT =         Method.v(jPyPy, 'str_to_uint', (jString,), jInt)
 PYPYSTRTOLONG =         Method.v(jPyPy, 'str_to_long', (jString,), jLong)
diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -110,7 +110,10 @@
                  do_dry_run=False, timeout=None,
                  _win32=(sys.platform=='win32')):
     args = interp + test_driver
-    args += ['-p', 'resultlog', '--resultlog=%s' % logfname, test]
+    args += ['-p', 'resultlog',
+             '--resultlog=%s' % logfname,
+             '--junitxml=%s.junit' % logfname,
+             test]
 
     args = map(str, args)
     interp0 = args[0]
diff --git a/testrunner/scratchbox_runner.py b/testrunner/scratchbox_runner.py
--- a/testrunner/scratchbox_runner.py
+++ b/testrunner/scratchbox_runner.py
@@ -14,14 +14,14 @@
 def dry_run_scratchbox(args, cwd, out, timeout=None):
     return dry_run(args_for_scratchbox(cwd, args), cwd, out, timeout)
 
-import runner
-# XXX hack hack hack
-dry_run = runner.dry_run
-run = runner.run
+if __name__ == '__main__':
+    import runner
+    # XXX hack hack hack
+    dry_run = runner.dry_run
+    run = runner.run
 
-runner.dry_run = dry_run_scratchbox
-runner.run = run_scratchbox
+    runner.dry_run = dry_run_scratchbox
+    runner.run = run_scratchbox
 
-if __name__ == '__main__':
     import sys
     runner.main(sys.argv)
diff --git a/testrunner/test/conftest.py b/testrunner/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/testrunner/test/conftest.py
@@ -0,0 +1,6 @@
+
+def pytest_runtest_makereport(__multicall__, item):
+    report = __multicall__.execute()
+    if 'out' in item.funcargs:
+        report.sections.append(('out', item.funcargs['out'].read()))
+    return report
diff --git a/testrunner/test/test_runner.py b/testrunner/test/test_runner.py
--- a/testrunner/test/test_runner.py
+++ b/testrunner/test/test_runner.py
@@ -53,49 +53,44 @@
     assert not should_report_failure("F Def\n. Ghi\n. Jkl\n")
 
 
+
 class TestRunHelper(object):
+    def pytest_funcarg__out(self, request):
+        tmpdir = request.getfuncargvalue('tmpdir')
+        return tmpdir.ensure('out')
 
-    def setup_method(self, meth):
-        h, self.fn = tempfile.mkstemp()
-        os.close(h)
+    def test_run(self, out):
+        res = runner.run([sys.executable, "-c", "print 42"], '.', out)
+        assert res == 0
+        assert out.read() == "42\n"
 
-    def teardown_method(self, meth):
-        os.unlink(self.fn)
-
-    def test_run(self):
-        res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn))
-        assert res == 0
-        out = py.path.local(self.fn).read('r')
-        assert out == "42\n"
-
-    def test_error(self):
-        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', py.path.local(self.fn))
+    def test_error(self, out):
+        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', out)
         assert res == 3
 
-    def test_signal(self):
+    def test_signal(self, out):
         if sys.platform == 'win32':
             py.test.skip("no death by signal on windows")
-        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', py.path.local(self.fn))
+        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', out)
         assert res == -9
 
-    def test_timeout(self):
-        res = runner.run([sys.executable, "-c", "while True: pass"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout(self, out):
+        res = runner.run([sys.executable, "-c", "while True: pass"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_lock(self):
-        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_lock(self, out):
+        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_syscall(self):
-        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_syscall(self, out):
+        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', out, timeout=3)
         assert res == -999        
 
-    def test_timeout_success(self):
+    def test_timeout_success(self, out):
         res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn), timeout=2)
+                         out, timeout=2)
         assert res == 0
-        out = py.path.local(self.fn).read('r')
+        out = out.read()
         assert out == "42\n"        
 
 
@@ -122,7 +117,10 @@
 
         expected = ['INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
+
                     'test_one']
 
         assert self.called == (expected, '/wd', 'out', 'secs')        
@@ -138,9 +136,11 @@
 
         expected = ['/wd' + os.sep + './INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
                     'test_one']
-
+        assert self.called[0] == expected
         assert self.called == (expected, '/wd', 'out', 'secs')        
         assert res == 0
 
@@ -251,7 +251,7 @@
         assert '\n' in log        
         log_lines = log.splitlines()
 
-        assert log_lines[0] == ". test_normal/test_example.py:test_one"
+        assert ". test_normal/test_example.py::test_one" in log_lines
         nfailures = 0
         noutcomes = 0
         for line in log_lines:

From noreply at buildbot.pypy.org  Fri Apr 13 11:00:54 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 13 Apr 2012 11:00:54 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: implement
	convert_longlong_bytes_to_float
Message-ID: <20120413090054.7976C82F4E@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54327:ad3a31a28410
Date: 2012-04-13 08:03 +0000
http://bitbucket.org/pypy/pypy/changeset/ad3a31a28410/

Log:	implement convert_longlong_bytes_to_float

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -1299,6 +1299,7 @@
         return fcond
 
     emit_op_convert_float_bytes_to_longlong = gen_emit_unary_float_op('float_bytes_to_longlong', 'VMOV_cc')
+    emit_op_convert_longlong_bytes_to_float = gen_emit_unary_float_op('longlong_bytes_to_float', 'VMOV_cc')
 
     def emit_op_read_timestamp(self, op, arglocs, regalloc, fcond):
         tmp = arglocs[0]
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -1206,6 +1206,8 @@
 
     prepare_op_convert_float_bytes_to_longlong = prepare_float_op(base=False,
                               name='prepare_op_convert_float_bytes_to_longlong')
+    prepare_op_convert_longlong_bytes_to_float = prepare_float_op(base=False,
+                              name='prepare_op_convert_longlong_bytes_to_float')
 
     def prepare_op_read_timestamp(self, op, fcond):
         loc = self.get_scratch_reg(INT)

From noreply at buildbot.pypy.org  Fri Apr 13 11:00:55 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 13 Apr 2012 11:00:55 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: fix math_sqrt
Message-ID: <20120413090055.C648182F4E@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r54328:6bac78dc2ff5
Date: 2012-04-13 08:06 +0000
http://bitbucket.org/pypy/pypy/changeset/6bac78dc2ff5/

Log:	fix math_sqrt

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -934,7 +934,7 @@
     def regalloc_emit_math(self, op, arglocs, fcond, regalloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
-        asm_math_operations[oopspecindex](self, op, arglocs, resloc)
+        asm_math_operations[oopspecindex](self, op, arglocs, regalloc, fcond)
         return fcond
 
 

From noreply at buildbot.pypy.org  Fri Apr 13 12:52:59 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 12:52:59 +0200 (CEST)
Subject: [pypy-commit] pypy pytest: apply patch from upstream that fixes
	armins py25 issue
Message-ID: <20120413105259.E012582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: pytest
Changeset: r54329:9b3c28e1af6c
Date: 2012-04-13 12:46 +0200
http://bitbucket.org/pypy/pypy/changeset/9b3c28e1af6c/

Log:	apply patch from upstream that fixes armins py25 issue

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was

From noreply at buildbot.pypy.org  Fri Apr 13 12:53:01 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 12:53:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge from pytest
Message-ID: <20120413105301.400F082F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54330:8bafd99b776b
Date: 2012-04-13 12:47 +0200
http://bitbucket.org/pypy/pypy/changeset/8bafd99b776b/

Log:	merge from pytest

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was

From noreply at buildbot.pypy.org  Fri Apr 13 12:53:02 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 12:53:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge upstream
Message-ID: <20120413105302.CEE1D82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54331:f7ddf20f141f
Date: 2012-04-13 12:52 +0200
http://bitbucket.org/pypy/pypy/changeset/f7ddf20f141f/

Log:	merge upstream

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -169,9 +169,11 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +188,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +226,26 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
+        if self.keywords is None:
+            self.keywords = keywords[:] # copy to make non-resizable
+            self.keywords_w = keywords_w[:]
+        else:
+            # looks quadratic, but the JIT should remove all of it nicely.
+            # Also, all the lists should be small
+            for key in keywords:
+                for otherkey in self.keywords:
+                    if otherkey == key:
+                        raise operationerrfmt(self.space.w_TypeError,
+                                              "got multiple values "
+                                              "for keyword argument "
+                                              "'%s'", key)
+            self.keywords = self.keywords + keywords
+            self.keywords_w = self.keywords_w + keywords_w
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +404,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+

From noreply at buildbot.pypy.org  Fri Apr 13 16:06:26 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 13 Apr 2012 16:06:26 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Intermediate check-in.
Message-ID: <20120413140626.33D6982F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54332:2abd27c473de
Date: 2012-04-13 15:21 +0200
http://bitbucket.org/pypy/pypy/changeset/2abd27c473de/

Log:	Intermediate check-in.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -33,7 +33,10 @@
 #   - The LOCAL objects might be YOUNG or OLD depending on whether they
 #     already survived a collection.  YOUNG LOCAL objects are either in
 #     the nursery or, if they are big, raw-malloced.  OLD LOCAL objects
-#     are in the shared area.
+#     are in the shared area.  Getting the write barrier right for both
+#     this and the general STM mechanisms is tricky, so for now this GC
+#     is not actually generational (slow when running long transactions
+#     or before running transactions at all).
 #
 GCFLAG_GLOBAL     = first_gcflag << 0     # keep in sync with et.c
 GCFLAG_WAS_COPIED = first_gcflag << 1     # keep in sync with et.c
@@ -65,7 +68,7 @@
 
     TRANSLATION_PARAMS = {
         'stm_operations': 'use_real_one',
-        'nursery_size': 4*1024*1024,            # 4 MB
+        'nursery_size': 32*1024*1024,           # 32 MB
 
         "page_size": 1024*WORD,                 # copied from minimark.py
         "arena_size": 65536*WORD,               # copied from minimark.py
@@ -120,32 +123,20 @@
         self.main_thread_tls = StmGCTLS(self, in_main_thread=True)
         self.main_thread_tls.start_transaction()
 
+    @always_inline
     def get_tls(self):
         from pypy.rpython.memory.gc.stmtls import StmGCTLS
         tls = self.stm_operations.get_tls()
         return StmGCTLS.cast_address_to_tls_object(tls)
 
+    def enter_transactional_mode(self):
+        self.main_thread_tls.enter_transactional_mode()
+
+    def leave_transactional_mode(self):
+        self.main_thread_tls.leave_transactional_mode()
+
     # ----------
 
-    @always_inline
-    def allocate_bump_pointer(self, size):
-        tls = self.collector.get_tls()
-        free = tls.nursery_free
-        top  = tls.nursery_top
-        if (top - free) < llmemory.raw_malloc_usage(size):
-            free = self.local_collection(size)
-        tls.nursery_free = free + size
-        return free
-
-    @dont_inline
-    def local_collection(self, size):
-        tls = self.collector.get_tls()
-        if not tls.nursery_free:
-            fatalerror("malloc in a non-main thread but outside a transaction")
-        #...
-        xxxxxxxxx
-
-
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False,
                                is_finalizer_light=False,
@@ -159,7 +150,7 @@
         # Get the memory from the nursery.
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
-        result = self.allocate_bump_pointer(totalsize)
+        result = self.get_tls().allocate_bump_pointer(totalsize)
         #
         # Build the object.
         llarena.arena_reserve(result, totalsize)
@@ -180,7 +171,7 @@
         nonvarsize = size_gc_header + size
         totalsize = nonvarsize + itemsize * length
         totalsize = llarena.round_up_for_allocation(totalsize)
-        result = self.allocate_bump_pointer(totalsize)
+        result = self.get_tls().allocate_bump_pointer(totalsize)
         llarena.arena_reserve(result, totalsize)
         obj = result + size_gc_header
         self.init_gc_object(result, typeid, flags=0)
@@ -315,7 +306,7 @@
         def _stm_write_barrier_global(obj):
             if not stm_operations.in_transaction():
                 return obj
-            # we need to find of make a local copy
+            # we need to find or make a local copy
             hdr = self.header(obj)
             if hdr.tid & GCFLAG_WAS_COPIED == 0:
                 # in this case, we are sure that we don't have a copy
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -3,9 +3,11 @@
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, base_ptr_lltype
 from pypy.rlib.objectmodel import we_are_translated, free_non_gc_object
 from pypy.rlib.rarithmetic import r_uint
-from pypy.rlib.debug import ll_assert
+from pypy.rlib.debug import ll_assert, debug_start, debug_stop, fatalerror
 
 from pypy.rpython.memory.gc.stmgc import WORD, NULL
+from pypy.rpython.memory.gc.stmgc import always_inline, dont_inline
+from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL
 
 
 class StmGCTLS(object):
@@ -34,21 +36,18 @@
         self.nursery_size  = self.gc.nursery_size
         self.nursery_start = self._alloc_nursery(self.nursery_size)
         #
-        # --- the local raw-malloced objects, young and old
-        self.rawmalloced_young_objects = self.null_address_dict()
-        self.rawmalloced_old_objects = None
-        self.rawmalloced_total_size = r_uint(0)
-        # --- the local objects with weakrefs, young and old
-        self.young_objects_with_weakrefs = self.AddressStack()
-        self.old_objects_with_weakrefs = self.AddressStack()
-        # --- support for id and identityhash: maps nursery objects with
-        #     GCFLAG_HAS_SHADOW to their future location at the next
-        #     local collection
-        self.nursery_objects_shadows = self.AddressDict()
+        # --- the local raw-malloced objects (chained list via hdr.version)
+        self.rawmalloced_objects = NULL
+        # --- the local "normal" old objects (chained list via hdr.version)
+        self.old_objects = NULL
+        # --- the local objects with weakrefs (chained list via hdr.version)
+        #self.young_objects_with_weakrefs = NULL
+        #self.old_objects_with_weakrefs = NULL
         #
         self._register_with_C_code()
 
     def teardown_thread(self):
+        self._cleanup_state()
         self._unregister_with_C_code()
         self._free_nursery(self.nursery_start)
         free_non_gc_object(self)
@@ -67,9 +66,9 @@
             tls = cast_instance_to_base_ptr(self)
             tlsaddr = llmemory.cast_ptr_to_adr(tls)
         else:
-            n = 10000 + len(self.nontranslated_dict)
+            n = 10000 + len(StmGCTLS.nontranslated_dict)
             tlsaddr = rffi.cast(llmemory.Address, n)
-            self.nontranslated_dict[n] = self
+            StmGCTLS.nontranslated_dict[n] = self
         self.stm_operations.set_tls(tlsaddr, int(self.in_main_thread))
 
     def _unregister_with_C_code(self):
@@ -78,16 +77,40 @@
         self.stm_operations.del_tls()
 
     @staticmethod
-    def cast_address_to_tls_object(self, tlsaddr):
+    @always_inline
+    def cast_address_to_tls_object(tlsaddr):
         if we_are_translated():
             tls = llmemory.cast_adr_to_ptr(tlsaddr, base_ptr_lltype())
             return cast_base_ptr_to_instance(tls)
         else:
             n = rffi.cast(lltype.Signed, tlsaddr)
-            return self.nontranslated_dict[n]
+            return StmGCTLS.nontranslated_dict[n]
+
+    def _disable_mallocs(self):
+        ll_assert(bool(self.nursery_free), "disable_mallocs: already disabled")
+        self.nursery_pending_clear = self.nursery_free - self.nursery_start
+        self.nursery_free = NULL
+        self.nursery_top  = NULL
 
     # ------------------------------------------------------------
 
+    def enter_transactional_mode(self):
+        """Called on the main thread, just before spawning the other
+        threads."""
+        self.local_collection()
+        if not self.local_nursery_is_empty():
+            self.local_collection(run_finalizers=False)
+        self._promote_locals_to_globals()
+        self._disable_mallocs()
+
+    def leave_transactional_mode(self):
+        """Restart using the main thread for mallocs."""
+        if not we_are_translated():
+            for key, value in StmGCTLS.nontranslated_dict.items():
+                if value is not self:
+                    del StmGCTLS.nontranslated_dict[key]
+        self.start_transaction()
+
     def start_transaction(self):
         """Enter a thread: performs any pending cleanups, and set
         up a fresh state for allocating.  Called at the start of
@@ -96,29 +119,88 @@
         # end_of_transaction_collection() are not balanced: if a
         # transaction is aborted, the latter might never be called.
         # Be ready here to clean up any state.
+        self._cleanup_state()
         if self.nursery_free:
             clear_size = self.nursery_free - self.nursery_start
         else:
             clear_size = self.nursery_pending_clear
+        self.nursery_pending_clear = 0
         if clear_size > 0:
             llarena.arena_reset(self.nursery_start, clear_size, 2)
-            self.nursery_pending_clear = 0
-        if self.rawmalloced_young_objects:
-            xxx
-        if self.rawmalloced_old_objects:
-            xxx
         self.nursery_free = self.nursery_start
         self.nursery_top  = self.nursery_start + self.nursery_size
 
+    def local_nursery_is_empty(self):
+        ll_assert(self.nursery_free, "local_nursery_is_empty: gc not running")
+        return self.nursery_free == self.nursery_start
+
     # ------------------------------------------------------------
 
-    def local_collection(self):
+    def local_collection(self, run_finalizers=True):
         """Do a local collection.  Finds all surviving young objects
         and make them old.  Also looks for roots from the stack.
         The flag GCFLAG_WAS_COPIED is kept and the C tree is updated
         if the local young object moves.
         """
-        xxx
+        #
+        debug_start("gc-local")
+        #
+        # First, find the roots that point to young objects.  All nursery
+        # objects found are copied out of the nursery, and the occasional
+        # young raw-malloced object is flagged with GCFLAG_VISITED.
+        # Note that during this step, we ignore references to further
+        # young objects; only objects directly referenced by roots
+        # are copied out or flagged.  They are also added to the list
+        # 'old_objects_pointing_to_young'.
+        self.collect_roots_in_nursery()
+        #
+        while True:
+            # If we are using card marking, do a partial trace of the arrays
+            # that are flagged with GCFLAG_CARDS_SET.
+            if self.card_page_indices > 0:
+                self.collect_cardrefs_to_nursery()
+            #
+            # Now trace objects from 'old_objects_pointing_to_young'.
+            # All nursery objects they reference are copied out of the
+            # nursery, and again added to 'old_objects_pointing_to_young'.
+            # All young raw-malloced object found are flagged GCFLAG_VISITED.
+            # We proceed until 'old_objects_pointing_to_young' is empty.
+            self.collect_oldrefs_to_nursery()
+            #
+            # We have to loop back if collect_oldrefs_to_nursery caused
+            # new objects to show up in old_objects_with_cards_set
+            if self.card_page_indices > 0:
+                if self.old_objects_with_cards_set.non_empty():
+                    continue
+            break
+        #
+        # Now all live nursery objects should be out.  Update the young
+        # weakrefs' targets.
+        if self.young_objects_with_weakrefs.non_empty():
+            self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
+        #
+        # Clear this mapping.
+        if self.nursery_objects_shadows.length() > 0:
+            self.nursery_objects_shadows.clear()
+        #
+        # Walk the list of young raw-malloced objects, and either free
+        # them or make them old.
+        if self.young_rawmalloced_objects:
+            self.free_young_rawmalloced_objects()
+        #
+        # All live nursery objects are out, and the rest dies.  Fill
+        # the whole nursery with zero and reset the current nursery pointer.
+        llarena.arena_reset(self.nursery, self.nursery_size, 2)
+        self.debug_rotate_nursery()
+        self.nursery_free = self.nursery
+        #
+        debug_print("minor collect, total memory used:",
+                    self.get_total_memory_used())
+        if self.DEBUG >= 2:
+            self.debug_check_consistency()     # expensive!
+        debug_stop("gc-minor")
 
     def end_of_transaction_collection(self):
         """Do an end-of-transaction collection.  Finds all surviving
@@ -133,3 +215,49 @@
         xxx
 
     # ------------------------------------------------------------
+
+    @always_inline
+    def allocate_bump_pointer(self, size):
+        free = self.nursery_free
+        top  = self.nursery_top
+        if (top - free) < llmemory.raw_malloc_usage(size):
+            free = self.allocate_object_of_size(size)
+        self.nursery_free = free + size
+        return free
+
+    @dont_inline
+    def allocate_object_of_size(self, size):
+        if not self.nursery_free:
+            fatalerror("malloc in a non-main thread but outside a transaction")
+        if size > self.nursery_size:
+            fatalerror("object too large to ever fit in the nursery")
+        while True:
+            self.local_collection()
+            free = self.nursery_free
+            top  = self.nursery_top
+            if (top - free) < llmemory.raw_malloc_usage(size):
+                continue         # try again
+            return free
+
+    # ------------------------------------------------------------
+
+    def _promote_locals_to_globals(self):
+        ll_assert(self.local_nursery_is_empty(), "nursery must be empty [1]")
+        #
+        obj = self.old_objects
+        self.old_objects = NULL
+        while obj:
+            hdr = self.header(obj)
+            hdr.tid |= GCFLAG_GLOBAL
+            obj = hdr.version
+        #
+        obj = self.rawmalloced_objects
+        self.rawmalloced_objects = NULL
+        while obj:
+            hdr = self.header(obj)
+            hdr.tid |= GCFLAG_GLOBAL
+            obj = hdr.version
+
+    def _cleanup_state(self):
+        if self.rawmalloced_objects:
+            xxx     # free the rawmalloced_objects still around
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -133,7 +133,7 @@
     return llmemory.offsetof(WR, 'wadr')
 
 
-class TestBasic:
+class StmGCTests:
     GCClass = StmGC
 
     def setup_method(self, meth):
@@ -155,6 +155,7 @@
             if key != 0:
                 self.gc.stm_operations.threadnum = key
                 self.gc.teardown_thread()
+        self.gc.stm_operations.threadnum = 0
 
     # ----------
     # test helpers
@@ -187,14 +188,18 @@
         meth = getattr(self.gc, 'read_int%d' % WORD)
         return meth(obj, offset)
 
+
+class TestBasic(StmGCTests):
+
     def test_gc_creation_works(self):
         pass
 
     def test_allocate_bump_pointer(self):
-        a3 = self.gc.allocate_bump_pointer(3)
-        a4 = self.gc.allocate_bump_pointer(4)
-        a5 = self.gc.allocate_bump_pointer(5)
-        a6 = self.gc.allocate_bump_pointer(6)
+        tls = self.gc.main_thread_tls
+        a3 = tls.allocate_bump_pointer(3)
+        a4 = tls.allocate_bump_pointer(4)
+        a5 = tls.allocate_bump_pointer(5)
+        a6 = tls.allocate_bump_pointer(6)
         assert a4 - a3 == 3
         assert a5 - a4 == 4
         assert a6 - a5 == 5
@@ -210,7 +215,7 @@
     def test_malloc_main_vs_thread(self):
         gcref = self.gc.malloc_fixedsize_clear(123, llmemory.sizeof(S))
         obj = llmemory.cast_ptr_to_adr(gcref)
-        assert self.gc.header(obj).tid & GCFLAG_GLOBAL != 0
+        assert self.gc.header(obj).tid & GCFLAG_GLOBAL == 0
         #
         self.select_thread(1)
         gcref = self.gc.malloc_fixedsize_clear(123, llmemory.sizeof(S))
@@ -588,7 +593,3 @@
         assert a == sr1_adr
         a = self.gc.stm_normalize_global(tr1_adr)
         assert a == sr1_adr
-
-    def test_alloc_a_lot_from_main_thread(self):
-        for i in range(1000):
-            sr1, sr1_adr = self.malloc(SR)
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -0,0 +1,50 @@
+import py
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
+from pypy.rpython.memory.gc.stmtls import StmGCTLS, WORD
+from pypy.rpython.memory.gc.test.test_stmgc import StmGCTests
+
+
+S = lltype.GcStruct('S', ('a', lltype.Signed), ('b', lltype.Signed),
+                         ('c', lltype.Signed))
+
+
+class TestStmGCTLS(StmGCTests):
+    current_stack = ()
+
+    def stack_add(self, p):
+        if self.current_stack == ():
+            self.current_stack = []
+        self.current_stack.append(p)
+
+    def stack_pop(self):
+        return self.current_stack.pop()
+
+    # ----------
+
+    def test_creation_works(self):
+        pass
+
+    def test_allocate_bump_pointer(self):
+        tls = self.gc.main_thread_tls
+        a3 = tls.allocate_bump_pointer(3)
+        a4 = tls.allocate_bump_pointer(4)
+        a5 = tls.allocate_bump_pointer(5)
+        a6 = tls.allocate_bump_pointer(6)
+        assert a4 - a3 == 3
+        assert a5 - a4 == 4
+        assert a6 - a5 == 5
+
+    def test_local_collection(self):
+        s1, _ = self.malloc(S); s1.a = 111
+        s2, _ = self.malloc(S); s2.a = 222
+        self.stack_add(s2)
+        self.gc.main_thread_tls.local_collection()
+        s3 = self.stack_pop()
+        assert s3.a == 222
+        xxxx # raises...
+        s1.a
+        s2.a
+
+    def test_alloc_a_lot(self):
+        for i in range(1000):
+            sr1, sr1_adr = self.malloc(SR)

From noreply at buildbot.pypy.org  Fri Apr 13 16:06:27 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 13 Apr 2012 16:06:27 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: Just move the blocks around
 to keep the number of flag checks
Message-ID: <20120413140627.742A382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-minimark-pinning
Changeset: r54333:549ac1d84b87
Date: 2012-04-13 16:06 +0200
http://bitbucket.org/pypy/pypy/changeset/549ac1d84b87/

Log:	Just move the blocks around to keep the number of flag checks
	minimal in the common case.

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1487,7 +1487,23 @@
             return
         #
         size_gc_header = self.gcheaderbuilder.size_gc_header
-        if self.header(obj).tid & GCFLAG_PINNED:
+        if self.header(obj).tid & (GCFLAG_HAS_SHADOW|GCFLAG_PINNED) == 0:
+            #
+            # Common case: 'obj' was not already forwarded (otherwise
+            # tid == -42, containing all flags), and it doesn't have the
+            # HAS_SHADOW or PINNED flags either.  We must move it out of
+            # the nursery, into a new nonmovable location.
+            totalsize = size_gc_header + self.get_size(obj)
+            newhdr = self._malloc_out_of_nursery(totalsize)
+            #
+        elif self.is_forwarded(obj):
+            #
+            # 'obj' was already forwarded.  Change the original reference
+            # to point to its forwarding address, and we're done.
+            root.address[0] = self.get_forwarding_address(obj)
+            return
+            #
+        elif self.header(obj).tid & GCFLAG_PINNED:
             hdr = self.header(obj)
             if hdr.tid & GCFLAG_VISITED:
                 return
@@ -1497,21 +1513,6 @@
             self.surviving_pinned_objects.insert(
                 llarena.getfakearenaaddress(obj - size_gc_header))
             return
-        elif self.header(obj).tid & GCFLAG_HAS_SHADOW == 0:
-            #
-            # Common case: 'obj' was not already forwarded (otherwise
-            # tid == -42, containing all flags), and it doesn't have the
-            # HAS_SHADOW flag either.  We must move it out of the nursery,
-            # into a new nonmovable location.
-            totalsize = size_gc_header + self.get_size(obj)
-            newhdr = self._malloc_out_of_nursery(totalsize)
-            #
-        elif self.is_forwarded(obj):
-            #
-            # 'obj' was already forwarded.  Change the original reference
-            # to point to its forwarding address, and we're done.
-            root.address[0] = self.get_forwarding_address(obj)
-            return
             #
         else:
             # First visit to an object that has already a shadow.

From noreply at buildbot.pypy.org  Fri Apr 13 16:09:10 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 13 Apr 2012 16:09:10 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: missing a "return"?
Message-ID: <20120413140910.15D1B82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-minimark-pinning
Changeset: r54334:d6855594831c
Date: 2012-04-13 16:08 +0200
http://bitbucket.org/pypy/pypy/changeset/d6855594831c/

Log:	missing a "return"?

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -600,6 +600,7 @@
                 llarena.arena_reserve(self.nursery_free, totalsize)
                 res = self.nursery_free
                 self.nursery_free = res + totalsize
+            XXX   # it's missing a 'return' somewhere think
         self.minor_collection(totalsize)
         # try allocating now, otherwise we do a major collect
         do_major_collect = False

From noreply at buildbot.pypy.org  Fri Apr 13 16:17:44 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Fri, 13 Apr 2012 16:17:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: (cfbolz, bivab): simplify
Message-ID: <20120413141744.D160682F4E@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r54335:83dbfcb6f927
Date: 2012-04-13 16:16 +0200
http://bitbucket.org/pypy/pypy/changeset/83dbfcb6f927/

Log:	(cfbolz, bivab): simplify

diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:

From noreply at buildbot.pypy.org  Fri Apr 13 17:04:33 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 13 Apr 2012 17:04:33 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: of course,
	put the return there
Message-ID: <20120413150433.B0CA282F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54336:dd9a443b9106
Date: 2012-04-13 16:53 +0200
http://bitbucket.org/pypy/pypy/changeset/dd9a443b9106/

Log:	of course, put the return there

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -600,7 +600,7 @@
                 llarena.arena_reserve(self.nursery_free, totalsize)
                 res = self.nursery_free
                 self.nursery_free = res + totalsize
-            XXX   # it's missing a 'return' somewhere think
+                return res
         self.minor_collection(totalsize)
         # try allocating now, otherwise we do a major collect
         do_major_collect = False

From noreply at buildbot.pypy.org  Fri Apr 13 17:04:35 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 13 Apr 2012 17:04:35 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: make sure I remember
Message-ID: <20120413150435.0CB0282F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54337:a2282c475c4d
Date: 2012-04-13 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/a2282c475c4d/

Log:	make sure I remember

diff --git a/TODO b/TODO
new file mode 100644
--- /dev/null
+++ b/TODO
@@ -0,0 +1,8 @@
+
+* implement limit on no of pinned objects + replace insert with qsort
+
+* use pinning
+
+* make sure objects with shadows work
+
+* implement tracing for pinned objects (check that it works)
\ No newline at end of file

From noreply at buildbot.pypy.org  Fri Apr 13 18:21:59 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 13 Apr 2012 18:21:59 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: transform gc support
Message-ID: <20120413162159.D114782F4F@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54339:175be1a2c933
Date: 2012-04-13 17:27 +0200
http://bitbucket.org/pypy/pypy/changeset/175be1a2c933/

Log:	transform gc support

diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -446,6 +446,10 @@
                                            [s_gc,
                                             annmodel.SomeInteger(nonneg=True)],
                                            annmodel.s_None)
+        self.pin_ptr = getfn(GCClass.pin,
+                             [s_gc, annmodel.SomeAddress()], annmodel.s_None)
+        self.unpin_ptr = getfn(GCClass.unpin,
+                               [s_gc, annmodel.SomeAddress()], annmodel.s_None)
 
         self.write_barrier_ptr = None
         self.write_barrier_from_array_ptr = None
@@ -747,6 +751,14 @@
         hop.genop("direct_call", [self.can_move_ptr, self.c_const_gc, v_addr],
                   resultvar=op.result)
 
+    def gct_gc_pin(self, hop):
+        op = hop.spaceop
+        hop.genop("direct_call", [self.pin_ptr, self.c_const_gc, op.args[0]])
+
+    def gct_gc_unpin(self, hop):
+        op = hop.spaceop
+        hop.genop("direct_call", [self.unpin_ptr, self.c_const_gc, op.args[0]])
+
     def gct_shrink_array(self, hop):
         if self.shrink_array_ptr is None:
             return GCTransformer.gct_shrink_array(self, hop)

From noreply at buildbot.pypy.org  Fri Apr 13 18:21:58 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 13 Apr 2012 18:21:58 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: rpython fixes and
	transformed test
Message-ID: <20120413162158.89C1182F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54338:0ac3b06d13d8
Date: 2012-04-13 17:23 +0200
http://bitbucket.org/pypy/pypy/changeset/0ac3b06d13d8/

Log:	rpython fixes and transformed test

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -247,8 +247,6 @@
         self.nursery_top  = NULL
         self.debug_tiny_nursery = -1
         self.debug_rotating_nurseries = None
-        self.surviving_pinned_objects = NULL
-        self.nursery_barriers = NULL
         #
         # The ArenaCollection() handles the nonmovable objects allocation.
         if ArenaCollectionClass is None:
@@ -452,7 +450,7 @@
 
     def debug_rotate_nursery(self):
         if self.debug_rotating_nurseries is not None:
-            ll_assert(self.nursery_barriers.empty(), "non empty nursery barriers with rotating nursery")
+            ll_assert(not self.nursery_barriers.non_empty(), "non empty nursery barriers with rotating nursery")
             debug_start("gc-debug")
             oldnurs = self.nursery
             llarena.arena_protect(oldnurs, self._nursery_memory_size(), True)
@@ -1339,7 +1337,6 @@
             nursery_barriers.append(next)
         llarena.arena_reset(prev, self.nursery_top - prev, 2)
         self.surviving_pinned_objects.delete()
-        self.surviving_pinned_objects = NULL
         self.nursery_barriers = nursery_barriers
         self.debug_rotate_nursery()
         self.nursery_free = self.nursery
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -43,6 +43,7 @@
     gcpolicy = None
     GC_CAN_MOVE = False
     GC_CAN_MALLOC_NONMOVABLE = True
+    GC_CAN_ALWAYS_PIN = False
     taggedpointers = False
 
     def setup_class(cls):
@@ -726,6 +727,28 @@
         res = fn([])
         assert res == ord('y')
 
+    def define_pinning(cls):
+        def f(i, j):
+            s = str(i)
+            if not rgc.can_move(s):
+                return 13
+            sum = 0
+            with rgc.pinned_object(s):
+                sum += int(rgc.can_move(s))
+            sum += 10 * int(rgc.can_move(s))
+            return sum
+        return f
+
+    def test_pinning(self):
+        res = self.runner("pinning")([10, 0])
+        if not self.GC_CAN_MOVE:
+            assert res == 13
+        elif self.GC_CAN_ALWAYS_PIN:
+            assert res == 10
+        else:
+            assert res == 11 or res == 13 # sometimes fresh objs can't move
+
+
 class GenericMovingGCTests(GenericGCTests):
     GC_CAN_MOVE = True
     GC_CAN_MALLOC_NONMOVABLE = False
@@ -1273,6 +1296,7 @@
 class TestMiniMarkGC(TestHybridGC):
     gcname = "minimark"
     GC_CAN_TEST_ID = True
+    GC_CAN_ALWAYS_PIN = True
 
     class gcpolicy(gc.FrameworkGcPolicy):
         class transformerclass(framework.FrameworkGCTransformer):

From noreply at buildbot.pypy.org  Fri Apr 13 18:22:01 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 13 Apr 2012 18:22:01 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: use pinning, easy (?)
Message-ID: <20120413162201.26A2B82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54340:049d60b7295d
Date: 2012-04-13 17:29 +0200
http://bitbucket.org/pypy/pypy/changeset/049d60b7295d/

Log:	use pinning, easy (?)

diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -714,6 +714,7 @@
         string is already nonmovable.  Must be followed by a
         free_nonmovingbuffer call.
         """
+        rgc.pin(data)
         if rgc.can_move(data):
             count = len(data)
             buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
@@ -743,7 +744,9 @@
             offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
         followed_2nd_path = (buf == cast(TYPEP, data_start))
         keepalive_until_here(data)
-        if not followed_2nd_path:
+        if followed_2nd_path:
+            rgc.unpin(data)
+        else:
             lltype.free(buf, flavor='raw')
     free_nonmovingbuffer._annenforceargs_ = [strtype, None]
 

From noreply at buildbot.pypy.org  Fri Apr 13 19:06:12 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 13 Apr 2012 19:06:12 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Random progress.
Message-ID: <20120413170612.5EAA482F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54341:e1e05308a3c9
Date: 2012-04-13 19:05 +0200
http://bitbucket.org/pypy/pypy/changeset/e1e05308a3c9/

Log:	Random progress.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -43,6 +43,7 @@
 GCFLAG_HAS_SHADOW = first_gcflag << 2
 GCFLAG_FIXED_HASH = first_gcflag << 3
 GCFLAG_WEAKREF    = first_gcflag << 4
+GCFLAG_VISITED    = first_gcflag << 5
 
 
 def always_inline(fn):
@@ -70,18 +71,18 @@
         'stm_operations': 'use_real_one',
         'nursery_size': 32*1024*1024,           # 32 MB
 
-        "page_size": 1024*WORD,                 # copied from minimark.py
-        "arena_size": 65536*WORD,               # copied from minimark.py
-        "small_request_threshold": 35*WORD,     # copied from minimark.py
+        #"page_size": 1024*WORD,                 # copied from minimark.py
+        #"arena_size": 65536*WORD,               # copied from minimark.py
+        #"small_request_threshold": 35*WORD,     # copied from minimark.py
     }
 
     def __init__(self, config,
                  stm_operations='use_emulator',
                  nursery_size=1024,
-                 page_size=16*WORD,
-                 arena_size=64*WORD,
-                 small_request_threshold=5*WORD,
-                 ArenaCollectionClass=None,
+                 #page_size=16*WORD,
+                 #arena_size=64*WORD,
+                 #small_request_threshold=5*WORD,
+                 #ArenaCollectionClass=None,
                  **kwds):
         MovingGCBase.__init__(self, config, **kwds)
         #
@@ -95,9 +96,7 @@
         from pypy.rpython.memory.gc import stmshared
         self.stm_operations = stm_operations
         self.nursery_size = nursery_size
-        self.sharedarea = stmshared.StmGCSharedArea(self, ArenaCollectionClass,
-                                                    page_size, arena_size,
-                                                    small_request_threshold)
+        self.sharedarea = stmshared.StmGCSharedArea(self)
         #
         def _get_size(obj):     # indirection to hide 'self'
             return self.get_size(obj)
diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -1,18 +1,58 @@
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rlib.objectmodel import free_non_gc_object
+
+NULL = llmemory.NULL
 
 
 class StmGCSharedArea(object):
+    _alloc_flavor_ = 'raw'
 
-    def __init__(self, gc, ArenaCollectionClass,
-                 page_size, arena_size, small_request_threshold):
+    def __init__(self, gc):
         self.gc = gc
-        # The ArenaCollection() handles the nonmovable objects allocation.
-        # It contains all small GCFLAG_GLOBAL objects.  The non-small ones
-        # are directly malloc'ed.
-        if ArenaCollectionClass is None:
-            from pypy.rpython.memory.gc import minimarkpage
-            ArenaCollectionClass = minimarkpage.ArenaCollection
-        self.ac = ArenaCollectionClass(arena_size, page_size,
-                                       small_request_threshold)
 
     def setup(self):
         pass
+
+
+class StmGCThreadLocalAllocator(object):
+    """A thread-local allocator for the shared area.
+    This is an optimization only: it lets us use thread-local variables
+    to keep track of what we allocated.
+    """
+    _alloc_flavor_ = 'raw'
+
+    def __init__(self, sharedarea):
+        self.gc = sharedarea.gc
+        self.sharedarea = sharedarea
+        self.chained_list = NULL
+        self.special_stack = self.gc.AddressStack()
+
+    def delete(self):
+        self.special_stack.delete()
+        free_non_gc_object(self)
+
+    def malloc_regular(self, size):
+        """Malloc for an object where the 'version' field can be used
+        internally for a chained list."""
+        adr1 = llarena.arena_malloc(size, 0)
+        adr2 = adr1 + self.gc.gcheaderbuilder
+        hdr = llmemory.cast_adr_to_ptr(adr1, lltype.Ptr(self.gc.HDR))
+        hdr.version = self.chained_list
+        self.chained_list = adr2
+        return adr2
+
+    def malloc_special(self, size):
+        """Malloc for an object where the 'version' field cannot be
+        used internally.  It's the rare case here."""
+        adr1 = llarena.arena_malloc(size, 0)
+        adr2 = adr1 + self.gc.gcheaderbuilder.size_gc_header
+        self.special_stack.append(adr2)
+        return adr2
+
+    def free_object(self, adr2):
+        adr1 = adr2 - self.gc.gcheaderbuilder.size_gc_header
+        llarena.arena_free(adr1)
+
+    def replace_special_stack(self, new_special_stack):
+        self.special_stack.delete()
+        self.special_stack = new_special_stack
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -7,7 +7,7 @@
 
 from pypy.rpython.memory.gc.stmgc import WORD, NULL
 from pypy.rpython.memory.gc.stmgc import always_inline, dont_inline
-from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL
+from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED
 
 
 class StmGCTLS(object):
@@ -19,6 +19,7 @@
     nontranslated_dict = {}
 
     def __init__(self, gc, in_main_thread):
+        from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
         self.gc = gc
         self.in_main_thread = in_main_thread
         self.stm_operations = self.gc.stm_operations
@@ -37,13 +38,16 @@
         self.nursery_start = self._alloc_nursery(self.nursery_size)
         #
         # --- the local raw-malloced objects (chained list via hdr.version)
-        self.rawmalloced_objects = NULL
+        #self.rawmalloced_objects = NULL
         # --- the local "normal" old objects (chained list via hdr.version)
         self.old_objects = NULL
         # --- the local objects with weakrefs (chained list via hdr.version)
         #self.young_objects_with_weakrefs = NULL
         #self.old_objects_with_weakrefs = NULL
         #
+        # --- a thread-local allocator for the shared area
+        self.sharedarea_tls = StmGCThreadLocalAllocator(gc.sharedarea)
+        #
         self._register_with_C_code()
 
     def teardown_thread(self):
@@ -137,70 +141,52 @@
     # ------------------------------------------------------------
 
     def local_collection(self, run_finalizers=True):
-        """Do a local collection.  Finds all surviving young objects
-        and make them old.  Also looks for roots from the stack.
-        The flag GCFLAG_WAS_COPIED is kept and the C tree is updated
-        if the local young object moves.
+        """Do a local collection.  This should be equivalent to a minor
+        collection only, but the GC is not generational so far, so it is
+        for now the same as a full collection --- but only on LOCAL
+        objects, not touching the GLOBAL objects.  More precisely, this
+        finds all YOUNG LOCAL objects, move them out of the nursery if
+        necessary, and make them OLD LOCAL objects.  This starts from
+        the roots from the stack.  The flag GCFLAG_WAS_COPIED is kept
+        and the C tree is updated if the local young objects move.
         """
         #
         debug_start("gc-local")
         #
-        # First, find the roots that point to young objects.  All nursery
-        # objects found are copied out of the nursery, and the occasional
-        # young raw-malloced object is flagged with GCFLAG_VISITED.
-        # Note that during this step, we ignore references to further
-        # young objects; only objects directly referenced by roots
-        # are copied out or flagged.  They are also added to the list
-        # 'old_objects_pointing_to_young'.
+        # Linked list of LOCAL objects pending a visit.  Note that no
+        # GLOBAL object can at any point contain a reference to a LOCAL
+        # object.
+        self.pending_list = NULL
+        #
+        # First, find the roots that point to LOCAL objects.  All YOUNG
+        # (i.e. nursery) objects found are copied out of the nursery.
+        # All OLD objects found are flagged with GCFLAG_VISITED.  At this
+        # point, the content of the objects is not modified; the objects
+        # are merely added to the chained list 'pending_list'.
         self.collect_roots_in_nursery()
         #
-        while True:
-            # If we are using card marking, do a partial trace of the arrays
-            # that are flagged with GCFLAG_CARDS_SET.
-            if self.card_page_indices > 0:
-                self.collect_cardrefs_to_nursery()
-            #
-            # Now trace objects from 'old_objects_pointing_to_young'.
-            # All nursery objects they reference are copied out of the
-            # nursery, and again added to 'old_objects_pointing_to_young'.
-            # All young raw-malloced object found are flagged GCFLAG_VISITED.
-            # We proceed until 'old_objects_pointing_to_young' is empty.
-            self.collect_oldrefs_to_nursery()
-            #
-            # We have to loop back if collect_oldrefs_to_nursery caused
-            # new objects to show up in old_objects_with_cards_set
-            if self.card_page_indices > 0:
-                if self.old_objects_with_cards_set.non_empty():
-                    continue
-            break
+        # Also find the roots that are the local copy of GCFLAG_WAS_COPIED
+        # objects.
+        self.collect_roots_from_tldict()
         #
-        # Now all live nursery objects should be out.  Update the young
-        # weakrefs' targets.
-        if self.young_objects_with_weakrefs.non_empty():
-            self.invalidate_young_weakrefs()
-        if self.young_objects_with_light_finalizers.non_empty():
-            self.deal_with_young_objects_with_finalizers()
+        # Now repeat following objects until 'pending_list' is empty.
+        self.collect_oldrefs_to_nursery()
         #
-        # Clear this mapping.
-        if self.nursery_objects_shadows.length() > 0:
-            self.nursery_objects_shadows.clear()
+        # Walk the list of LOCAL raw-malloced objects, and free them if
+        # necessary.
+        #self.free_local_rawmalloced_objects()
         #
-        # Walk the list of young raw-malloced objects, and either free
-        # them or make them old.
-        if self.young_rawmalloced_objects:
-            self.free_young_rawmalloced_objects()
+        # Ask the ArenaCollection to visit all objects.  Free the ones
+        # that have not been visited above, and reset GCFLAG_VISITED on
+        # the others.
+        self.ac.mass_free(self._free_if_unvisited)
         #
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
         llarena.arena_reset(self.nursery, self.nursery_size, 2)
-        self.debug_rotate_nursery()
-        self.nursery_free = self.nursery
+        self.nursery_free = self.nursery_start
         #
-        debug_print("minor collect, total memory used:",
-                    self.get_total_memory_used())
-        if self.DEBUG >= 2:
-            self.debug_check_consistency()     # expensive!
-        debug_stop("gc-minor")
+        debug_stop("gc-local")
 
     def end_of_transaction_collection(self):
         """Do an end-of-transaction collection.  Finds all surviving
@@ -229,7 +215,7 @@
     def allocate_object_of_size(self, size):
         if not self.nursery_free:
             fatalerror("malloc in a non-main thread but outside a transaction")
-        if size > self.nursery_size:
+        if size > self.nursery_size // 8 * 7:
             fatalerror("object too large to ever fit in the nursery")
         while True:
             self.local_collection()
@@ -251,12 +237,12 @@
             hdr.tid |= GCFLAG_GLOBAL
             obj = hdr.version
         #
-        obj = self.rawmalloced_objects
-        self.rawmalloced_objects = NULL
-        while obj:
-            hdr = self.header(obj)
-            hdr.tid |= GCFLAG_GLOBAL
-            obj = hdr.version
+##        obj = self.rawmalloced_objects
+##        self.rawmalloced_objects = NULL
+##        while obj:
+##            hdr = self.header(obj)
+##            hdr.tid |= GCFLAG_GLOBAL
+##            obj = hdr.version
 
     def _cleanup_state(self):
         if self.rawmalloced_objects:
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -2,18 +2,42 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
 from pypy.rpython.memory.gc.stmtls import StmGCTLS, WORD
 from pypy.rpython.memory.gc.test.test_stmgc import StmGCTests
+from pypy.rpython.memory.support import get_address_stack, get_address_deque
 
 
 S = lltype.GcStruct('S', ('a', lltype.Signed), ('b', lltype.Signed),
                          ('c', lltype.Signed))
 
 
-class TestStmGCTLS(StmGCTests):
-    current_stack = ()
+class FakeStmOperations:
+    def set_tls(self, tlsaddr, num):
+        pass
+    def del_tls(self, tlsaddr):
+        pass
+
+class FakeSharedArea:
+    pass
+
+class FakeGC:
+    from pypy.rpython.memory.support import AddressDict, null_address_dict
+    AddressStack = get_address_stack()
+    AddressDeque = get_address_deque()
+    nursery_size = 128
+    stm_operations = FakeStmOperations()
+    sharedarea = FakeSharedArea()
+
+
+class TestStmGCTLS(object):
+
+    def setup_method(self, meth):
+        self.current_stack = []
+        self.gc = FakeGC()
+        self.gc.sharedarea.gc = self.gc
+        self.gctls_main = StmGCTLS(self.gc, in_main_thread=True)
+        self.gctls_thrd = StmGCTLS(self.gc, in_main_thread=False)
+        self.gc.main_thread_tls = self.gctls_main
 
     def stack_add(self, p):
-        if self.current_stack == ():
-            self.current_stack = []
         self.current_stack.append(p)
 
     def stack_pop(self):
diff --git a/pypy/translator/c/src/allocator.h b/pypy/translator/c/src/allocator.h
--- a/pypy/translator/c/src/allocator.h
+++ b/pypy/translator/c/src/allocator.h
@@ -1,3 +1,14 @@
+#if defined(RPY_STM)
+
+
+/* XXX no special malloc function, use the thread-safe system-provided one */
+#define PyObject_Malloc malloc
+#define PyObject_Realloc realloc
+#define PyObject_Free free
+
+
+#else
+
 
 /* allocation functions prototypes */
 void *PyObject_Malloc(size_t n);
@@ -29,3 +40,4 @@
 #endif
 
 #endif
+#endif
diff --git a/pypy/translator/stm/stmgcintf.py b/pypy/translator/stm/stmgcintf.py
--- a/pypy/translator/stm/stmgcintf.py
+++ b/pypy/translator/stm/stmgcintf.py
@@ -11,7 +11,8 @@
 eci = ExternalCompilationInfo(
     include_dirs = [cdir, cdir2],
     includes = ['src_stm/et.h', 'src_stm/et.c'],
-    pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT],
+    pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT,
+                        '#define RPY_STM 1'],
     separate_module_sources = ['\n'],    # hack for test_rffi_stm
 )
 

From noreply at buildbot.pypy.org  Fri Apr 13 19:24:24 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 13 Apr 2012 19:24:24 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Update.
Message-ID: <20120413172424.650AD82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4191:028584929b5b
Date: 2012-04-13 19:24 +0200
http://bitbucket.org/pypy/extradoc/changeset/028584929b5b/

Log:	Update.

diff --git a/planning/stm.txt b/planning/stm.txt
--- a/planning/stm.txt
+++ b/planning/stm.txt
@@ -8,8 +8,35 @@
 |
 
 
-Overview
---------
+Python Interface
+----------------
+
+Planned interface refactoring: integrate the states of "running
+transactionally" and "not running transactionally".
+
+Running normally the program is done in one big top-level transaction.
+The "transaction" module should be provisionally renamed to "ame".  At
+any point (including in sub-transactions) we can create an instance of
+'ame.RandomOrder()', and call methods on it:
+
+  - 'add(callable, *args, **kwds)'
+  - 'run()'
+
+When run() is called the current transaction is suspended and all
+added transactions are run (including the ones added by them).  These
+transactions commit into the current transaction.  When run() returns,
+the parent transaction continues running.  If at any point any
+transaction becomes inevitable, its parent transaction does the same,
+and so on.
+
+This should give a model that composes more naturally than the current
+one.  Also, it opens the door to other 'ame' objects; e.g. an
+'ame.Sequential()' would run the subtransactions in the same order as
+they have been added.
+
+
+Overview of the GC
+------------------
 
 A saner approach (and likely better results that now): integrate with
 the GC.  Here is the basic plan.

From noreply at buildbot.pypy.org  Fri Apr 13 22:03:21 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:03:21 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: merge from default
Message-ID: <20120413200321.F24E182F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54342:817ce54a56bb
Date: 2012-04-13 13:07 +0200
http://bitbucket.org/pypy/pypy/changeset/817ce54a56bb/

Log:	merge from default

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -169,9 +169,11 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +188,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +226,26 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
+        if self.keywords is None:
+            self.keywords = keywords[:] # copy to make non-resizable
+            self.keywords_w = keywords_w[:]
+        else:
+            # looks quadratic, but the JIT should remove all of it nicely.
+            # Also, all the lists should be small
+            for key in keywords:
+                for otherkey in self.keywords:
+                    if otherkey == key:
+                        raise operationerrfmt(self.space.w_TypeError,
+                                              "got multiple values "
+                                              "for keyword argument "
+                                              "'%s'", key)
+            self.keywords = self.keywords + keywords
+            self.keywords_w = self.keywords_w + keywords_w
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +404,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+

From noreply at buildbot.pypy.org  Fri Apr 13 22:03:23 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:03:23 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: reenable the ability to run
	stdlib tests
Message-ID: <20120413200323.96AE982F4F@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54343:f8735b33c95a
Date: 2012-04-13 21:51 +0200
http://bitbucket.org/pypy/pypy/changeset/f8735b33c95a/

Log:	reenable the ability to run stdlib tests

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -17,8 +17,7 @@
 from pypy.conftest import gettestobjspace, option as pypy_option 
 
 from pypy.tool.pytest import appsupport 
-from pypy.tool.pytest.confpath import pypydir, libpythondir, \
-                                      regrtestdir, modregrtestdir, testresultdir
+from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir
 
 pytest_plugins = "resultlog",
 rsyncdirs = ['.', '../pypy/']
@@ -76,14 +75,11 @@
     compiler = property(compiler)
 
     def ismodified(self): 
-        return modregrtestdir.join(self.basename).check() 
+        #XXX: ask hg
+        return None
 
     def getfspath(self): 
-        fn = modregrtestdir.join(self.basename)
-        if fn.check(): 
-            return fn 
-        fn = regrtestdir.join(self.basename)
-        return fn 
+        return testdir.join(self.basename)
 
     def run_file(self, space): 
         fspath = self.getfspath()
@@ -526,7 +522,7 @@
     listed_names = dict.fromkeys([regrtest.basename for regrtest in testmap])
     listed_names['test_support.py'] = True     # ignore this
     missing = []
-    for path in regrtestdir.listdir(fil='test_*.py'):
+    for path in testdir.listdir(fil='test_*.py'):
         name = path.basename
         if name not in listed_names:
             missing.append('    RegrTest(%r),' % (name,))
@@ -547,7 +543,7 @@
     regrtest = parent.config._basename2spec.get(path.basename, None)
     if regrtest is None:
         return
-    if path.dirpath() not in (modregrtestdir, regrtestdir):
+    if path.dirpath() != testdir:
         return
     return RunFileExternal(path.basename, parent=parent, regrtest=regrtest)
 
@@ -715,14 +711,3 @@
             lst.append('core')
         return lst
 
-#
-# Sanity check  (could be done more nicely too)
-#
-import os
-samefile = getattr(os.path, 'samefile', 
-                   lambda x,y : str(x) == str(y))
-if samefile(os.getcwd(), str(regrtestdir.dirpath())):
-    raise NotImplementedError(
-        "Cannot run py.test with this current directory:\n"
-        "the app-level sys.path will contain %s before %s)." % (
-            regrtestdir.dirpath(), modregrtestdir.dirpath()))
diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py
--- a/pypy/tool/pytest/confpath.py
+++ b/pypy/tool/pytest/confpath.py
@@ -7,6 +7,4 @@
 distdir = pypydir.dirpath()
 testresultdir = distdir.join('testresult') 
 assert pypydir.check(dir=1) 
-libpythondir = lib_pypy.LIB_PYTHON
-regrtestdir = lib_pypy.LIB_PYTHON_VANILLA.join('test') 
-modregrtestdir = lib_pypy.LIB_PYTHON_MODIFIED.join('test') 
+testdir = lib_pypy.LIB_PYTHON.join('test')

From noreply at buildbot.pypy.org  Fri Apr 13 22:03:24 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:03:24 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: fix tests for initial
	sys.path
Message-ID: <20120413200324.CCE6582F50@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54344:39affc663dc5
Date: 2012-04-13 22:02 +0200
http://bitbucket.org/pypy/pypy/changeset/39affc663dc5/

Log:	fix tests for initial sys.path

diff --git a/pypy/module/sys/test/test_initialpath.py b/pypy/module/sys/test/test_initialpath.py
--- a/pypy/module/sys/test/test_initialpath.py
+++ b/pypy/module/sys/test/test_initialpath.py
@@ -3,12 +3,10 @@
 from pypy.module.sys.version import PYPY_VERSION, CPYTHON_VERSION
 
 def build_hierarchy(prefix):
-    dirname = '%d.%d' % (CPYTHON_VERSION[0],
-                         CPYTHON_VERSION[1])
+    dirname = '%d.%d' % CPYTHON_VERSION[:2]
     a = prefix.join('lib_pypy').ensure(dir=1)
-    b = prefix.join('lib-python', 'modified-%s' % dirname).ensure(dir=1)
-    c = prefix.join('lib-python', dirname).ensure(dir=1)
-    return a, b, c
+    b = prefix.join('lib-python', dirname).ensure(dir=1)
+    return a, b
 
 
 def test_stdlib_in_prefix(tmpdir):
@@ -18,10 +16,7 @@
     assert path[:len(dirs)] == map(str, dirs)
 
 def test_include_libtk(tmpdir):
-    lib_pypy, lib_python_modified, lib_python = build_hierarchy(tmpdir)
-    lib_tk_modified = lib_python_modified.join('lib-tk')
+    lib_pypy, lib_python = build_hierarchy(tmpdir)
     lib_tk = lib_python.join('lib-tk')
     path = getinitialpath(None, str(tmpdir))
-    i = path.index(str(lib_tk_modified))
-    j = path.index(str(lib_tk))
-    assert i < j
+    assert lib_tk in path

From noreply at buildbot.pypy.org  Fri Apr 13 22:16:17 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:16:17 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: remove unneeded comment from
	UserDict
Message-ID: <20120413201617.B123D82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54345:d0611e06277d
Date: 2012-04-13 22:06 +0200
http://bitbucket.org/pypy/pypy/changeset/d0611e06277d/

Log:	remove unneeded comment from UserDict

diff --git a/lib-python/2.7/UserDict.py b/lib-python/2.7/UserDict.py
--- a/lib-python/2.7/UserDict.py
+++ b/lib-python/2.7/UserDict.py
@@ -1,10 +1,5 @@
 """A more or less complete user-defined wrapper around dictionary objects."""
 
-# XXX This is a bit of a hack (as usual :-))
-# the actual content of the file is not changed, but we put it here to make
-# virtualenv happy (because its internal logic expects at least one of the
-# REQUIRED_MODULES to be in modified-*)
-
 class UserDict:
     def __init__(self, dict=None, **kwargs):
         self.data = {}

From noreply at buildbot.pypy.org  Fri Apr 13 22:16:20 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:16:20 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: revert stdlib test package
 __init__ to stdlib version
Message-ID: <20120413201620.1C85382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54346:1f72d5b26f8c
Date: 2012-04-13 22:08 +0200
http://bitbucket.org/pypy/pypy/changeset/1f72d5b26f8c/

Log:	revert stdlib test package __init__ to stdlib version

diff --git a/lib-python/2.7/test/__init__.py b/lib-python/2.7/test/__init__.py
--- a/lib-python/2.7/test/__init__.py
+++ b/lib-python/2.7/test/__init__.py
@@ -1,11 +1,1 @@
-"""
-This package only contains the tests that we have modified for PyPy.
-It uses the 'official' hack to include the rest of the standard
-'test' package from CPython.
-
-This assumes that sys.path is configured to contain
-'lib-python/modified-2.7.0' before 'lib-python/2.7.0'.
-"""
-
-from pkgutil import extend_path
-__path__ = extend_path(__path__, __name__)
+# Dummy file to make this directory a package.

From noreply at buildbot.pypy.org  Fri Apr 13 22:16:22 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:16:22 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: test.regrtest: undo the
 findtestdir change, keep some simplifications for now
Message-ID: <20120413201622.7E58782F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54347:092e9fa2cf0e
Date: 2012-04-13 22:14 +0200
http://bitbucket.org/pypy/pypy/changeset/092e9fa2cf0e/

Log:	test.regrtest: undo the findtestdir change, keep some
	simplifications for now

diff --git a/lib-python/2.7/test/regrtest.py b/lib-python/2.7/test/regrtest.py
--- a/lib-python/2.7/test/regrtest.py
+++ b/lib-python/2.7/test/regrtest.py
@@ -680,13 +680,8 @@
 
 def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
     """Return a list of all applicable test modules."""
-    if testdir:
-        testdirs = [testdir]
-    else:
-        testdirs = findtestdirs()
-    names = {}
-    for testdir in testdirs:
-        names.update(dict.fromkeys(os.listdir(testdir)))
+    testdir = findtestdir(testdir)
+    names = os.listdir(testdir)
     tests = []
     others = set(stdtests) | nottests
     for name in names:
@@ -1085,19 +1080,8 @@
     # Collect cyclic trash.
     gc.collect()
 
-def findtestdirs():
-    # XXX hacking: returns a list of both the '2.7.0/test' and the
-    # 'modified-2.7.0/test' directories, as full paths.
-    testdir = os.path.abspath(os.path.dirname(__file__) or os.curdir)
-    assert os.path.basename(testdir).lower() == 'test'
-    maindir = os.path.dirname(testdir)
-    libpythondir = os.path.dirname(maindir)
-    maindirname = os.path.basename(maindir).lower()
-    if maindirname.startswith('modified-'):
-        maindirname = maindirname[len('modified-'):]
-    testdir1 = os.path.join(libpythondir, maindirname, 'test')
-    testdir2 = os.path.join(libpythondir, 'modified-'+maindirname, 'test')
-    return [testdir1, testdir2]
+def findtestdir(path=None):
+    return path or os.path.dirname(__file__) or os.curdir
 
 def removepy(names):
     if not names:

From noreply at buildbot.pypy.org  Fri Apr 13 22:16:24 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:16:24 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: remove the modified-
 expectations from test_app_main
Message-ID: <20120413201624.F3C4C82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54348:c1ed24dc186f
Date: 2012-04-13 22:15 +0200
http://bitbucket.org/pypy/pypy/changeset/c1ed24dc186f/

Log:	remove the modified- expectations from test_app_main

diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py
--- a/pypy/translator/goal/test2/test_app_main.py
+++ b/pypy/translator/goal/test2/test_app_main.py
@@ -803,7 +803,6 @@
         fake_exe = prefix.join('bin/pypy-c').ensure(file=1)
         expected_path = [str(prefix.join(subdir).ensure(dir=1))
                          for subdir in ('lib_pypy',
-                                        'lib-python/modified-%s' % cpy_ver,
                                         'lib-python/%s' % cpy_ver)]
         
         self.w_goal_dir = self.space.wrap(goal_dir)
@@ -837,9 +836,9 @@
             app_main.os = os
             pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c')
             newpath = app_main.get_library_path(pypy_c)
-            # we get at least lib_pypy, lib-python/modified-X.Y.Z,
+            # we get at least lib_pypy 
             # lib-python/X.Y.Z, and maybe more (e.g. plat-linux2)
-            assert len(newpath) >= 3
+            assert len(newpath) >= 2
             for p in newpath:
                 assert p.startswith(self.trunkdir)
         finally:

From noreply at buildbot.pypy.org  Fri Apr 13 22:55:53 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:55:53 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: fix vanilla stdlib ref in
	stdlib___future__
Message-ID: <20120413205553.5241A82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54349:4ed884067730
Date: 2012-04-13 22:53 +0200
http://bitbucket.org/pypy/pypy/changeset/4ed884067730/

Log:	fix vanilla stdlib ref in stdlib___future__

diff --git a/pypy/tool/stdlib___future__.py b/pypy/tool/stdlib___future__.py
--- a/pypy/tool/stdlib___future__.py
+++ b/pypy/tool/stdlib___future__.py
@@ -1,8 +1,8 @@
 # load __future__.py constants
 
 def load_module():
-    from pypy.tool.lib_pypy import LIB_PYTHON_VANILLA
-    module_path = LIB_PYTHON_VANILLA.join('__future__.py')
+    from pypy.tool.lib_pypy import LIB_PYTHON
+    module_path = LIB_PYTHON.join('__future__.py')
     execfile(str(module_path), globals())
 
 load_module()

From noreply at buildbot.pypy.org  Fri Apr 13 22:55:54 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 13 Apr 2012 22:55:54 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: fix MODIFID/VANILLA refs of
	lib-pypy test
Message-ID: <20120413205554.93D3582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54350:77f9d750e08f
Date: 2012-04-13 22:55 +0200
http://bitbucket.org/pypy/pypy/changeset/77f9d750e08f/

Log:	fix MODIFID/VANILLA refs of lib-pypy test

diff --git a/pypy/tool/test/test_lib_pypy.py b/pypy/tool/test/test_lib_pypy.py
--- a/pypy/tool/test/test_lib_pypy.py
+++ b/pypy/tool/test/test_lib_pypy.py
@@ -7,8 +7,6 @@
 
 def test_lib_python_exists():
     assert lib_pypy.LIB_PYTHON.check(dir=1)
-    assert lib_pypy.LIB_PYTHON_VANILLA.check(dir=1)
-    assert lib_pypy.LIB_PYTHON_MODIFIED.check(dir=1)
 
 def test_import_from_lib_pypy():
     _functools = lib_pypy.import_from_lib_pypy('_functools')

From noreply at buildbot.pypy.org  Sat Apr 14 10:01:28 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 10:01:28 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: reverting merge so we dont
 get the deletions when merging from vendor/stdlib
Message-ID: <20120414080128.C6ADA46E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54352:087b58bc32e9
Date: 2012-04-14 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/087b58bc32e9/

Log:	reverting merge so we dont get the deletions when merging from
	vendor/stdlib


From noreply at buildbot.pypy.org  Sat Apr 14 10:01:27 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 10:01:27 +0200 (CEST)
Subject: [pypy-commit] pypy vendor/stdlib: remove everything thats not in
 the stdlib or tells about the version from the vendor/stdlib branch
Message-ID: <20120414080127.791E282F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: vendor/stdlib
Changeset: r54351:93769ea57f9b
Date: 2012-04-14 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/93769ea57f9b/

Log:	remove everything thats not in the stdlib or tells about the version
	from the vendor/stdlib branch

diff too long, truncating to 10000 out of 1166718 lines

diff --git a/.gitignore b/.gitignore
deleted file mode 100644
--- a/.gitignore
+++ /dev/null
@@ -1,21 +0,0 @@
-.hg
-.svn
-
-*.pyc
-*.pyo
-*~
-
-bin/pypy-c
-include/*.h
-lib_pypy/ctypes_config_cache/_[^_]*_*.py
-pypy/_cache
-pypy/doc/*.html
-pypy/doc/config/*.html
-pypy/doc/discussion/*.html
-pypy/module/cpyext/src/*.o
-pypy/module/cpyext/test/*.o
-pypy/module/test_lib_pypy/ctypes_tests/*.o
-pypy/translator/c/src/dtoa.o
-pypy/translator/goal/pypy-c
-pypy/translator/goal/target*-c
-release/
\ No newline at end of file
diff --git a/.hgignore b/.hgignore
deleted file mode 100644
--- a/.hgignore
+++ /dev/null
@@ -1,76 +0,0 @@
-syntax: glob
-*.py[co]
-*~
-.*.swp
-.idea
-.project
-.pydevproject
-
-syntax: regexp
-^testresult$
-^site-packages$
-^site-packages/.*$
-^site-packages/.*$
-^bin$
-^pypy/bin/pypy-c
-^pypy/module/cpyext/src/.+\.o$
-^pypy/module/cpyext/src/.+\.obj$
-^pypy/module/cpyext/test/.+\.errors$
-^pypy/module/cpyext/test/.+\.o$
-^pypy/module/cpyext/test/.+\.obj$
-^pypy/module/cpyext/test/.+\.manifest$
-^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
-^pypy/doc/.+\.html$
-^pypy/doc/config/.+\.rst$
-^pypy/doc/basicblock\.asc$
-^pypy/doc/.+\.svninfo$
-^pypy/translator/c/src/libffi_msvc/.+\.obj$
-^pypy/translator/c/src/libffi_msvc/.+\.dll$
-^pypy/translator/c/src/libffi_msvc/.+\.lib$
-^pypy/translator/c/src/libffi_msvc/.+\.exp$
-^pypy/translator/c/src/cjkcodecs/.+\.o$
-^pypy/translator/c/src/cjkcodecs/.+\.obj$
-^pypy/translator/jvm/\.project$
-^pypy/translator/jvm/\.classpath$
-^pypy/translator/jvm/eclipse-bin$
-^pypy/translator/jvm/src/pypy/.+\.class$
-^pypy/translator/benchmark/docutils$
-^pypy/translator/benchmark/templess$
-^pypy/translator/benchmark/gadfly$
-^pypy/translator/benchmark/mako$
-^pypy/translator/benchmark/bench-custom\.benchmark_result$
-^pypy/translator/benchmark/shootout_benchmarks$
-^pypy/translator/goal/pypy-translation-snapshot$
-^pypy/translator/goal/pypy-c
-^pypy/translator/goal/pypy-jvm
-^pypy/translator/goal/pypy-jvm.jar
-^pypy/translator/goal/.+\.exe$
-^pypy/translator/goal/.+\.dll$
-^pypy/translator/goal/target.+-c$
-^pypy/_cache$
-^pypy/doc/statistic/.+\.html$
-^pypy/doc/statistic/.+\.eps$
-^pypy/doc/statistic/.+\.pdf$
-^pypy/translator/cli/src/pypylib\.dll$
-^pypy/translator/cli/src/query\.exe$
-^pypy/translator/cli/src/main\.exe$
-^lib_pypy/ctypes_config_cache/_.+_cache\.py$
-^lib_pypy/ctypes_config_cache/_.+_.+_\.py$
-^pypy/translator/cli/query-descriptions$
-^pypy/doc/discussion/.+\.html$
-^include/.+\.h$
-^include/.+\.inl$
-^pypy/doc/_build/.*$
-^pypy/doc/config/.+\.html$
-^pypy/doc/config/style\.css$
-^pypy/doc/jit/.+\.html$
-^pypy/doc/jit/style\.css$
-^pypy/doc/image/lattice1\.png$
-^pypy/doc/image/lattice2\.png$
-^pypy/doc/image/lattice3\.png$
-^pypy/doc/image/stackless_informal\.png$
-^pypy/doc/image/parsing_example.+\.png$
-^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
-^compiled
-^.git/
-^release/
diff --git a/.hgsubstate b/.hgsubstate
deleted file mode 100644
diff --git a/.hgtags b/.hgtags
deleted file mode 100644
--- a/.hgtags
+++ /dev/null
@@ -1,4 +0,0 @@
-b590cf6de4190623aad9aa698694c22e614d67b9 release-1.5
-b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked
-d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
-ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
--- a/LICENSE
+++ /dev/null
@@ -1,270 +0,0 @@
-License for files in the pypy/ directory 
-==================================================
-
-Except when otherwise stated (look for LICENSE files in directories or
-information at the beginning of each file) all software and
-documentation in the 'pypy' directories is licensed as follows: 
-
-    The MIT License
-
-    Permission is hereby granted, free of charge, to any person 
-    obtaining a copy of this software and associated documentation 
-    files (the "Software"), to deal in the Software without 
-    restriction, including without limitation the rights to use, 
-    copy, modify, merge, publish, distribute, sublicense, and/or 
-    sell copies of the Software, and to permit persons to whom the 
-    Software is furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be included 
-    in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-    OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
-    DEALINGS IN THE SOFTWARE.
-
-
-PyPy Copyright holders 2003-2012
------------------------------------ 
-
-Except when otherwise stated (look for LICENSE files or information at
-the beginning of each file) the files in the 'pypy' directory are each
-copyrighted by one or more of the following people and organizations:    
-
-    Armin Rigo
-    Maciej Fijalkowski
-    Carl Friedrich Bolz
-    Amaury Forgeot d'Arc
-    Antonio Cuni
-    Samuele Pedroni
-    Michael Hudson
-    Holger Krekel
-    Alex Gaynor
-    Christian Tismer
-    Hakan Ardo
-    Benjamin Peterson
-    David Schneider
-    Eric van Riet Paap
-    Anders Chrigstrom
-    Richard Emslie
-    Dan Villiom Podlaski Christiansen
-    Alexander Schremmer
-    Lukas Diekmann
-    Aurelien Campeas
-    Anders Lehmann
-    Camillo Bruni
-    Niklaus Haldimann
-    Sven Hager
-    Leonardo Santagada
-    Toon Verwaest
-    Seo Sanghyeon
-    Justin Peel
-    Lawrence Oluyede
-    Bartosz Skowron
-    Jakub Gustak
-    Guido Wesdorp
-    Daniel Roberts
-    Laura Creighton
-    Adrien Di Mascio
-    Ludovic Aubry
-    Niko Matsakis
-    Wim Lavrijsen
-    Matti Picus
-    Jason Creighton
-    Jacob Hallen
-    Alex Martelli
-    Anders Hammarquist
-    Jan de Mooij
-    Stephan Diehl
-    Michael Foord
-    Stefan Schwarzer
-    Tomek Meka
-    Patrick Maupin
-    Bob Ippolito
-    Bruno Gola
-    Alexandre Fayolle
-    Marius Gedminas
-    Simon Burton
-    David Edelsohn
-    Jean-Paul Calderone
-    John Witulski
-    Timo Paulssen
-    holger krekel
-    Dario Bertini
-    Mark Pearse
-    Andreas St&#252;hrk
-    Jean-Philippe St. Pierre
-    Guido van Rossum
-    Pavel Vinogradov
-    Valentino Volonghi
-    Paul deGrandis
-    Ilya Osadchiy
-    Ronny Pfannschmidt
-    Adrian Kuhn
-    tav
-    Georg Brandl
-    Philip Jenvey
-    Gerald Klix
-    Wanja Saatkamp
-    Boris Feigin
-    Oscar Nierstrasz
-    David Malcolm
-    Eugene Oden
-    Henry Mason
-    Jeff Terrace
-    Lukas Renggli
-    Guenter Jantzen
-    Ned Batchelder
-    Bert Freudenberg
-    Amit Regmi
-    Ben Young
-    Nicolas Chauvat
-    Andrew Durdin
-    Michael Schneider
-    Nicholas Riley
-    Rocco Moretti
-    Gintautas Miliauskas
-    Michael Twomey
-    Igor Trindade Oliveira
-    Lucian Branescu Mihaila
-    Olivier Dormond
-    Jared Grubb
-    Karl Bartel
-    Gabriel Lavoie
-    Victor Stinner
-    Brian Dorsey
-    Stuart Williams
-    Toby Watson
-    Antoine Pitrou
-    Justas Sadzevicius
-    Neil Shepperd
-    Mikael Sch&#246;nenberg
-    Gasper Zejn
-    Jonathan David Riehl
-    Elmo M&#228;ntynen
-    Anders Qvist
-    Beatrice During
-    Alexander Sedov
-    Corbin Simpson
-    Vincent Legoll
-    Romain Guillebert
-    Alan McIntyre
-    Alex Perry
-    Jens-Uwe Mager
-    Simon Cross
-    Dan Stromberg
-    Guillebert Romain
-    Carl Meyer
-    Pieter Zieschang
-    Alejandro J. Cura
-    Sylvain Thenault
-    Christoph Gerum
-    Travis Francis Athougies
-    Henrik Vendelbo
-    Lutz Paelike
-    Jacob Oscarson
-    Martin Blais
-    Lucio Torre
-    Lene Wagner
-    Miguel de Val Borro
-    Artur Lisiecki
-    Bruno Gola
-    Ignas Mikalajunas
-    Stefano Rivera
-    Joshua Gilbert
-    Godefroid Chappelle
-    Yusei Tahara
-    Christopher Armstrong
-    Stephan Busemann
-    Gustavo Niemeyer
-    William Leslie
-    Akira Li
-    Kristjan Valur Jonsson
-    Bobby Impollonia
-    Michael Hudson-Doyle
-    Laurence Tratt
-    Yasir Suhail
-    Andrew Thompson
-    Anders Sigfridsson
-    Floris Bruynooghe
-    Jacek Generowicz
-    Dan Colish
-    Zooko Wilcox-O Hearn
-    Dan Loewenherz
-    Chris Lambacher
-    Dinu Gherman
-    Brett Cannon
-    Daniel Neuh&#228;user
-    Michael Chermside
-    Konrad Delong
-    Anna Ravencroft
-    Greg Price
-    Armin Ronacher
-    Christian Muirhead
-    Jim Baker
-    Rodrigo Ara&#250;jo
-    Romain Guillebert
-
-    Heinrich-Heine University, Germany 
-    Open End AB (formerly AB Strakt), Sweden
-    merlinux GmbH, Germany 
-    tismerysoft GmbH, Germany 
-    Logilab Paris, France 
-    DFKI GmbH, Germany 
-    Impara, Germany
-    Change Maker, Sweden 
-
-The PyPy Logo as used by http://speed.pypy.org and others was created
-by Samuel Reis and is distributed on terms of Creative Commons Share Alike
-License.
- 
-License for 'lib-python/2.7.0' and 'lib-python/2.7.0-modified'
-============================================================== 
-
-Except when otherwise stated (look for LICENSE files or
-copyright/license information at the beginning of each file) the files
-in the 'lib-python/2.7.0' and 'lib-python/2.7.0-modified' directories
-are all copyrighted by the Python Software Foundation and licensed under
-the Python Software License of which you can find a copy here:
-http://www.python.org/doc/Copyright.html 
-
-License for 'pypy/translator/jvm/src/jna.jar'
-=============================================
-
-The file 'pypy/translator/jvm/src/jna.jar' is licensed under the GNU
-Lesser General Public License of which you can find a copy here:
-http://www.gnu.org/licenses/lgpl.html
-
-License for 'pypy/translator/jvm/src/jasmin.jar'
-================================================
-
-The file 'pypy/translator/jvm/src/jasmin.jar' is copyright (c) 1996-2004 Jon Meyer
-and distributed with permission.  The use of Jasmin by PyPy does not imply
-that PyPy is endorsed by Jon Meyer nor any of Jasmin's contributors.  Furthermore,
-the following disclaimer applies to Jasmin:
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
-WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
-TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
-ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-License for 'pypy/module/unicodedata/'
-======================================
-
-The following files are from the website of The Unicode Consortium
-at http://www.unicode.org/.  For the terms of use of these files, see
-http://www.unicode.org/terms_of_use.html .  Or they are derived from
-files from the above website, and the same terms of use apply.
-
-    CompositionExclusions-*.txt
-    EastAsianWidth-*.txt
-    LineBreak-*.txt
-    UnicodeData-*.txt
-    UnihanNumeric-*.txt
diff --git a/README b/README
deleted file mode 100644
--- a/README
+++ /dev/null
@@ -1,24 +0,0 @@
-=====================================
-PyPy: Python in Python Implementation 
-=====================================
-
-Welcome to PyPy!
-
-PyPy is both an implementation of the Python programming language, and
-an extensive compiler framework for dynamic language implementations.
-You can build self-contained Python implementations which execute
-independently from CPython.
-
-The home page is:
-
-    http://pypy.org/
-
-The getting-started document will help guide you:
-
-    http://doc.pypy.org/en/latest/getting-started.html
-
-It will also point you to the rest of the documentation which is generated
-from files in the pypy/doc directory within the source repositories. Enjoy
-and send us feedback!
-
-    the pypy-dev team <pypy-dev at python.org>
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
deleted file mode 100644
--- a/_pytest/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-#
-__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
deleted file mode 100644
--- a/_pytest/assertion/__init__.py
+++ /dev/null
@@ -1,119 +0,0 @@
-"""
-support for presenting detailed information in failing assertions.
-"""
-import py
-import sys
-import pytest
-from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import util
-
-def pytest_addoption(parser):
-    group = parser.getgroup("debugconfig")
-    group.addoption('--assert', action="store", dest="assertmode",
-                    choices=("rewrite", "reinterp", "plain",),
-                    default="rewrite", metavar="MODE",
-                    help="""control assertion debugging tools.
-'plain' performs no assertion debugging.
-'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
-'rewrite' (the default) rewrites assert statements in test modules on import
-to provide assert expression information. """)
-    group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
-    group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
-
-class AssertionState:
-    """State for the assertion plugin."""
-
-    def __init__(self, config, mode):
-        self.mode = mode
-        self.trace = config.trace.root.get("assertion")
-
-def pytest_configure(config):
-    mode = config.getvalue("assertmode")
-    if config.getvalue("noassert") or config.getvalue("nomagic"):
-        mode = "plain"
-    if mode == "rewrite":
-        try:
-            import ast
-        except ImportError:
-            mode = "reinterp"
-        else:
-            if sys.platform.startswith('java'):
-                mode = "reinterp"
-    if mode != "plain":
-        _load_modules(mode)
-        m = monkeypatch()
-        config._cleanup.append(m.undo)
-        m.setattr(py.builtin.builtins, 'AssertionError',
-                  reinterpret.AssertionError)
-    hook = None
-    if mode == "rewrite":
-        hook = rewrite.AssertionRewritingHook()
-        sys.meta_path.append(hook)
-    warn_about_missing_assertion(mode)
-    config._assertstate = AssertionState(config, mode)
-    config._assertstate.hook = hook
-    config._assertstate.trace("configured with mode set to %r" % (mode,))
-
-def pytest_unconfigure(config):
-    hook = config._assertstate.hook
-    if hook is not None:
-        sys.meta_path.remove(hook)
-
-def pytest_collection(session):
-    # this hook is only called when test modules are collected
-    # so for example not in the master process of pytest-xdist
-    # (which does not collect test modules)
-    hook = session.config._assertstate.hook
-    if hook is not None:
-        hook.set_session(session)
-
-def pytest_runtest_setup(item):
-    def callbinrepr(op, left, right):
-        hook_result = item.ihook.pytest_assertrepr_compare(
-            config=item.config, op=op, left=left, right=right)
-        for new_expl in hook_result:
-            if new_expl:
-                res = '\n~'.join(new_expl)
-                if item.config.getvalue("assertmode") == "rewrite":
-                    # The result will be fed back a python % formatting
-                    # operation, which will fail if there are extraneous
-                    # '%'s in the string. Escape them here.
-                    res = res.replace("%", "%%")
-                return res
-    util._reprcompare = callbinrepr
-
-def pytest_runtest_teardown(item):
-    util._reprcompare = None
-
-def pytest_sessionfinish(session):
-    hook = session.config._assertstate.hook
-    if hook is not None:
-        hook.session = None
-
-def _load_modules(mode):
-    """Lazily import assertion related code."""
-    global rewrite, reinterpret
-    from _pytest.assertion import reinterpret
-    if mode == "rewrite":
-        from _pytest.assertion import rewrite
-
-def warn_about_missing_assertion(mode):
-    try:
-        assert False
-    except AssertionError:
-        pass
-    else:
-        if mode == "rewrite":
-            specifically = ("assertions which are not in test modules "
-                            "will be ignored")
-        else:
-            specifically = "failing tests may report as passing"
-
-        sys.stderr.write("WARNING: " + specifically +
-                        " because assert statements are not executed "
-                        "by the underlying Python interpreter "
-                        "(are you using python -O?)\n")
-
-pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/newinterpret.py
+++ /dev/null
@@ -1,333 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace oldinterpret.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from _pytest.assertion import util
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-
-if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(e):
-    explanation = util.format_explanation(e.explanation)
-    value = e.cause[1]
-    if str(value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.cause[0].__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = None
-        if local is None or not self.frame.is_true(local):
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not self.frame.is_true(result):
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        if util._reprcompare is not None:
-            res = util._reprcompare(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        source_explanation, source_result = self.visit(attr.value)
-        explanation = "%s.%s" % (source_explanation, attr.attr)
-        source = "__exprinfo_expr.%s" % (attr.attr,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            raise Failure(explanation)
-        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
-                                              self.frame.repr(result),
-                                              source_explanation, attr.attr)
-        # Check if the attr is from an instance.
-        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
-        source = source % (attr.attr,)
-        co = self._compile(source)
-        try:
-            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
-        except Exception:
-            from_instance = None
-        if from_instance is None or self.frame.is_true(from_instance):
-            rep = self.frame.repr(result)
-            pattern = "%s\n{%s = %s\n}"
-            explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def visit_Assert(self, assrt):
-        test_explanation, test_result = self.visit(assrt.test)
-        explanation = "assert %s" % (test_explanation,)
-        if not self.frame.is_true(test_result):
-            try:
-                raise BuiltinAssertionError
-            except Exception:
-                raise Failure(explanation)
-        return explanation, test_result
-
-    def visit_Assign(self, assign):
-        value_explanation, value_result = self.visit(assign.value)
-        explanation = "... = %s" % (value_explanation,)
-        name = ast.Name("__exprinfo_expr", ast.Load(),
-                        lineno=assign.value.lineno,
-                        col_offset=assign.value.col_offset)
-        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
-                                col_offset=assign.col_offset)
-        mod = ast.Module([new_assign])
-        co = self._compile(mod, "exec")
-        try:
-            self.frame.exec_(co, __exprinfo_expr=value_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/oldinterpret.py
+++ /dev/null
@@ -1,552 +0,0 @@
-import py
-import sys, inspect
-from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
-
-passthroughex = py.builtin._sysex
-
-class Failure:
-    def __init__(self, node):
-        self.exc, self.value, self.tb = sys.exc_info()
-        self.node = node
-
-class View(object):
-    """View base class.
-
-    If C is a subclass of View, then C(x) creates a proxy object around
-    the object x.  The actual class of the proxy is not C in general,
-    but a *subclass* of C determined by the rules below.  To avoid confusion
-    we call view class the class of the proxy (a subclass of C, so of View)
-    and object class the class of x.
-
-    Attributes and methods not found in the proxy are automatically read on x.
-    Other operations like setting attributes are performed on the proxy, as
-    determined by its view class.  The object x is available from the proxy
-    as its __obj__ attribute.
-
-    The view class selection is determined by the __view__ tuples and the
-    optional __viewkey__ method.  By default, the selected view class is the
-    most specific subclass of C whose __view__ mentions the class of x.
-    If no such subclass is found, the search proceeds with the parent
-    object classes.  For example, C(True) will first look for a subclass
-    of C with __view__ = (..., bool, ...) and only if it doesn't find any
-    look for one with __view__ = (..., int, ...), and then ..., object,...
-    If everything fails the class C itself is considered to be the default.
-
-    Alternatively, the view class selection can be driven by another aspect
-    of the object x, instead of the class of x, by overriding __viewkey__.
-    See last example at the end of this module.
-    """
-
-    _viewcache = {}
-    __view__ = ()
-
-    def __new__(rootclass, obj, *args, **kwds):
-        self = object.__new__(rootclass)
-        self.__obj__ = obj
-        self.__rootclass__ = rootclass
-        key = self.__viewkey__()
-        try:
-            self.__class__ = self._viewcache[key]
-        except KeyError:
-            self.__class__ = self._selectsubclass(key)
-        return self
-
-    def __getattr__(self, attr):
-        # attributes not found in the normal hierarchy rooted on View
-        # are looked up in the object's real class
-        return getattr(self.__obj__, attr)
-
-    def __viewkey__(self):
-        return self.__obj__.__class__
-
-    def __matchkey__(self, key, subclasses):
-        if inspect.isclass(key):
-            keys = inspect.getmro(key)
-        else:
-            keys = [key]
-        for key in keys:
-            result = [C for C in subclasses if key in C.__view__]
-            if result:
-                return result
-        return []
-
-    def _selectsubclass(self, key):
-        subclasses = list(enumsubclasses(self.__rootclass__))
-        for C in subclasses:
-            if not isinstance(C.__view__, tuple):
-                C.__view__ = (C.__view__,)
-        choices = self.__matchkey__(key, subclasses)
-        if not choices:
-            return self.__rootclass__
-        elif len(choices) == 1:
-            return choices[0]
-        else:
-            # combine the multiple choices
-            return type('?', tuple(choices), {})
-
-    def __repr__(self):
-        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
-
-
-def enumsubclasses(cls):
-    for subcls in cls.__subclasses__():
-        for subsubclass in enumsubclasses(subcls):
-            yield subsubclass
-    yield cls
-
-
-class Interpretable(View):
-    """A parse tree node with a few extra methods."""
-    explanation = None
-
-    def is_builtin(self, frame):
-        return False
-
-    def eval(self, frame):
-        # fall-back for unknown expression nodes
-        try:
-            expr = ast.Expression(self.__obj__)
-            expr.filename = '<eval>'
-            self.__obj__.filename = '<eval>'
-            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
-            result = frame.eval(co)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-        self.result = result
-        self.explanation = self.explanation or frame.repr(self.result)
-
-    def run(self, frame):
-        # fall-back for unknown statement nodes
-        try:
-            expr = ast.Module(None, ast.Stmt([self.__obj__]))
-            expr.filename = '<run>'
-            co = pycodegen.ModuleCodeGenerator(expr).getCode()
-            frame.exec_(co)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-
-    def nice_explanation(self):
-        return format_explanation(self.explanation)
-
-
-class Name(Interpretable):
-    __view__ = ast.Name
-
-    def is_local(self, frame):
-        source = '%r in locals() is not globals()' % self.name
-        try:
-            return frame.is_true(frame.eval(source))
-        except passthroughex:
-            raise
-        except:
-            return False
-
-    def is_global(self, frame):
-        source = '%r in globals()' % self.name
-        try:
-            return frame.is_true(frame.eval(source))
-        except passthroughex:
-            raise
-        except:
-            return False
-
-    def is_builtin(self, frame):
-        source = '%r not in locals() and %r not in globals()' % (
-            self.name, self.name)
-        try:
-            return frame.is_true(frame.eval(source))
-        except passthroughex:
-            raise
-        except:
-            return False
-
-    def eval(self, frame):
-        super(Name, self).eval(frame)
-        if not self.is_local(frame):
-            self.explanation = self.name
-
-class Compare(Interpretable):
-    __view__ = ast.Compare
-
-    def eval(self, frame):
-        expr = Interpretable(self.expr)
-        expr.eval(frame)
-        for operation, expr2 in self.ops:
-            if hasattr(self, 'result'):
-                # shortcutting in chained expressions
-                if not frame.is_true(self.result):
-                    break
-            expr2 = Interpretable(expr2)
-            expr2.eval(frame)
-            self.explanation = "%s %s %s" % (
-                expr.explanation, operation, expr2.explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % operation
-            try:
-                self.result = frame.eval(source,
-                                         __exprinfo_left=expr.result,
-                                         __exprinfo_right=expr2.result)
-            except passthroughex:
-                raise
-            except:
-                raise Failure(self)
-            expr = expr2
-
-class And(Interpretable):
-    __view__ = ast.And
-
-    def eval(self, frame):
-        explanations = []
-        for expr in self.nodes:
-            expr = Interpretable(expr)
-            expr.eval(frame)
-            explanations.append(expr.explanation)
-            self.result = expr.result
-            if not frame.is_true(expr.result):
-                break
-        self.explanation = '(' + ' and '.join(explanations) + ')'
-
-class Or(Interpretable):
-    __view__ = ast.Or
-
-    def eval(self, frame):
-        explanations = []
-        for expr in self.nodes:
-            expr = Interpretable(expr)
-            expr.eval(frame)
-            explanations.append(expr.explanation)
-            self.result = expr.result
-            if frame.is_true(expr.result):
-                break
-        self.explanation = '(' + ' or '.join(explanations) + ')'
-
-
-# == Unary operations ==
-keepalive = []
-for astclass, astpattern in {
-    ast.Not    : 'not __exprinfo_expr',
-    ast.Invert : '(~__exprinfo_expr)',
-    }.items():
-
-    class UnaryArith(Interpretable):
-        __view__ = astclass
-
-        def eval(self, frame, astpattern=astpattern):
-            expr = Interpretable(self.expr)
-            expr.eval(frame)
-            self.explanation = astpattern.replace('__exprinfo_expr',
-                                                  expr.explanation)
-            try:
-                self.result = frame.eval(astpattern,
-                                         __exprinfo_expr=expr.result)
-            except passthroughex:
-                raise
-            except:
-                raise Failure(self)
-
-    keepalive.append(UnaryArith)
-
-# == Binary operations ==
-for astclass, astpattern in {
-    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
-    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
-    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
-    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
-    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
-    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
-    }.items():
-
-    class BinaryArith(Interpretable):
-        __view__ = astclass
-
-        def eval(self, frame, astpattern=astpattern):
-            left = Interpretable(self.left)
-            left.eval(frame)
-            right = Interpretable(self.right)
-            right.eval(frame)
-            self.explanation = (astpattern
-                                .replace('__exprinfo_left',  left .explanation)
-                                .replace('__exprinfo_right', right.explanation))
-            try:
-                self.result = frame.eval(astpattern,
-                                         __exprinfo_left=left.result,
-                                         __exprinfo_right=right.result)
-            except passthroughex:
-                raise
-            except:
-                raise Failure(self)
-
-    keepalive.append(BinaryArith)
-
-
-class CallFunc(Interpretable):
-    __view__ = ast.CallFunc
-
-    def is_bool(self, frame):
-        source = 'isinstance(__exprinfo_value, bool)'
-        try:
-            return frame.is_true(frame.eval(source,
-                                            __exprinfo_value=self.result))
-        except passthroughex:
-            raise
-        except:
-            return False
-
-    def eval(self, frame):
-        node = Interpretable(self.node)
-        node.eval(frame)
-        explanations = []
-        vars = {'__exprinfo_fn': node.result}
-        source = '__exprinfo_fn('
-        for a in self.args:
-            if isinstance(a, ast.Keyword):
-                keyword = a.name
-                a = a.expr
-            else:
-                keyword = None
-            a = Interpretable(a)
-            a.eval(frame)
-            argname = '__exprinfo_%d' % len(vars)
-            vars[argname] = a.result
-            if keyword is None:
-                source += argname + ','
-                explanations.append(a.explanation)
-            else:
-                source += '%s=%s,' % (keyword, argname)
-                explanations.append('%s=%s' % (keyword, a.explanation))
-        if self.star_args:
-            star_args = Interpretable(self.star_args)
-            star_args.eval(frame)
-            argname = '__exprinfo_star'
-            vars[argname] = star_args.result
-            source += '*' + argname + ','
-            explanations.append('*' + star_args.explanation)
-        if self.dstar_args:
-            dstar_args = Interpretable(self.dstar_args)
-            dstar_args.eval(frame)
-            argname = '__exprinfo_kwds'
-            vars[argname] = dstar_args.result
-            source += '**' + argname + ','
-            explanations.append('**' + dstar_args.explanation)
-        self.explanation = "%s(%s)" % (
-            node.explanation, ', '.join(explanations))
-        if source.endswith(','):
-            source = source[:-1]
-        source += ')'
-        try:
-            self.result = frame.eval(source, **vars)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-        if not node.is_builtin(frame) or not self.is_bool(frame):
-            r = frame.repr(self.result)
-            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
-
-class Getattr(Interpretable):
-    __view__ = ast.Getattr
-
-    def eval(self, frame):
-        expr = Interpretable(self.expr)
-        expr.eval(frame)
-        source = '__exprinfo_expr.%s' % self.attrname
-        try:
-            self.result = frame.eval(source, __exprinfo_expr=expr.result)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
-        # if the attribute comes from the instance, its value is interesting
-        source = ('hasattr(__exprinfo_expr, "__dict__") and '
-                  '%r in __exprinfo_expr.__dict__' % self.attrname)
-        try:
-            from_instance = frame.is_true(
-                frame.eval(source, __exprinfo_expr=expr.result))
-        except passthroughex:
-            raise
-        except:
-            from_instance = True
-        if from_instance:
-            r = frame.repr(self.result)
-            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
-
-# == Re-interpretation of full statements ==
-
-class Assert(Interpretable):
-    __view__ = ast.Assert
-
-    def run(self, frame):
-        test = Interpretable(self.test)
-        test.eval(frame)
-        # print the result as  'assert <explanation>'
-        self.result = test.result
-        self.explanation = 'assert ' + test.explanation
-        if not frame.is_true(test.result):
-            try:
-                raise BuiltinAssertionError
-            except passthroughex:
-                raise
-            except:
-                raise Failure(self)
-
-class Assign(Interpretable):
-    __view__ = ast.Assign
-
-    def run(self, frame):
-        expr = Interpretable(self.expr)
-        expr.eval(frame)
-        self.result = expr.result
-        self.explanation = '... = ' + expr.explanation
-        # fall-back-run the rest of the assignment
-        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
-        mod = ast.Module(None, ast.Stmt([ass]))
-        mod.filename = '<run>'
-        co = pycodegen.ModuleCodeGenerator(mod).getCode()
-        try:
-            frame.exec_(co, __exprinfo_expr=expr.result)
-        except passthroughex:
-            raise
-        except:
-            raise Failure(self)
-
-class Discard(Interpretable):
-    __view__ = ast.Discard
-
-    def run(self, frame):
-        expr = Interpretable(self.expr)
-        expr.eval(frame)
-        self.result = expr.result
-        self.explanation = expr.explanation
-
-class Stmt(Interpretable):
-    __view__ = ast.Stmt
-
-    def run(self, frame):
-        for stmt in self.nodes:
-            stmt = Interpretable(stmt)
-            stmt.run(frame)
-
-
-def report_failure(e):
-    explanation = e.node.nice_explanation()
-    if explanation:
-        explanation = ", in: " + explanation
-    else:
-        explanation = ""
-    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
-
-def check(s, frame=None):
-    if frame is None:
-        frame = sys._getframe(1)
-        frame = py.code.Frame(frame)
-    expr = parse(s, 'eval')
-    assert isinstance(expr, ast.Expression)
-    node = Interpretable(expr.node)
-    try:
-        node.eval(frame)
-    except passthroughex:
-        raise
-    except Failure:
-        e = sys.exc_info()[1]
-        report_failure(e)
-    else:
-        if not frame.is_true(node.result):
-            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
-
-
-###########################################################
-# API / Entry points
-# #########################################################
-
-def interpret(source, frame, should_fail=False):
-    module = Interpretable(parse(source, 'exec').node)
-    #print "got module", module
-    if isinstance(frame, py.std.types.FrameType):
-        frame = py.code.Frame(frame)
-    try:
-        module.run(frame)
-    except Failure:
-        e = sys.exc_info()[1]
-        return getfailure(e)
-    except passthroughex:
-        raise
-    except:
-        import traceback
-        traceback.print_exc()
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --assert=plain)")
-    else:
-        return None
-
-def getmsg(excinfo):
-    if isinstance(excinfo, tuple):
-        excinfo = py.code.ExceptionInfo(excinfo)
-    #frame, line = gettbline(tb)
-    #frame = py.code.Frame(frame)
-    #return interpret(line, frame)
-
-    tb = excinfo.traceback[-1]
-    source = str(tb.statement).strip()
-    x = interpret(source, tb.frame, should_fail=True)
-    if not isinstance(x, str):
-        raise TypeError("interpret returned non-string %r" % (x,))
-    return x
-
-def getfailure(e):
-    explanation = e.node.nice_explanation()
-    if str(e.value):
-        lines = explanation.split('\n')
-        lines[0] += "  << %s" % (e.value,)
-        explanation = '\n'.join(lines)
-    text = "%s: %s" % (e.exc.__name__, explanation)
-    if text.startswith('AssertionError: assert '):
-        text = text[16:]
-    return text
-
-def run(s, frame=None):
-    if frame is None:
-        frame = sys._getframe(1)
-        frame = py.code.Frame(frame)
-    module = Interpretable(parse(s, 'exec').node)
-    try:
-        module.run(frame)
-    except Failure:
-        e = sys.exc_info()[1]
-        report_failure(e)
-
-
-if __name__ == '__main__':
-    # example:
-    def f():
-        return 5
-    def g():
-        return 3
-    def h(x):
-        return 'never'
-    check("f() * g() == 5")
-    check("not f()")
-    check("not (f() and g() or 0)")
-    check("f() == g()")
-    i = 4
-    check("i == f()")
-    check("len(f()) == 0")
-    check("isinstance(2+3+4, float)")
-
-    run("x = i")
-    check("x == 5")
-
-    run("assert not f(), 'oops'")
-    run("a, b, c = 1, 2")
-    run("a, b, c = f()")
-
-    check("max([f(),g()]) == 4")
-    check("'hello'[g()] == 'h'")
-    run("'guk%d' % h(f())")
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
deleted file mode 100644
--- a/_pytest/assertion/reinterpret.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import sys
-import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
-
-class AssertionError(BuiltinAssertionError):
-    def __init__(self, *args):
-        BuiltinAssertionError.__init__(self, *args)
-        if args:
-            try:
-                self.msg = str(args[0])
-            except py.builtin._sysex:
-                raise
-            except:
-                self.msg = "<[broken __repr__] %s at %0xd>" %(
-                    args[0].__class__, id(args[0]))
-        else:
-            f = py.code.Frame(sys._getframe(1))
-            try:
-                source = f.code.fullsource
-                if source is not None:
-                    try:
-                        source = source.getstatement(f.lineno, assertion=True)
-                    except IndexError:
-                        source = None
-                    else:
-                        source = str(source.deindent()).strip()
-            except py.error.ENOENT:
-                source = None
-                # this can also occur during reinterpretation, when the
-                # co_filename is set to "<run>".
-            if source:
-                self.msg = reinterpret(source, f, should_fail=True)
-            else:
-                self.msg = "<could not determine information>"
-            if not self.args:
-                self.args = (self.msg,)
-
-if sys.version_info > (3, 0):
-    AssertionError.__module__ = "builtins"
-    reinterpret_old = "old reinterpretation not available for py3"
-else:
-    from _pytest.assertion.oldinterpret import interpret as reinterpret_old
-if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
-    from _pytest.assertion.newinterpret import interpret as reinterpret
-else:
-    reinterpret = reinterpret_old
-
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
deleted file mode 100644
--- a/_pytest/assertion/rewrite.py
+++ /dev/null
@@ -1,597 +0,0 @@
-"""Rewrite assertion AST to produce nice error messages"""
-
-import ast
-import errno
-import itertools
-import imp
-import marshal
-import os
-import struct
-import sys
-import types
-
-import py
-from _pytest.assertion import util
-
-
-# Windows gives ENOENT in places *nix gives ENOTDIR.
-if sys.platform.startswith("win"):
-    PATH_COMPONENT_NOT_DIR = errno.ENOENT
-else:
-    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
-
-# py.test caches rewritten pycs in __pycache__.
-if hasattr(imp, "get_tag"):
-    PYTEST_TAG = imp.get_tag() + "-PYTEST"
-else:
-    if hasattr(sys, "pypy_version_info"):
-        impl = "pypy"
-    elif sys.platform == "java":
-        impl = "jython"
-    else:
-        impl = "cpython"
-    ver = sys.version_info
-    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
-    del ver, impl
-
-PYC_EXT = ".py" + "c" if __debug__ else "o"
-PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
-
-REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
-
-class AssertionRewritingHook(object):
-    """Import hook which rewrites asserts."""
-
-    def __init__(self):
-        self.session = None
-        self.modules = {}
-
-    def set_session(self, session):
-        self.fnpats = session.config.getini("python_files")
-        self.session = session
-
-    def find_module(self, name, path=None):
-        if self.session is None:
-            return None
-        sess = self.session
-        state = sess.config._assertstate
-        state.trace("find_module called for: %s" % name)
-        names = name.rsplit(".", 1)
-        lastname = names[-1]
-        pth = None
-        if path is not None and len(path) == 1:
-            pth = path[0]
-        if pth is None:
-            try:
-                fd, fn, desc = imp.find_module(lastname, path)
-            except ImportError:
-                return None
-            if fd is not None:
-                fd.close()
-            tp = desc[2]
-            if tp == imp.PY_COMPILED:
-                if hasattr(imp, "source_from_cache"):
-                    fn = imp.source_from_cache(fn)
-                else:
-                    fn = fn[:-1]
-            elif tp != imp.PY_SOURCE:
-                # Don't know what this is.
-                return None
-        else:
-            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
-        fn_pypath = py.path.local(fn)
-        # Is this a test file?
-        if not sess.isinitpath(fn):
-            # We have to be very careful here because imports in this code can
-            # trigger a cycle.
-            self.session = None
-            try:
-                for pat in self.fnpats:
-                    if fn_pypath.fnmatch(pat):
-                        state.trace("matched test file %r" % (fn,))
-                        break
-                else:
-                    return None
-            finally:
-                self.session = sess
-        else:
-            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
-        # The requested module looks like a test file, so rewrite it. This is
-        # the most magical part of the process: load the source, rewrite the
-        # asserts, and load the rewritten source. We also cache the rewritten
-        # module code in a special pyc. We must be aware of the possibility of
-        # concurrent py.test processes rewriting and loading pycs. To avoid
-        # tricky race conditions, we maintain the following invariant: The
-        # cached pyc is always a complete, valid pyc. Operations on it must be
-        # atomic. POSIX's atomic rename comes in handy.
-        write = not sys.dont_write_bytecode
-        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
-        if write:
-            try:
-                os.mkdir(cache_dir)
-            except OSError:
-                e = sys.exc_info()[1].errno
-                if e == errno.EEXIST:
-                    # Either the __pycache__ directory already exists (the
-                    # common case) or it's blocked by a non-dir node. In the
-                    # latter case, we'll ignore it in _write_pyc.
-                    pass
-                elif e == PATH_COMPONENT_NOT_DIR:
-                    # One of the path components was not a directory, likely
-                    # because we're in a zip file.
-                    write = False
-                elif e == errno.EACCES:
-                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
-                    write = False
-                else:
-                    raise
-        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
-        pyc = os.path.join(cache_dir, cache_name)
-        # Notice that even if we're in a read-only directory, I'm going to check
-        # for a cached pyc. This may not be optimal...
-        co = _read_pyc(fn_pypath, pyc)
-        if co is None:
-            state.trace("rewriting %r" % (fn,))
-            co = _rewrite_test(state, fn_pypath)
-            if co is None:
-                # Probably a SyntaxError in the test.
-                return None
-            if write:
-                _make_rewritten_pyc(state, fn_pypath, pyc, co)
-        else:
-            state.trace("found cached rewritten pyc for %r" % (fn,))
-        self.modules[name] = co, pyc
-        return self
-
-    def load_module(self, name):
-        co, pyc = self.modules.pop(name)
-        # I wish I could just call imp.load_compiled here, but __file__ has to
-        # be set properly. In Python 3.2+, this all would be handled correctly
-        # by load_compiled.
-        mod = sys.modules[name] = imp.new_module(name)
-        try:
-            mod.__file__ = co.co_filename
-            # Normally, this attribute is 3.2+.
-            mod.__cached__ = pyc
-            py.builtin.exec_(co, mod.__dict__)
-        except:
-            del sys.modules[name]
-            raise
-        return sys.modules[name]
-
-def _write_pyc(co, source_path, pyc):
-    # Technically, we don't have to have the same pyc format as (C)Python, since
-    # these "pycs" should never be seen by builtin import. However, there's
-    # little reason deviate, and I hope sometime to be able to use
-    # imp.load_compiled to load them. (See the comment in load_module above.)
-    mtime = int(source_path.mtime())
-    try:
-        fp = open(pyc, "wb")
-    except IOError:
-        err = sys.exc_info()[1].errno
-        if err == PATH_COMPONENT_NOT_DIR:
-            # This happens when we get a EEXIST in find_module creating the
-            # __pycache__ directory and __pycache__ is by some non-dir node.
-            return False
-        raise
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return True
-
-RN = "\r\n".encode("utf-8")
-N = "\n".encode("utf-8")
-
-def _rewrite_test(state, fn):
-    """Try to read and rewrite *fn* and return the code object."""
-    try:
-        source = fn.read("rb")
-    except EnvironmentError:
-        return None
-    # On Python versions which are not 2.7 and less than or equal to 3.1, the
-    # parser expects *nix newlines.
-    if REWRITE_NEWLINES:
-        source = source.replace(RN, N) + N
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        state.trace("failed to parse: %r" % (fn,))
-        return None
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, fn.strpath, "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the
-        # assertion rewriting, but I don't know of a fast way to tell.
-        state.trace("failed to compile: %r" % (fn,))
-        return None
-    return co
-
-def _make_rewritten_pyc(state, fn, pyc, co):
-    """Try to dump rewritten code to *pyc*."""
-    if sys.platform.startswith("win"):
-        # Windows grants exclusive access to open files and doesn't have atomic
-        # rename, so just write into the final file.
-        _write_pyc(co, fn, pyc)
-    else:
-        # When not on windows, assume rename is atomic. Dump the code object
-        # into a file specific to this process and atomically replace it.
-        proc_pyc = pyc + "." + str(os.getpid())
-        if _write_pyc(co, fn, proc_pyc):
-            os.rename(proc_pyc, pyc)
-
-def _read_pyc(source, pyc):
-    """Possibly read a py.test pyc containing rewritten code.
-
-    Return rewritten code if successful or None if not.
-    """
-    try:
-        fp = open(pyc, "rb")
-    except IOError:
-        return None
-    try:
-        try:
-            mtime = int(source.mtime())
-            data = fp.read(8)
-        except EnvironmentError:
-            return None
-        # Check for invalid or out of date pyc file.
-        if (len(data) != 8 or
-            data[:4] != imp.get_magic() or
-            struct.unpack("<l", data[4:])[0] != mtime):
-            return None
-        co = marshal.load(fp)
-        if not isinstance(co, types.CodeType):
-            # That's interesting....
-            return None
-        return co
-    finally:
-        fp.close()
-
-
-def rewrite_asserts(mod):
-    """Rewrite the assert statements in mod."""
-    AssertionRewriter().run(mod)
-
-
-_saferepr = py.io.saferepr
-from _pytest.assertion.util import format_explanation as _format_explanation
-
-def _format_boolop(explanations, is_or):
-    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
-
-def _call_reprcompare(ops, results, expls, each_obj):
-    for i, res, expl in zip(range(len(ops)), results, expls):
-        try:
-            done = not res
-        except Exception:
-            done = True
-        if done:
-            break
-    if util._reprcompare is not None:
-        custom = util._reprcompare(ops[i], each_obj[i], each_obj[i + 1])
-        if custom is not None:
-            return custom
-    return expl
-
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-binop_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-
-def set_location(node, lineno, col_offset):
-    """Set node location information recursively."""
-    def _fix(node, lineno, col_offset):
-        if "lineno" in node._attributes:
-            node.lineno = lineno
-        if "col_offset" in node._attributes:
-            node.col_offset = col_offset
-        for child in ast.iter_child_nodes(node):
-            _fix(child, lineno, col_offset)
-    _fix(node, lineno, col_offset)
-    return node
-
-
-class AssertionRewriter(ast.NodeVisitor):
-
-    def run(self, mod):
-        """Find all assert statements in *mod* and rewrite them."""
-        if not mod.body:
-            # Nothing to do.
-            return
-        # Insert some special imports at the top of the module but after any
-        # docstrings and __future__ imports.
-        aliases = [ast.alias(py.builtin.builtins.__name__, "@py_builtins"),
-                   ast.alias("_pytest.assertion.rewrite", "@pytest_ar")]
-        expect_docstring = True
-        pos = 0
-        lineno = 0
-        for item in mod.body:
-            if (expect_docstring and isinstance(item, ast.Expr) and
-                isinstance(item.value, ast.Str)):
-                doc = item.value.s
-                if "PYTEST_DONT_REWRITE" in doc:
-                    # The module has disabled assertion rewriting.
-                    return
-                lineno += len(doc) - 1
-                expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
-                  item.module != "__future__"):
-                lineno = item.lineno
-                break
-            pos += 1
-        imports = [ast.Import([alias], lineno=lineno, col_offset=0)
-                   for alias in aliases]
-        mod.body[pos:pos] = imports
-        # Collect asserts.
-        nodes = [mod]
-        while nodes:
-            node = nodes.pop()
-            for name, field in ast.iter_fields(node):
-                if isinstance(field, list):
-                    new = []
-                    for i, child in enumerate(field):
-                        if isinstance(child, ast.Assert):
-                            # Transform assert.
-                            new.extend(self.visit(child))
-                        else:
-                            new.append(child)
-                            if isinstance(child, ast.AST):
-                                nodes.append(child)
-                    setattr(node, name, new)
-                elif (isinstance(field, ast.AST) and
-                      # Don't recurse into expressions as they can't contain
-                      # asserts.
-                      not isinstance(field, ast.expr)):
-                    nodes.append(field)
-
-    def variable(self):
-        """Get a new variable."""
-        # Use a character invalid in python identifiers to avoid clashing.
-        name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.append(name)
-        return name
-
-    def assign(self, expr):
-        """Give *expr* a name."""
-        name = self.variable()
-        self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr))
-        return ast.Name(name, ast.Load())
-
-    def display(self, expr):
-        """Call py.io.saferepr on the expression."""
-        return self.helper("saferepr", expr)
-
-    def helper(self, name, *args):
-        """Call a helper in this module."""
-        py_name = ast.Name("@pytest_ar", ast.Load())
-        attr = ast.Attribute(py_name, "_" + name, ast.Load())
-        return ast.Call(attr, list(args), [], None, None)
-
-    def builtin(self, name):
-        """Return the builtin called *name*."""
-        builtin_name = ast.Name("@py_builtins", ast.Load())
-        return ast.Attribute(builtin_name, name, ast.Load())
-
-    def explanation_param(self, expr):
-        specifier = "py" + str(next(self.variable_counter))
-        self.explanation_specifiers[specifier] = expr
-        return "%(" + specifier + ")s"
-
-    def push_format_context(self):
-        self.explanation_specifiers = {}
-        self.stack.append(self.explanation_specifiers)
-
-    def pop_format_context(self, expl_expr):
-        current = self.stack.pop()
-        if self.stack:
-            self.explanation_specifiers = self.stack[-1]
-        keys = [ast.Str(key) for key in current.keys()]
-        format_dict = ast.Dict(keys, list(current.values()))
-        form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
-        name = "@py_format" + str(next(self.variable_counter))
-        self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
-        return ast.Name(name, ast.Load())
-
-    def generic_visit(self, node):
-        """Handle expressions we don't have custom code for."""
-        assert isinstance(node, ast.expr)
-        res = self.assign(node)
-        return res, self.explanation_param(self.display(res))
-
-    def visit_Assert(self, assert_):
-        if assert_.msg:
-            # There's already a message. Don't mess with it.
-            return [assert_]
-        self.statements = []
-        self.cond_chain = ()
-        self.variables = []
-        self.variable_counter = itertools.count()
-        self.stack = []
-        self.on_failure = []
-        self.push_format_context()
-        # Rewrite assert into a bunch of statements.
-        top_condition, explanation = self.visit(assert_.test)
-        # Create failure message.
-        body = self.on_failure
-        negation = ast.UnaryOp(ast.Not(), top_condition)
-        self.statements.append(ast.If(negation, body, []))
-        explanation = "assert " + explanation
-        template = ast.Str(explanation)
-        msg = self.pop_format_context(template)
-        fmt = self.helper("format_explanation", msg)
-        err_name = ast.Name("AssertionError", ast.Load())
-        exc = ast.Call(err_name, [fmt], [], None, None)
-        if sys.version_info[0] >= 3:
-            raise_ = ast.Raise(exc, None)
-        else:
-            raise_ = ast.Raise(exc, None, None)
-        body.append(raise_)
-        # Clear temporary variables by setting them to None.
-        if self.variables:
-            variables = [ast.Name(name, ast.Store()) for name in self.variables]
-            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
-            self.statements.append(clear)
-        # Fix line numbers.
-        for stmt in self.statements:
-            set_location(stmt, assert_.lineno, assert_.col_offset)
-        return self.statements
-
-    def visit_Name(self, name):
-        # Check if the name is local or not.
-        locs = ast.Call(self.builtin("locals"), [], [], None, None)
-        globs = ast.Call(self.builtin("globals"), [], [], None, None)
-        ops = [ast.In(), ast.IsNot()]
-        test = ast.Compare(ast.Str(name.id), ops, [locs, globs])
-        expr = ast.IfExp(test, self.display(name), ast.Str(name.id))
-        return name, self.explanation_param(expr)
-
-    def visit_BoolOp(self, boolop):
-        res_var = self.variable()
-        expl_list = self.assign(ast.List([], ast.Load()))
-        app = ast.Attribute(expl_list, "append", ast.Load())
-        is_or = int(isinstance(boolop.op, ast.Or))
-        body = save = self.statements
-        fail_save = self.on_failure
-        levels = len(boolop.values) - 1
-        self.push_format_context()
-        # Process each operand, short-circuting if needed.
-        for i, v in enumerate(boolop.values):
-            if i:
-                fail_inner = []
-                self.on_failure.append(ast.If(cond, fail_inner, []))
-                self.on_failure = fail_inner
-            self.push_format_context()
-            res, expl = self.visit(v)
-            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
-            expl_format = self.pop_format_context(ast.Str(expl))
-            call = ast.Call(app, [expl_format], [], None, None)
-            self.on_failure.append(ast.Expr(call))
-            if i < levels:
-                cond = res
-                if is_or:
-                    cond = ast.UnaryOp(ast.Not(), cond)
-                inner = []
-                self.statements.append(ast.If(cond, inner, []))
-                self.statements = body = inner
-        self.statements = save
-        self.on_failure = fail_save
-        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
-        expl = self.pop_format_context(expl_template)
-        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_res, operand_expl = self.visit(unary.operand)
-        res = self.assign(ast.UnaryOp(unary.op, operand_res))
-        return res, pattern % (operand_expl,)
-
-    def visit_BinOp(self, binop):
-        symbol = binop_map[binop.op.__class__]
-        left_expr, left_expl = self.visit(binop.left)
-        right_expr, right_expl = self.visit(binop.right)
-        explanation = "(%s %s %s)" % (left_expl, symbol, right_expl)
-        res = self.assign(ast.BinOp(left_expr, binop.op, right_expr))
-        return res, explanation
-
-    def visit_Call(self, call):
-        new_func, func_expl = self.visit(call.func)
-        arg_expls = []
-        new_args = []
-        new_kwargs = []
-        new_star = new_kwarg = None
-        for arg in call.args:
-            res, expl = self.visit(arg)
-            new_args.append(res)
-            arg_expls.append(expl)
-        for keyword in call.keywords:
-            res, expl = self.visit(keyword.value)
-            new_kwargs.append(ast.keyword(keyword.arg, res))
-            arg_expls.append(keyword.arg + "=" + expl)
-        if call.starargs:
-            new_star, expl = self.visit(call.starargs)
-            arg_expls.append("*" + expl)
-        if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwargs)
-            arg_expls.append("**" + expl)
-        expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
-        new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
-        res = self.assign(new_call)
-        res_expl = self.explanation_param(self.display(res))
-        outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
-        return res, outer_expl
-
-    def visit_Attribute(self, attr):
-        if not isinstance(attr.ctx, ast.Load):
-            return self.generic_visit(attr)
-        value, value_expl = self.visit(attr.value)
-        res = self.assign(ast.Attribute(value, attr.attr, ast.Load()))
-        res_expl = self.explanation_param(self.display(res))
-        pat = "%s\n{%s = %s.%s\n}"
-        expl = pat % (res_expl, res_expl, value_expl, attr.attr)
-        return res, expl
-
-    def visit_Compare(self, comp):
-        self.push_format_context()
-        left_res, left_expl = self.visit(comp.left)
-        res_variables = [self.variable() for i in range(len(comp.ops))]
-        load_names = [ast.Name(v, ast.Load()) for v in res_variables]
-        store_names = [ast.Name(v, ast.Store()) for v in res_variables]
-        it = zip(range(len(comp.ops)), comp.ops, comp.comparators)
-        expls = []
-        syms = []
-        results = [left_res]
-        for i, op, next_operand in it:
-            next_res, next_expl = self.visit(next_operand)
-            results.append(next_res)
-            sym = binop_map[op.__class__]
-            syms.append(ast.Str(sym))
-            expl = "%s %s %s" % (left_expl, sym, next_expl)
-            expls.append(ast.Str(expl))
-            res_expr = ast.Compare(left_res, [op], [next_res])
-            self.statements.append(ast.Assign([store_names[i]], res_expr))
-            left_res, left_expl = next_res, next_expl
-        # Use py.code._reprcompare if that's available.
-        expl_call = self.helper("call_reprcompare",
-                                ast.Tuple(syms, ast.Load()),
-                                ast.Tuple(load_names, ast.Load()),
-                                ast.Tuple(expls, ast.Load()),
-                                ast.Tuple(results, ast.Load()))
-        if len(comp.ops) > 1:
-            res = ast.BoolOp(ast.And(), load_names)
-        else:
-            res = load_names[0]
-        return res, self.explanation_param(self.pop_format_context(expl_call))
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
deleted file mode 100644
--- a/_pytest/assertion/util.py
+++ /dev/null
@@ -1,213 +0,0 @@
-"""Utilities for assertion debugging"""
-
-import py
-
-
-# The _reprcompare attribute on the util module is used by the new assertion
-# interpretation code and assertion rewriter to detect this plugin was
-# loaded and in turn call the hooks defined here as part of the
-# DebugInterpreter.
-_reprcompare = None
-
-def format_explanation(explanation):
-    """This formats an explanation
-
-    Normally all embedded newlines are escaped, however there are
-    three exceptions: \n{, \n} and \n~.  The first two are intended
-    cover nested explanations, see function and attribute explanations
-    for examples (.visit_Call(), visit_Attribute()).  The last one is
-    for when one explanation needs to span multiple lines, e.g. when
-    displaying diffs.
-    """
-    # simplify 'assert False where False = ...'
-    where = 0
-    while True:
-        start = where = explanation.find("False\n{False = ", where)
-        if where == -1:
-            break
-        level = 0
-        for i, c in enumerate(explanation[start:]):
-            if c == "{":
-                level += 1
-            elif c == "}":
-                level -= 1
-                if not level:
-                    break
-        else:
-            raise AssertionError("unbalanced braces: %r" % (explanation,))
-        end = start + i
-        where = end
-        if explanation[end - 1] == '\n':
-            explanation = (explanation[:start] + explanation[start+15:end-1] +
-                           explanation[end+1:])
-            where -= 17
-    raw_lines = (explanation or '').split('\n')
-    # escape newlines not followed by {, } and ~
-    lines = [raw_lines[0]]
-    for l in raw_lines[1:]:
-        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
-            lines.append(l)
-        else:
-            lines[-1] += '\\n' + l
-
-    result = lines[:1]
-    stack = [0]
-    stackcnt = [0]
-    for line in lines[1:]:
-        if line.startswith('{'):
-            if stackcnt[-1]:
-                s = 'and   '
-            else:
-                s = 'where '
-            stack.append(len(result))
-            stackcnt[-1] += 1
-            stackcnt.append(0)
-            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
-        elif line.startswith('}'):
-            assert line.startswith('}')
-            stack.pop()
-            stackcnt.pop()
-            result[stack[-1]] += line[1:]
-        else:
-            assert line.startswith('~')
-            result.append('  '*len(stack) + line[1:])
-    assert len(stack) == 1
-    return '\n'.join(result)
-
-
-# Provide basestring in python3
-try:
-    basestring = basestring
-except NameError:
-    basestring = str
-
-
-def assertrepr_compare(op, left, right):
-    """return specialised explanations for some operators/operands"""
-    width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
-    left_repr = py.io.saferepr(left, maxsize=int(width/2))
-    right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
-    summary = '%s %s %s' % (left_repr, op, right_repr)
-
-    issequence = lambda x: isinstance(x, (list, tuple))
-    istext = lambda x: isinstance(x, basestring)
-    isdict = lambda x: isinstance(x, dict)
-    isset = lambda x: isinstance(x, set)
-
-    explanation = None
-    try:
-        if op == '==':
-            if istext(left) and istext(right):
-                explanation = _diff_text(left, right)
-            elif issequence(left) and issequence(right):
-                explanation = _compare_eq_sequence(left, right)
-            elif isset(left) and isset(right):
-                explanation = _compare_eq_set(left, right)
-            elif isdict(left) and isdict(right):
-                explanation = _diff_text(py.std.pprint.pformat(left),
-                                         py.std.pprint.pformat(right))
-        elif op == 'not in':
-            if istext(left) and istext(right):
-                explanation = _notin_text(left, right)
-    except py.builtin._sysex:
-        raise
-    except:
-        excinfo = py.code.ExceptionInfo()
-        explanation = ['(pytest_assertion plugin: representation of '
-            'details failed. Probably an object has a faulty __repr__.)',
-            str(excinfo)
-            ]
-
-
-    if not explanation:
-        return None
-
-    # Don't include pageloads of data, should be configurable
-    if len(''.join(explanation)) > 80*8:
-        explanation = ['Detailed information too verbose, truncated']
-
-    return [summary] + explanation
-
-
-def _diff_text(left, right):
-    """Return the explanation for the diff between text
-
-    This will skip leading and trailing characters which are
-    identical to keep the diff minimal.
-    """
-    explanation = []
-    i = 0 # just in case left or right has zero length
-    for i in range(min(len(left), len(right))):
-        if left[i] != right[i]:
-            break
-    if i > 42:
-        i -= 10                 # Provide some context
-        explanation = ['Skipping %s identical '
-                       'leading characters in diff' % i]
-        left = left[i:]
-        right = right[i:]
-    if len(left) == len(right):
-        for i in range(len(left)):
-            if left[-i] != right[-i]:
-                break
-        if i > 42:
-            i -= 10     # Provide some context
-            explanation += ['Skipping %s identical '
-                            'trailing characters in diff' % i]
-            left = left[:-i]
-            right = right[:-i]
-    explanation += [line.strip('\n')
-                    for line in py.std.difflib.ndiff(left.splitlines(),
-                                                     right.splitlines())]
-    return explanation
-
-
-def _compare_eq_sequence(left, right):
-    explanation = []
-    for i in range(min(len(left), len(right))):
-        if left[i] != right[i]:
-            explanation += ['At index %s diff: %r != %r' %
-                            (i, left[i], right[i])]
-            break
-    if len(left) > len(right):
-        explanation += ['Left contains more items, '
-            'first extra item: %s' % py.io.saferepr(left[len(right)],)]
-    elif len(left) < len(right):
-        explanation += ['Right contains more items, '
-            'first extra item: %s' % py.io.saferepr(right[len(left)],)]
-    return explanation # + _diff_text(py.std.pprint.pformat(left),
-                       #             py.std.pprint.pformat(right))
-
-
-def _compare_eq_set(left, right):
-    explanation = []
-    diff_left = left - right
-    diff_right = right - left
-    if diff_left:
-        explanation.append('Extra items in the left set:')
-        for item in diff_left:
-            explanation.append(py.io.saferepr(item))
-    if diff_right:
-        explanation.append('Extra items in the right set:')
-        for item in diff_right:
-            explanation.append(py.io.saferepr(item))
-    return explanation
-
-
-def _notin_text(term, text):
-    index = text.find(term)
-    head = text[:index]
-    tail = text[index+len(term):]
-    correct_text = head + tail
-    diff = _diff_text(correct_text, text)
-    newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
-    for line in diff:
-        if line.startswith('Skipping'):
-            continue
-        if line.startswith('- '):
-            continue
-        if line.startswith('+ '):
-            newdiff.append('  ' + line[2:])
-        else:
-            newdiff.append(line)
-    return newdiff
diff --git a/_pytest/capture.py b/_pytest/capture.py
deleted file mode 100644
--- a/_pytest/capture.py
+++ /dev/null
@@ -1,214 +0,0 @@
-""" per-test stdout/stderr capturing mechanisms, ``capsys`` and ``capfd`` function arguments.  """
-
-import pytest, py
-import os
-
-def pytest_addoption(parser):
-    group = parser.getgroup("general")
-    group._addoption('--capture', action="store", default=None,
-        metavar="method", type="choice", choices=['fd', 'sys', 'no'],
-        help="per-test capturing method: one of fd (default)|sys|no.")
-    group._addoption('-s', action="store_const", const="no", dest="capture",
-        help="shortcut for --capture=no.")
-
- at pytest.mark.tryfirst
-def pytest_cmdline_parse(pluginmanager, args):
-    # we want to perform capturing already for plugin/conftest loading
-    if '-s' in args or "--capture=no" in args:
-        method = "no"
-    elif hasattr(os, 'dup') and '--capture=sys' not in args:
-        method = "fd"
-    else:
-        method = "sys"
-    capman = CaptureManager(method)
-    pluginmanager.register(capman, "capturemanager")
-
-def addouterr(rep, outerr):
-    for secname, content in zip(["out", "err"], outerr):
-        if content:
-            rep.sections.append(("Captured std%s" % secname, content))
-
-class NoCapture:
-    def startall(self):
-        pass
-    def resume(self):
-        pass
-    def reset(self):
-        pass
-    def suspend(self):
-        return "", ""
-
-class CaptureManager:
-    def __init__(self, defaultmethod=None):
-        self._method2capture = {}
-        self._defaultmethod = defaultmethod
-
-    def _maketempfile(self):
-        f = py.std.tempfile.TemporaryFile()
-        newf = py.io.dupfile(f, encoding="UTF-8")
-        f.close()
-        return newf
-
-    def _makestringio(self):
-        return py.io.TextIO()
-
-    def _getcapture(self, method):
-        if method == "fd":
-            return py.io.StdCaptureFD(now=False,
-                out=self._maketempfile(), err=self._maketempfile()
-            )
-        elif method == "sys":
-            return py.io.StdCapture(now=False,
-                out=self._makestringio(), err=self._makestringio()
-            )
-        elif method == "no":
-            return NoCapture()
-        else:
-            raise ValueError("unknown capturing method: %r" % method)
-
-    def _getmethod(self, config, fspath):
-        if config.option.capture:
-            method = config.option.capture
-        else:
-            try:
-                method = config._conftest.rget("option_capture", path=fspath)
-            except KeyError:
-                method = "fd"
-        if method == "fd" and not hasattr(os, 'dup'): # e.g. jython
-            method = "sys"
-        return method
-
-    def reset_capturings(self):
-        for name, cap in self._method2capture.items():
-            cap.reset()
-
-    def resumecapture_item(self, item):
-        method = self._getmethod(item.config, item.fspath)
-        if not hasattr(item, 'outerr'):
-            item.outerr = ('', '') # we accumulate outerr on the item
-        return self.resumecapture(method)
-
-    def resumecapture(self, method=None):
-        if hasattr(self, '_capturing'):
-            raise ValueError("cannot resume, already capturing with %r" %
-                (self._capturing,))
-        if method is None:
-            method = self._defaultmethod
-        cap = self._method2capture.get(method)
-        self._capturing = method
-        if cap is None:
-            self._method2capture[method] = cap = self._getcapture(method)
-            cap.startall()
-        else:
-            cap.resume()
-
-    def suspendcapture(self, item=None):
-        self.deactivate_funcargs()
-        if hasattr(self, '_capturing'):
-            method = self._capturing
-            cap = self._method2capture.get(method)
-            if cap is not None:
-                outerr = cap.suspend()
-            del self._capturing
-            if item:
-                outerr = (item.outerr[0] + outerr[0],
-                          item.outerr[1] + outerr[1])
-            return outerr
-        if hasattr(item, 'outerr'):
-            return item.outerr
-        return "", ""
-
-    def activate_funcargs(self, pyfuncitem):
-        if not hasattr(pyfuncitem, 'funcargs'):
-            return
-        assert not hasattr(self, '_capturing_funcargs')
-        self._capturing_funcargs = capturing_funcargs = []
-        for name, capfuncarg in pyfuncitem.funcargs.items():
-            if name in ('capsys', 'capfd'):
-                capturing_funcargs.append(capfuncarg)
-                capfuncarg._start()
-
-    def deactivate_funcargs(self):
-        capturing_funcargs = getattr(self, '_capturing_funcargs', None)
-        if capturing_funcargs is not None:
-            while capturing_funcargs:
-                capfuncarg = capturing_funcargs.pop()
-                capfuncarg._finalize()
-            del self._capturing_funcargs
-
-    def pytest_make_collect_report(self, __multicall__, collector):
-        method = self._getmethod(collector.config, collector.fspath)
-        try:
-            self.resumecapture(method)
-        except ValueError:
-            return # recursive collect, XXX refactor capturing
-                   # to allow for more lightweight recursive capturing
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        addouterr(rep, outerr)
-        return rep
-
-    @pytest.mark.tryfirst
-    def pytest_runtest_setup(self, item):
-        self.resumecapture_item(item)
-
-    @pytest.mark.tryfirst
-    def pytest_runtest_call(self, item):
-        self.resumecapture_item(item)
-        self.activate_funcargs(item)
-
-    @pytest.mark.tryfirst
-    def pytest_runtest_teardown(self, item):
-        self.resumecapture_item(item)
-
-    def pytest_keyboard_interrupt(self, excinfo):
-        if hasattr(self, '_capturing'):
-            self.suspendcapture()
-
-    @pytest.mark.tryfirst
-    def pytest_runtest_makereport(self, __multicall__, item, call):
-        self.deactivate_funcargs()
-        rep = __multicall__.execute()
-        outerr = self.suspendcapture(item)
-        if not rep.passed:
-            addouterr(rep, outerr)
-        if not rep.passed or rep.when == "teardown":
-            outerr = ('', '')
-        item.outerr = outerr
-        return rep
-
-def pytest_funcarg__capsys(request):
-    """enables capturing of writes to sys.stdout/sys.stderr and makes
-    captured output available via ``capsys.readouterr()`` method calls
-    which return a ``(out, err)`` tuple.
-    """
-    return CaptureFuncarg(py.io.StdCapture)
-
-def pytest_funcarg__capfd(request):
-    """enables capturing of writes to file descriptors 1 and 2 and makes
-    captured output available via ``capsys.readouterr()`` method calls
-    which return a ``(out, err)`` tuple.
-    """
-    if not hasattr(os, 'dup'):
-        py.test.skip("capfd funcarg needs os.dup")
-    return CaptureFuncarg(py.io.StdCaptureFD)
-
-class CaptureFuncarg:
-    def __init__(self, captureclass):
-        self.capture = captureclass(now=False)
-
-    def _start(self):
-        self.capture.startall()
-
-    def _finalize(self):
-        if hasattr(self, 'capture'):
-            self.capture.reset()
-            del self.capture
-
-    def readouterr(self):
-        return self.capture.readouterr()
-
-    def close(self):
-        self._finalize()
diff --git a/_pytest/config.py b/_pytest/config.py
deleted file mode 100644
--- a/_pytest/config.py
+++ /dev/null
@@ -1,463 +0,0 @@
-""" command line options, ini-file and conftest.py processing. """
-
-import py
-import sys, os
-from _pytest.core import PluginManager
-import pytest
-
-def pytest_cmdline_parse(pluginmanager, args):
-    config = Config(pluginmanager)
-    config.parse(args)
-    return config
-
-def pytest_unconfigure(config):
-    while 1:
-        try:
-            fin = config._cleanup.pop()
-        except IndexError:
-            break
-        fin()
-
-class Parser:
-    """ Parser for command line arguments. """
-
-    def __init__(self, usage=None, processopt=None):
-        self._anonymous = OptionGroup("custom options", parser=self)
-        self._groups = []
-        self._processopt = processopt
-        self._usage = usage
-        self._inidict = {}
-        self._ininames = []
-        self.hints = []
-
-    def processoption(self, option):
-        if self._processopt:
-            if option.dest:
-                self._processopt(option)
-
-    def addnote(self, note):
-        self._notes.append(note)
-
-    def getgroup(self, name, description="", after=None):
-        """ get (or create) a named option Group.
-
-        :name: unique name of the option group.
-        :description: long description for --help output.
-        :after: name of other group, used for ordering --help output.
-        """
-        for group in self._groups:
-            if group.name == name:
-                return group
-        group = OptionGroup(name, description, parser=self)
-        i = 0
-        for i, grp in enumerate(self._groups):
-            if grp.name == after:
-                break
-        self._groups.insert(i+1, group)
-        return group
-
-    def addoption(self, *opts, **attrs):
-        """ add an optparse-style option. """
-        self._anonymous.addoption(*opts, **attrs)
-
-    def parse(self, args):
-        self.optparser = optparser = MyOptionParser(self)
-        groups = self._groups + [self._anonymous]
-        for group in groups:
-            if group.options:
-                desc = group.description or group.name
-                optgroup = py.std.optparse.OptionGroup(optparser, desc)
-                optgroup.add_options(group.options)
-                optparser.add_option_group(optgroup)
-        return self.optparser.parse_args([str(x) for x in args])
-
-    def parse_setoption(self, args, option):
-        parsedoption, args = self.parse(args)
-        for name, value in parsedoption.__dict__.items():
-            setattr(option, name, value)
-        return args
-
-    def addini(self, name, help, type=None, default=None):
-        """ add an ini-file option with the given name and description. """
-        assert type in (None, "pathlist", "args", "linelist")
-        self._inidict[name] = (help, type, default)
-        self._ininames.append(name)
-
-
-class OptionGroup:
-    def __init__(self, name, description="", parser=None):
-        self.name = name
-        self.description = description
-        self.options = []
-        self.parser = parser
-
-    def addoption(self, *optnames, **attrs):
-        """ add an option to this group. """
-        option = py.std.optparse.Option(*optnames, **attrs)
-        self._addoption_instance(option, shortupper=False)
-
-    def _addoption(self, *optnames, **attrs):
-        option = py.std.optparse.Option(*optnames, **attrs)
-        self._addoption_instance(option, shortupper=True)
-
-    def _addoption_instance(self, option, shortupper=False):
-        if not shortupper:
-            for opt in option._short_opts:
-                if opt[0] == '-' and opt[1].islower():
-                    raise ValueError("lowercase shortoptions reserved")
-        if self.parser:
-            self.parser.processoption(option)
-        self.options.append(option)
-
-
-class MyOptionParser(py.std.optparse.OptionParser):
-    def __init__(self, parser):
-        self._parser = parser
-        py.std.optparse.OptionParser.__init__(self, usage=parser._usage,
-            add_help_option=False)
-    def format_epilog(self, formatter):
-        hints = self._parser.hints
-        if hints:
-            s = "\n".join(["hint: " + x for x in hints]) + "\n"
-            s = "\n" + s + "\n"
-            return s
-        return ""
-
-class Conftest(object):
-    """ the single place for accessing values and interacting
-        towards conftest modules from py.test objects.
-    """
-    def __init__(self, onimport=None, confcutdir=None):
-        self._path2confmods = {}
-        self._onimport = onimport
-        self._conftestpath2mod = {}
-        self._confcutdir = confcutdir
-
-    def setinitial(self, args):
-        """ try to find a first anchor path for looking up global values
-            from conftests. This function is usually called _before_
-            argument parsing.  conftest files may add command line options
-            and we thus have no completely safe way of determining
-            which parts of the arguments are actually related to options
-            and which are file system paths.  We just try here to get
-            bootstrapped ...
-        """
-        current = py.path.local()
-        opt = '--confcutdir'
-        for i in range(len(args)):
-            opt1 = str(args[i])
-            if opt1.startswith(opt):
-                if opt1 == opt:
-                    if len(args) > i:
-                        p = current.join(args[i+1], abs=True)
-                elif opt1.startswith(opt + "="):
-                    p = current.join(opt1[len(opt)+1:], abs=1)
-                self._confcutdir = p
-                break
-        for arg in args + [current]:
-            if hasattr(arg, 'startswith') and arg.startswith("--"):
-                continue
-            anchor = current.join(arg, abs=1)
-            if anchor.check(): # we found some file object
-                self._path2confmods[None] = self.getconftestmodules(anchor)
-                # let's also consider test* dirs
-                if anchor.check(dir=1):
-                    for x in anchor.listdir("test*"):
-                        if x.check(dir=1):
-                            self.getconftestmodules(x)
-                break
-        else:
-            assert 0, "no root of filesystem?"
-
-    def getconftestmodules(self, path):
-        """ return a list of imported conftest modules for the given path.  """
-        try:
-            clist = self._path2confmods[path]
-        except KeyError:
-            if path is None:
-                raise ValueError("missing default confest.")
-            dp = path.dirpath()
-            clist = []
-            if dp != path:
-                cutdir = self._confcutdir
-                if cutdir and path != cutdir and not path.relto(cutdir):
-                    pass
-                else:
-                    conftestpath = path.join("conftest.py")
-                    if conftestpath.check(file=1):
-                        clist.append(self.importconftest(conftestpath))
-                clist[:0] = self.getconftestmodules(dp)
-            self._path2confmods[path] = clist
-        # be defensive: avoid changes from caller side to
-        # affect us by always returning a copy of the actual list
-        return clist[:]
-
-    def rget(self, name, path=None):
-        mod, value = self.rget_with_confmod(name, path)
-        return value
-
-    def rget_with_confmod(self, name, path=None):
-        modules = self.getconftestmodules(path)
-        modules.reverse()
-        for mod in modules:
-            try:
-                return mod, getattr(mod, name)
-            except AttributeError:
-                continue
-        raise KeyError(name)
-
-    def importconftest(self, conftestpath):
-        assert conftestpath.check(), conftestpath
-        try:
-            return self._conftestpath2mod[conftestpath]
-        except KeyError:
-            pkgpath = conftestpath.pypkgpath()
-            if pkgpath is None:
-                _ensure_removed_sysmodule(conftestpath.purebasename)
-            self._conftestpath2mod[conftestpath] = mod = conftestpath.pyimport()
-            dirpath = conftestpath.dirpath()
-            if dirpath in self._path2confmods:
-                for path, mods in self._path2confmods.items():
-                    if path and path.relto(dirpath) or path == dirpath:
-                        assert mod not in mods
-                        mods.append(mod)
-            self._postimport(mod)
-            return mod
-
-    def _postimport(self, mod):
-        if self._onimport:
-            self._onimport(mod)
-        return mod
-
-def _ensure_removed_sysmodule(modname):
-    try:
-        del sys.modules[modname]
-    except KeyError:
-        pass
-
-class CmdOptions(object):
-    """ holds cmdline options as attributes."""
-    def __init__(self, **kwargs):
-        self.__dict__.update(kwargs)
-    def __repr__(self):
-        return "<CmdOptions %r>" %(self.__dict__,)
-
-class Config(object):
-    """ access to configuration values, pluginmanager and plugin hooks.  """
-    def __init__(self, pluginmanager=None):
-        #: command line option values, usually added via parser.addoption(...)
-        #: or parser.getgroup(...).addoption(...) calls
-        self.option = CmdOptions()
-        self._parser = Parser(
-            usage="usage: %prog [options] [file_or_dir] [file_or_dir] [...]",
-            processopt=self._processopt,
-        )
-        #: a pluginmanager instance
-        self.pluginmanager = pluginmanager or PluginManager(load=True)
-        self.trace = self.pluginmanager.trace.root.get("config")
-        self._conftest = Conftest(onimport=self._onimportconftest)
-        self.hook = self.pluginmanager.hook
-        self._inicache = {}
-        self._cleanup = []
-
-    @classmethod
-    def fromdictargs(cls, option_dict, args):
-        """ constructor useable for subprocesses. """
-        config = cls()
-        # XXX slightly crude way to initialize capturing
-        import _pytest.capture
-        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
-        config._preparse(args, addopts=False)
-        config.option.__dict__.update(option_dict)
-        for x in config.option.plugins:
-            config.pluginmanager.consider_pluginarg(x)
-        return config
-
-    def _onimportconftest(self, conftestmodule):
-        self.trace("loaded conftestmodule %r" %(conftestmodule,))
-        self.pluginmanager.consider_conftest(conftestmodule)
-
-    def _processopt(self, opt):
-        if hasattr(opt, 'default') and opt.dest:
-            if not hasattr(self.option, opt.dest):
-                setattr(self.option, opt.dest, opt.default)
-
-    def _getmatchingplugins(self, fspath):
-        allconftests = self._conftest._conftestpath2mod.values()
-        plugins = [x for x in self.pluginmanager.getplugins()
-                        if x not in allconftests]
-        plugins += self._conftest.getconftestmodules(fspath)
-        return plugins
-
-    def _setinitialconftest(self, args):
-        # capture output during conftest init (#issue93)
-        # XXX introduce load_conftest hook to avoid needing to know
-        # about capturing plugin here
-        capman = self.pluginmanager.getplugin("capturemanager")
-        capman.resumecapture()
-        try:
-            try:
-                self._conftest.setinitial(args)
-            finally:
-                out, err = capman.suspendcapture() # logging might have got it
-        except:
-            sys.stdout.write(out)
-            sys.stderr.write(err)
-            raise
-
-    def _initini(self, args):
-        self.inicfg = getcfg(args, ["pytest.ini", "tox.ini", "setup.cfg"])
-        self._parser.addini('addopts', 'extra command line options', 'args')
-        self._parser.addini('minversion', 'minimally required pytest version')
-
-    def _preparse(self, args, addopts=True):
-        self._initini(args)
-        if addopts:
-            args[:] = self.getini("addopts") + args
-        self._checkversion()
-        self.pluginmanager.consider_preparse(args)
-        self.pluginmanager.consider_setuptools_entrypoints()
-        self.pluginmanager.consider_env()
-        self._setinitialconftest(args)
-        self.pluginmanager.do_addoption(self._parser)
-        if addopts:
-            self.hook.pytest_cmdline_preparse(config=self, args=args)
-
-    def _checkversion(self):
-        minver = self.inicfg.get('minversion', None)
-        if minver:
-            ver = minver.split(".")
-            myver = pytest.__version__.split(".")
-            if myver < ver:
-                raise pytest.UsageError(
-                    "%s:%d: requires pytest-%s, actual pytest-%s'" %(
-                    self.inicfg.config.path, self.inicfg.lineof('minversion'),
-                    minver, pytest.__version__))
-
-    def parse(self, args):
-        # parse given cmdline arguments into this config object.
-        # Note that this can only be called once per testing process.
-        assert not hasattr(self, 'args'), (
-                "can only parse cmdline args at most once per Config object")
-        self._origargs = args
-        self._preparse(args)
-        self._parser.hints.extend(self.pluginmanager._hints)
-        args = self._parser.parse_setoption(args, self.option)
-        if not args:
-            args.append(py.std.os.getcwd())
-        self.args = args
-
-    def addinivalue_line(self, name, line):
-        """ add a line to an ini-file option. The option must have been
-        declared but might not yet be set in which case the line becomes the
-        the first line in its value. """
-        x = self.getini(name)
-        assert isinstance(x, list)
-        x.append(line) # modifies the cached list inline
-
-    def getini(self, name):
-        """ return configuration value from an ini file. If the
-        specified name hasn't been registered through a prior ``parse.addini``
-        call (usually from a plugin), a ValueError is raised. """
-        try:
-            return self._inicache[name]
-        except KeyError:
-            self._inicache[name] = val = self._getini(name)
-            return val
-
-    def _getini(self, name):
-        try:
-            description, type, default = self._parser._inidict[name]
-        except KeyError:
-            raise ValueError("unknown configuration value: %r" %(name,))
-        try:
-            value = self.inicfg[name]
-        except KeyError:
-            if default is not None:
-                return default
-            if type is None:
-                return ''
-            return []
-        if type == "pathlist":
-            dp = py.path.local(self.inicfg.config.path).dirpath()
-            l = []
-            for relpath in py.std.shlex.split(value):
-                l.append(dp.join(relpath, abs=True))
-            return l
-        elif type == "args":
-            return py.std.shlex.split(value)
-        elif type == "linelist":
-            return [t for t in map(lambda x: x.strip(), value.split("\n")) if t]
-        else:
-            assert type is None
-            return value
-
-    def _getconftest_pathlist(self, name, path=None):
-        try:
-            mod, relroots = self._conftest.rget_with_confmod(name, path)
-        except KeyError:
-            return None
-        modpath = py.path.local(mod.__file__).dirpath()
-        l = []
-        for relroot in relroots:
-            if not isinstance(relroot, py.path.local):
-                relroot = relroot.replace("/", py.path.local.sep)
-                relroot = modpath.join(relroot, abs=True)
-            l.append(relroot)
-        return l
-
-    def _getconftest(self, name, path=None, check=False):
-        if check:
-            self._checkconftest(name)
-        return self._conftest.rget(name, path)
-
-    def getvalue(self, name, path=None):
-        """ return ``name`` value looked set from command line options.
-
-        (deprecated) if we can't find the option also lookup
-        the name in a matching conftest file.
-        """
-        try:
-            return getattr(self.option, name)
-        except AttributeError:
-            return self._getconftest(name, path, check=False)
-
-    def getvalueorskip(self, name, path=None):
-        """ (deprecated) return getvalue(name) or call
-        py.test.skip if no value exists. """
-        __tracebackhide__ = True
-        try:
-            val = self.getvalue(name, path)
-            if val is None:
-                raise KeyError(name)
-            return val
-        except KeyError:
-            py.test.skip("no %r value found" %(name,))
-
-
-def getcfg(args, inibasenames):
-    args = [x for x in args if not str(x).startswith("-")]
-    if not args:
-        args = [py.path.local()]
-    for arg in args:
-        arg = py.path.local(arg)
-        for base in arg.parts(reverse=True):
-            for inibasename in inibasenames:
-                p = base.join(inibasename)
-                if p.check():
-                    iniconfig = py.iniconfig.IniConfig(p)
-                    if 'pytest' in iniconfig.sections:
-                        return iniconfig['pytest']
-    return {}
-
-def findupwards(current, basename):
-    current = py.path.local(current)
-    while 1:
-        p = current.join(basename)
-        if p.check():
-            return p
-        p = current.dirpath()
-        if p == current:
-            return
-        current = p
-
diff --git a/_pytest/core.py b/_pytest/core.py
deleted file mode 100644
--- a/_pytest/core.py
+++ /dev/null
@@ -1,477 +0,0 @@
-"""
-pytest PluginManager, basic initialization and tracing.
-(c) Holger Krekel 2004-2010
-"""
-import sys, os
-import inspect
-import py
-from _pytest import hookspec # the extension point definitions
-
-assert py.__version__.split(".")[:2] >= ['1', '4'], ("installation problem: "
-    "%s is too old, remove or upgrade 'py'" % (py.__version__))
-
-default_plugins = (
- "config mark main terminal runner python pdb unittest capture skipping "
- "tmpdir monkeypatch recwarn pastebin helpconfig nose assertion genscript "
- "junitxml resultlog doctest").split()
-
-class TagTracer:
-    def __init__(self):
-        self._tag2proc = {}
-        self.writer = None
-        self.indent = 0
-
-    def get(self, name):
-        return TagTracerSub(self, (name,))
-
-    def processmessage(self, tags, args):
-        if self.writer is not None:
-            if args:
-                indent = "  " * self.indent
-                content = " ".join(map(str, args))
-                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
-        try:
-            self._tag2proc[tags](tags, args)
-        except KeyError:
-            pass
-
-    def setwriter(self, writer):
-        self.writer = writer
-
-    def setprocessor(self, tags, processor):
-        if isinstance(tags, str):
-            tags = tuple(tags.split(":"))
-        else:
-            assert isinstance(tags, tuple)
-        self._tag2proc[tags] = processor
-
-class TagTracerSub:
-    def __init__(self, root, tags):
-        self.root = root
-        self.tags = tags
-    def __call__(self, *args):
-        self.root.processmessage(self.tags, args)
-    def setmyprocessor(self, processor):
-        self.root.setprocessor(self.tags, processor)
-    def get(self, name):
-        return self.__class__(self.root, self.tags + (name,))
-
-class PluginManager(object):
-    def __init__(self, load=False):
-        self._name2plugin = {}
-        self._listattrcache = {}
-        self._plugins = []
-        self._hints = []
-        self.trace = TagTracer().get("pluginmanage")
-        self._plugin_distinfo = []
-        if os.environ.get('PYTEST_DEBUG'):
-            err = sys.stderr
-            encoding = getattr(err, 'encoding', 'utf8')
-            try:
-                err = py.io.dupfile(err, encoding=encoding)
-            except Exception:
-                pass
-            self.trace.root.setwriter(err.write)
-        self.hook = HookRelay([hookspec], pm=self)
-        self.register(self)
-        if load:
-            for spec in default_plugins:
-                self.import_plugin(spec)
-
-    def register(self, plugin, name=None, prepend=False):
-        assert not self.isregistered(plugin), plugin
-        name = name or getattr(plugin, '__name__', str(id(plugin)))
-        if name in self._name2plugin:
-            return False
-        #self.trace("registering", name, plugin)
-        self._name2plugin[name] = plugin
-        self.call_plugin(plugin, "pytest_addhooks", {'pluginmanager': self})
-        self.hook.pytest_plugin_registered(manager=self, plugin=plugin)
-        if not prepend:
-            self._plugins.append(plugin)
-        else:
-            self._plugins.insert(0, plugin)
-        return True
-
-    def unregister(self, plugin=None, name=None):
-        if plugin is None:
-            plugin = self.getplugin(name=name)
-        self._plugins.remove(plugin)
-        self.hook.pytest_plugin_unregistered(plugin=plugin)
-        for name, value in list(self._name2plugin.items()):
-            if value == plugin:
-                del self._name2plugin[name]
-
-    def isregistered(self, plugin, name=None):
-        if self.getplugin(name) is not None:
-            return True
-        for val in self._name2plugin.values():
-            if plugin == val:
-                return True
-
-    def addhooks(self, spec):
-        self.hook._addhooks(spec, prefix="pytest_")
-
-    def getplugins(self):
-        return list(self._plugins)
-
-    def skipifmissing(self, name):
-        if not self.hasplugin(name):
-            py.test.skip("plugin %r is missing" % name)
-
-    def hasplugin(self, name):
-        return bool(self.getplugin(name))
-
-    def getplugin(self, name):
-        if name is None:
-            return None
-        try:
-            return self._name2plugin[name]
-        except KeyError:
-            return self._name2plugin.get("_pytest." + name, None)
-
-    # API for bootstrapping
-    #
-    def _envlist(self, varname):
-        val = py.std.os.environ.get(varname, None)
-        if val is not None:
-            return val.split(',')
-        return ()
-
-    def consider_env(self):
-        for spec in self._envlist("PYTEST_PLUGINS"):
-            self.import_plugin(spec)
-
-    def consider_setuptools_entrypoints(self):
-        try:
-            from pkg_resources import iter_entry_points, DistributionNotFound
-        except ImportError:
-            return # XXX issue a warning
-        for ep in iter_entry_points('pytest11'):
-            name = ep.name
-            if name.startswith("pytest_"):
-                name = name[7:]
-            if ep.name in self._name2plugin or name in self._name2plugin:
-                continue
-            try:
-                plugin = ep.load()
-            except DistributionNotFound:
-                continue
-            self._plugin_distinfo.append((ep.dist, plugin))
-            self.register(plugin, name=name)
-
-    def consider_preparse(self, args):
-        for opt1,opt2 in zip(args, args[1:]):
-            if opt1 == "-p":
-                self.consider_pluginarg(opt2)
-
-    def consider_pluginarg(self, arg):
-        if arg.startswith("no:"):
-            name = arg[3:]
-            if self.getplugin(name) is not None:
-                self.unregister(None, name=name)
-            self._name2plugin[name] = -1
-        else:
-            if self.getplugin(arg) is None:
-                self.import_plugin(arg)
-
-    def consider_conftest(self, conftestmodule):
-        if self.register(conftestmodule, name=conftestmodule.__file__):
-            self.consider_module(conftestmodule)
-
-    def consider_module(self, mod):
-        attr = getattr(mod, "pytest_plugins", ())
-        if attr:
-            if not isinstance(attr, (list, tuple)):
-                attr = (attr,)
-            for spec in attr:
-                self.import_plugin(spec)
-
-    def import_plugin(self, modname):
-        assert isinstance(modname, str)
-        if self.getplugin(modname) is not None:
-            return
-        try:
-            #self.trace("importing", modname)
-            mod = importplugin(modname)
-        except KeyboardInterrupt:
-            raise
-        except ImportError:
-            if modname.startswith("pytest_"):
-                return self.import_plugin(modname[7:])
-            raise
-        except:
-            e = py.std.sys.exc_info()[1]
-            if not hasattr(py.test, 'skip'):
-                raise
-            elif not isinstance(e, py.test.skip.Exception):
-                raise
-            self._hints.append("skipped plugin %r: %s" %((modname, e.msg)))
-        else:
-            self.register(mod, modname)
-            self.consider_module(mod)
-
-    def pytest_configure(self, config):
-        config.addinivalue_line("markers",
-            "tryfirst: mark a hook implementation function such that the "
-            "plugin machinery will try to call it first/as early as possible.")
-        config.addinivalue_line("markers",
-            "trylast: mark a hook implementation function such that the "
-            "plugin machinery will try to call it last/as late as possible.")
-
-    def pytest_plugin_registered(self, plugin):
-        import pytest
-        dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
-        if dic:
-            self._setns(pytest, dic)
-        if hasattr(self, '_config'):
-            self.call_plugin(plugin, "pytest_addoption",
-                {'parser': self._config._parser})
-            self.call_plugin(plugin, "pytest_configure",
-                {'config': self._config})
-
-    def _setns(self, obj, dic):
-        import pytest
-        for name, value in dic.items():
-            if isinstance(value, dict):
-                mod = getattr(obj, name, None)
-                if mod is None:
-                    modname = "pytest.%s" % name
-                    mod = py.std.types.ModuleType(modname)
-                    sys.modules[modname] = mod
-                    mod.__all__ = []
-                    setattr(obj, name, mod)
-                obj.__all__.append(name)
-                self._setns(mod, value)
-            else:
-                setattr(obj, name, value)
-                obj.__all__.append(name)
-                #if obj != pytest:
-                #    pytest.__all__.append(name)
-                setattr(pytest, name, value)
-
-    def pytest_terminal_summary(self, terminalreporter):
-        tw = terminalreporter._tw
-        if terminalreporter.config.option.traceconfig:
-            for hint in self._hints:
-                tw.line("hint: %s" % hint)
-
-    def do_addoption(self, parser):
-        mname = "pytest_addoption"
-        methods = reversed(self.listattr(mname))
-        MultiCall(methods, {'parser': parser}).execute()
-
-    def do_configure(self, config):
-        assert not hasattr(self, '_config')
-        self._config = config
-        config.hook.pytest_configure(config=self._config)
-
-    def do_unconfigure(self, config):
-        config = self._config
-        del self._config
-        config.hook.pytest_unconfigure(config=config)
-        config.pluginmanager.unregister(self)
-
-    def notify_exception(self, excinfo, option=None):
-        if option and option.fulltrace:
-            style = "long"
-        else:
-            style = "native"
-        excrepr = excinfo.getrepr(funcargs=True,
-            showlocals=getattr(option, 'showlocals', False),
-            style=style,
-        )
-        res = self.hook.pytest_internalerror(excrepr=excrepr)
-        if not py.builtin.any(res):
-            for line in str(excrepr).split("\n"):
-                sys.stderr.write("INTERNALERROR> %s\n" %line)
-                sys.stderr.flush()
-
-    def listattr(self, attrname, plugins=None):
-        if plugins is None:
-            plugins = self._plugins
-        key = (attrname,) + tuple(plugins)
-        try:
-            return list(self._listattrcache[key])
-        except KeyError:
-            pass
-        l = []
-        last = []
-        for plugin in plugins:
-            try:
-                meth = getattr(plugin, attrname)
-                if hasattr(meth, 'tryfirst'):
-                    last.append(meth)
-                elif hasattr(meth, 'trylast'):
-                    l.insert(0, meth)
-                else:
-                    l.append(meth)
-            except AttributeError:
-                continue
-        l.extend(last)
-        self._listattrcache[key] = list(l)
-        return l
-
-    def call_plugin(self, plugin, methname, kwargs):
-        return MultiCall(methods=self.listattr(methname, plugins=[plugin]),
-                kwargs=kwargs, firstresult=True).execute()
-
-
-def importplugin(importspec):
-    name = importspec
-    try:
-        mod = "_pytest." + name
-        return __import__(mod, None, None, '__doc__')
-    except ImportError:
-        #e = py.std.sys.exc_info()[1]
-        #if str(e).find(name) == -1:
-        #    raise
-        pass #
-    return __import__(importspec, None, None, '__doc__')
-
-class MultiCall:
-    """ execute a call into multiple python functions/methods. """
-    def __init__(self, methods, kwargs, firstresult=False):
-        self.methods = list(methods)
-        self.kwargs = kwargs
-        self.results = []
-        self.firstresult = firstresult
-
-    def __repr__(self):
-        status = "%d results, %d meths" % (len(self.results), len(self.methods))
-        return "<MultiCall %s, kwargs=%r>" %(status, self.kwargs)
-
-    def execute(self):
-        while self.methods:
-            method = self.methods.pop()
-            kwargs = self.getkwargs(method)
-            res = method(**kwargs)
-            if res is not None:
-                self.results.append(res)
-                if self.firstresult:
-                    return res
-        if not self.firstresult:
-            return self.results
-
-    def getkwargs(self, method):
-        kwargs = {}
-        for argname in varnames(method):
-            try:
-                kwargs[argname] = self.kwargs[argname]
-            except KeyError:
-                if argname == "__multicall__":
-                    kwargs[argname] = self
-        return kwargs
-
-def varnames(func):
-    try:
-        return func._varnames
-    except AttributeError:
-        pass
-    if not inspect.isfunction(func) and not inspect.ismethod(func):
-        func = getattr(func, '__call__', func)
-    ismethod = inspect.ismethod(func)
-    rawcode = py.code.getrawcode(func)
-    try:
-        x = rawcode.co_varnames[ismethod:rawcode.co_argcount]
-    except AttributeError:
-        x = ()
-    py.builtin._getfuncdict(func)['_varnames'] = x
-    return x
-
-class HookRelay:
-    def __init__(self, hookspecs, pm, prefix="pytest_"):
-        if not isinstance(hookspecs, list):
-            hookspecs = [hookspecs]
-        self._hookspecs = []
-        self._pm = pm
-        self.trace = pm.trace.root.get("hook")
-        for hookspec in hookspecs:
-            self._addhooks(hookspec, prefix)
-
-    def _addhooks(self, hookspecs, prefix):
-        self._hookspecs.append(hookspecs)
-        added = False
-        for name, method in vars(hookspecs).items():
-            if name.startswith(prefix):
-                firstresult = getattr(method, 'firstresult', False)
-                hc = HookCaller(self, name, firstresult=firstresult)
-                setattr(self, name, hc)
-                added = True
-                #print ("setting new hook", name)
-        if not added:
-            raise ValueError("did not find new %r hooks in %r" %(
-                prefix, hookspecs,))
-
-
-class HookCaller:
-    def __init__(self, hookrelay, name, firstresult):
-        self.hookrelay = hookrelay
-        self.name = name
-        self.firstresult = firstresult
-        self.trace = self.hookrelay.trace
-
-    def __repr__(self):
-        return "<HookCaller %r>" %(self.name,)
-
-    def __call__(self, **kwargs):
-        methods = self.hookrelay._pm.listattr(self.name)
-        return self._docall(methods, kwargs)
-
-    def pcall(self, plugins, **kwargs):
-        methods = self.hookrelay._pm.listattr(self.name, plugins=plugins)
-        return self._docall(methods, kwargs)
-
-    def _docall(self, methods, kwargs):
-        self.trace(self.name, kwargs)
-        self.trace.root.indent += 1
-        mc = MultiCall(methods, kwargs, firstresult=self.firstresult)
-        try:
-            res = mc.execute()
-            if res:
-                self.trace("finish", self.name, "-->", res)
-        finally:
-            self.trace.root.indent -= 1
-        return res
-
-_preinit = []
-
-def _preloadplugins():
-    _preinit.append(PluginManager(load=True))
-
-def _prepareconfig(args=None, plugins=None):
-    if args is None:
-        args = sys.argv[1:]
-    elif isinstance(args, py.path.local):
-        args = [str(args)]
-    elif not isinstance(args, (tuple, list)):
-        if not isinstance(args, str):
-            raise ValueError("not a string or argument list: %r" % (args,))
-        args = py.std.shlex.split(args)
-    if _preinit:
-       _pluginmanager = _preinit.pop(0)
-    else: # subsequent calls to main will create a fresh instance
-        _pluginmanager = PluginManager(load=True)
-    hook = _pluginmanager.hook
-    if plugins:
-        for plugin in plugins:
-            _pluginmanager.register(plugin)
-    return hook.pytest_cmdline_parse(
-            pluginmanager=_pluginmanager, args=args)
-
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
-    try:
-        config = _prepareconfig(args, plugins)
-        exitstatus = config.hook.pytest_cmdline_main(config=config)
-    except UsageError:
-        e = sys.exc_info()[1]
-        sys.stderr.write("ERROR: %s\n" %(e.args[0],))
-        exitstatus = 3
-    return exitstatus
-
-class UsageError(Exception):
-    """ error in py.test usage or invocation"""
-
diff --git a/_pytest/doctest.py b/_pytest/doctest.py
deleted file mode 100644
--- a/_pytest/doctest.py
+++ /dev/null
@@ -1,87 +0,0 @@
-""" discover and run doctests in modules and test files."""
-
-import pytest, py
-from py._code.code import TerminalRepr, ReprFileLocation
-
-def pytest_addoption(parser):
-    group = parser.getgroup("collect")
-    group.addoption("--doctest-modules",
-        action="store_true", default=False,
-        help="run doctests in all .py modules",
-        dest="doctestmodules")
-    group.addoption("--doctest-glob",
-        action="store", default="test*.txt", metavar="pat",
-        help="doctests file matching pattern, default: test*.txt",
-        dest="doctestglob")
-
-def pytest_collect_file(path, parent):
-    config = parent.config
-    if path.ext == ".py":
-        if config.option.doctestmodules:
-            return DoctestModule(path, parent)
-    elif (path.ext in ('.txt', '.rst') and parent.session.isinitpath(path)) or \
-        path.check(fnmatch=config.getvalue("doctestglob")):
-        return DoctestTextfile(path, parent)
-
-class ReprFailDoctest(TerminalRepr):
-    def __init__(self, reprlocation, lines):
-        self.reprlocation = reprlocation
-        self.lines = lines
-    def toterminal(self, tw):
-        for line in self.lines:
-            tw.line(line)
-        self.reprlocation.toterminal(tw)
-
-class DoctestItem(pytest.Item):
-    def repr_failure(self, excinfo):
-        doctest = py.std.doctest
-        if excinfo.errisinstance((doctest.DocTestFailure,
-                                  doctest.UnexpectedException)):
-            doctestfailure = excinfo.value
-            example = doctestfailure.example
-            test = doctestfailure.test
-            filename = test.filename
-            lineno = test.lineno + example.lineno + 1
-            message = excinfo.type.__name__
-            reprlocation = ReprFileLocation(filename, lineno, message)
-            checker = py.std.doctest.OutputChecker()
-            REPORT_UDIFF = py.std.doctest.REPORT_UDIFF
-            filelines = py.path.local(filename).readlines(cr=0)
-            i = max(test.lineno, max(0, lineno - 10)) # XXX?
-            lines = []
-            for line in filelines[i:lineno]:
-                lines.append("%03d %s" % (i+1, line))
-                i += 1
-            if excinfo.errisinstance(doctest.DocTestFailure):
-                lines += checker.output_difference(example,
-                        doctestfailure.got, REPORT_UDIFF).split("\n")
-            else:
-                inner_excinfo = py.code.ExceptionInfo(excinfo.value.exc_info)
-                lines += ["UNEXPECTED EXCEPTION: %s" %
-                            repr(inner_excinfo.value)]
-                lines += py.std.traceback.format_exception(*excinfo.value.exc_info)
-            return ReprFailDoctest(reprlocation, lines)
-        else:
-            return super(DoctestItem, self).repr_failure(excinfo)
-
-    def reportinfo(self):
-        return self.fspath, None, "[doctest]"
-
-class DoctestTextfile(DoctestItem, pytest.File):
-    def runtest(self):
-        doctest = py.std.doctest
-        failed, tot = doctest.testfile(
-            str(self.fspath), module_relative=False,
-            optionflags=doctest.ELLIPSIS,
-            raise_on_error=True, verbose=0)
-
-class DoctestModule(DoctestItem, pytest.File):
-    def runtest(self):
-        doctest = py.std.doctest
-        if self.fspath.basename == "conftest.py":
-            module = self.config._conftest.importconftest(self.fspath)
-        else:
-            module = self.fspath.pyimport()
-        failed, tot = doctest.testmod(
-            module, raise_on_error=True, verbose=0,
-            optionflags=doctest.ELLIPSIS)
diff --git a/_pytest/genscript.py b/_pytest/genscript.py
deleted file mode 100755
--- a/_pytest/genscript.py
+++ /dev/null
@@ -1,69 +0,0 @@
-""" generate a single-file self-contained version of py.test """
-import py
-
-def find_toplevel(name):
-    for syspath in py.std.sys.path:
-        base = py.path.local(syspath)
-        lib = base/name
-        if lib.check(dir=1):
-            return lib
-        mod = base.join("%s.py" % name)
-        if mod.check(file=1):
-            return mod
-    raise LookupError(name)
-
-def pkgname(toplevel, rootpath, path):
-    parts = path.parts()[len(rootpath.parts()):]
-    return '.'.join([toplevel] + [x.purebasename for x in parts])
-
-def pkg_to_mapping(name):
-    toplevel = find_toplevel(name)
-    name2src = {}
-    if toplevel.check(file=1): # module
-        name2src[toplevel.purebasename] = toplevel.read()
-    else: # package
-        for pyfile in toplevel.visit('*.py'):
-            pkg = pkgname(name, toplevel, pyfile)
-            name2src[pkg] = pyfile.read()
-    return name2src
-
-def compress_mapping(mapping):
-    data = py.std.pickle.dumps(mapping, 2)
-    data = py.std.zlib.compress(data, 9)
-    data = py.std.base64.encodestring(data)
-    data = data.decode('ascii')
-    return data
-
-
-def compress_packages(names):
-    mapping = {}
-    for name in names:
-        mapping.update(pkg_to_mapping(name))
-    return compress_mapping(mapping)
-
-def generate_script(entry, packages):
-    data = compress_packages(packages)
-    tmpl = py.path.local(__file__).dirpath().join('standalonetemplate.py')
-    exe = tmpl.read()
-    exe = exe.replace('@SOURCES@', data)
-    exe = exe.replace('@ENTRY@', entry)
-    return exe
-
-
-def pytest_addoption(parser):
-    group = parser.getgroup("debugconfig")
-    group.addoption("--genscript", action="store", default=None,
-        dest="genscript", metavar="path",
-        help="create standalone py.test script at given target path.")
-
-def pytest_cmdline_main(config):
-    genscript = config.getvalue("genscript")
-    if genscript:
-        script = generate_script(
-            'import py; raise SystemExit(py.test.cmdline.main())',
-            ['py', '_pytest', 'pytest'],
-        )
-
-        genscript = py.path.local(genscript)
-        genscript.write(script)
-        return 0
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
deleted file mode 100644
--- a/_pytest/helpconfig.py
+++ /dev/null
@@ -1,197 +0,0 @@
-""" version info, help messages, tracing configuration.  """
-import py
-import pytest
-import os, inspect, sys
-from _pytest.core import varnames
-
-def pytest_addoption(parser):
-    group = parser.getgroup('debugconfig')
-    group.addoption('--version', action="store_true",
-            help="display pytest lib version and import information.")
-    group._addoption("-h", "--help", action="store_true", dest="help",
-            help="show help message and configuration info")
-    group._addoption('-p', action="append", dest="plugins", default = [],
-               metavar="name",
-               help="early-load given plugin (multi-allowed).")
-    group.addoption('--traceconfig',
-               action="store_true", dest="traceconfig", default=False,
-               help="trace considerations of conftest.py files."),
-    group.addoption('--debug',
-               action="store_true", dest="debug", default=False,
-               help="store internal tracing debug information in 'pytestdebug.log'.")
-
-
-def pytest_cmdline_parse(__multicall__):
-    config = __multicall__.execute()
-    if config.option.debug:
-        path = os.path.abspath("pytestdebug.log")
-        f = open(path, 'w')
-        config._debugfile = f
-        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
-            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
-            os.getcwd(), config._origargs))
-        config.trace.root.setwriter(f.write)
-        sys.stderr.write("writing pytestdebug information to %s\n" % path)
-    return config
-
- at pytest.mark.trylast
-def pytest_unconfigure(config):
-    if hasattr(config, '_debugfile'):
-        config._debugfile.close()
-        sys.stderr.write("wrote pytestdebug information to %s\n" %
-            config._debugfile.name)
-        config.trace.root.setwriter(None)
-
-
-def pytest_cmdline_main(config):
-    if config.option.version:
-        p = py.path.local(pytest.__file__)
-        sys.stderr.write("This is py.test version %s, imported from %s\n" %
-            (pytest.__version__, p))
-        plugininfo = getpluginversioninfo(config)
-        if plugininfo:
-            for line in plugininfo:
-                sys.stderr.write(line + "\n")
-        return 0
-    elif config.option.help:
-        config.pluginmanager.do_configure(config)
-        showhelp(config)
-        config.pluginmanager.do_unconfigure(config)
-        return 0
-
-def showhelp(config):
-    tw = py.io.TerminalWriter()
-    tw.write(config._parser.optparser.format_help())
-    tw.line()
-    tw.line()
-    #tw.sep( "=", "config file settings")
-    tw.line("[pytest] ini-options in the next "
-            "pytest.ini|tox.ini|setup.cfg file:")
-    tw.line()
-
-    for name in config._parser._ininames:
-        help, type, default = config._parser._inidict[name]
-        if type is None:
-            type = "string"
-        spec = "%s (%s)" % (name, type)
-        line = "  %-24s %s" %(spec, help)
-        tw.line(line[:tw.fullwidth])
-
-    tw.line() ; tw.line()
-    #tw.sep("=")
-    return
-
-    tw.line("conftest.py options:")
-    tw.line()
-    conftestitems = sorted(config._parser._conftestdict.items())
-    for name, help in conftest_options + conftestitems:
-        line = "   %-15s  %s" %(name, help)
-        tw.line(line[:tw.fullwidth])
-    tw.line()
-    #tw.sep( "=")
-
-conftest_options = [
-    ('pytest_plugins', 'list of plugin names to load'),
-]
-
-def getpluginversioninfo(config):
-    lines = []
-    plugininfo = config.pluginmanager._plugin_distinfo
-    if plugininfo:
-        lines.append("setuptools registered plugins:")
-        for dist, plugin in plugininfo:
-            loc = getattr(plugin, '__file__', repr(plugin))
-            content = "%s-%s at %s" % (dist.project_name, dist.version, loc)
-            lines.append("  " + content)
-    return lines
-
-def pytest_report_header(config):
-    lines = []
-    if config.option.debug or config.option.traceconfig:
-        lines.append("using: pytest-%s pylib-%s" %
-            (pytest.__version__,py.__version__))
-
-        verinfo = getpluginversioninfo(config)
-        if verinfo:
-            lines.extend(verinfo)
-
-    if config.option.traceconfig:
-        lines.append("active plugins:")
-        plugins = []
-        items = config.pluginmanager._name2plugin.items()
-        for name, plugin in items:
-            if hasattr(plugin, '__file__'):
-                r = plugin.__file__
-            else:
-                r = repr(plugin)
-            lines.append("    %-20s: %s" %(name, r))
-    return lines
-
-
-# =====================================================
-# validate plugin syntax and hooks
-# =====================================================
-
-def pytest_plugin_registered(manager, plugin):
-    methods = collectattr(plugin)
-    hooks = {}
-    for hookspec in manager.hook._hookspecs:
-        hooks.update(collectattr(hookspec))
-
-    stringio = py.io.TextIO()
-    def Print(*args):
-        if args:
-            stringio.write(" ".join(map(str, args)))
-        stringio.write("\n")
-
-    fail = False
-    while methods:
-        name, method = methods.popitem()
-        #print "checking", name
-        if isgenerichook(name):
-            continue
-        if name not in hooks:
-            if not getattr(method, 'optionalhook', False):
-                Print("found unknown hook:", name)
-                fail = True
-        else:
-            #print "checking", method
-            method_args = list(varnames(method))
-            if '__multicall__' in method_args:
-                method_args.remove('__multicall__')
-            hook = hooks[name]
-            hookargs = varnames(hook)
-            for arg in method_args:
-                if arg not in hookargs:
-                    Print("argument %r not available"  %(arg, ))
-                    Print("actual definition: %s" %(formatdef(method)))
-                    Print("available hook arguments: %s" %
-                            ", ".join(hookargs))
-                    fail = True
-                    break
-            #if not fail:
-            #    print "matching hook:", formatdef(method)
-        if fail:
-            name = getattr(plugin, '__name__', plugin)
-            raise PluginValidationError("%s:\n%s" % (name, stringio.getvalue()))
-
-class PluginValidationError(Exception):
-    """ plugin failed validation. """
-
-def isgenerichook(name):
-    return name == "pytest_plugins" or \
-           name.startswith("pytest_funcarg__")
-
-def collectattr(obj):
-    methods = {}
-    for apiname in dir(obj):
-        if apiname.startswith("pytest_"):
-            methods[apiname] = getattr(obj, apiname)
-    return methods
-
-def formatdef(func):
-    return "%s%s" % (
-        func.__name__,
-        inspect.formatargspec(*inspect.getargspec(func))
-    )
-
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
deleted file mode 100644
--- a/_pytest/hookspec.py
+++ /dev/null
@@ -1,228 +0,0 @@
-""" hook specifications for pytest plugins, invoked from main.py and builtin plugins.  """
-
-# -------------------------------------------------------------------------
-# Initialization
-# -------------------------------------------------------------------------
-
-def pytest_addhooks(pluginmanager):
-    """called at plugin load time to allow adding new hooks via a call to
-    pluginmanager.registerhooks(module)."""
-
-
-def pytest_namespace():
-    """return dict of name->object to be made globally available in
-    the py.test/pytest namespace.  This hook is called before command
-    line options are parsed.
-    """
-
-def pytest_cmdline_parse(pluginmanager, args):
-    """return initialized config object, parsing the specified args. """
-pytest_cmdline_parse.firstresult = True
-
-def pytest_cmdline_preparse(config, args):
-    """modify command line arguments before option parsing. """
-
-def pytest_addoption(parser):
-    """add optparse-style options and ini-style config values via calls
-    to ``parser.addoption`` and ``parser.addini(...)``.
-    """
-
-def pytest_cmdline_main(config):
-    """ called for performing the main command line action. The default
-    implementation will invoke the configure hooks and runtest_mainloop. """
-pytest_cmdline_main.firstresult = True
-
-def pytest_configure(config):
-    """ called after command line options have been parsed.
-        and all plugins and initial conftest files been loaded.
-    """
-
-def pytest_unconfigure(config):
-    """ called before test process is exited.  """
-
-def pytest_runtestloop(session):
-    """ called for performing the main runtest loop
-    (after collection finished). """
-pytest_runtestloop.firstresult = True
-
-# -------------------------------------------------------------------------
-# collection hooks
-# -------------------------------------------------------------------------
-
-def pytest_collection(session):
-    """ perform the collection protocol for the given session. """
-pytest_collection.firstresult = True
-
-def pytest_collection_modifyitems(session, config, items):
-    """ called after collection has been performed, may filter or re-order
-    the items in-place."""
-
-def pytest_collection_finish(session):
-    """ called after collection has been performed and modified. """
-
-def pytest_ignore_collect(path, config):
-    """ return True to prevent considering this path for collection.
-    This hook is consulted for all files and directories prior to calling
-    more specific hooks.
-    """
-pytest_ignore_collect.firstresult = True
-
-def pytest_collect_directory(path, parent):
-    """ called before traversing a directory for collection files. """
-pytest_collect_directory.firstresult = True
-
-def pytest_collect_file(path, parent):
-    """ return collection Node or None for the given path. Any new node
-    needs to have the specified ``parent`` as a parent."""
-
-# logging hooks for collection
-def pytest_collectstart(collector):
-    """ collector starts collecting. """
-
-def pytest_itemcollected(item):
-    """ we just collected a test item. """
-
-def pytest_collectreport(report):
-    """ collector finished collecting. """
-
-def pytest_deselected(items):
-    """ called for test items deselected by keyword. """
-
-def pytest_make_collect_report(collector):
-    """ perform ``collector.collect()`` and return a CollectReport. """
-pytest_make_collect_report.firstresult = True
-
-# -------------------------------------------------------------------------
-# Python test function related hooks
-# -------------------------------------------------------------------------
-
-def pytest_pycollect_makemodule(path, parent):
-    """ return a Module collector or None for the given path.
-    This hook will be called for each matching test module path.
-    The pytest_collect_file hook needs to be used if you want to
-    create test modules for files that do not match as a test module.
-    """
-pytest_pycollect_makemodule.firstresult = True
-
-def pytest_pycollect_makeitem(collector, name, obj):
-    """ return custom item/collector for a python object in a module, or None.  """
-pytest_pycollect_makeitem.firstresult = True
-
-def pytest_pyfunc_call(pyfuncitem):
-    """ call underlying test function. """
-pytest_pyfunc_call.firstresult = True
-
-def pytest_generate_tests(metafunc):
-    """ generate (multiple) parametrized calls to a test function."""
-
-# -------------------------------------------------------------------------
-# generic runtest related hooks
-# -------------------------------------------------------------------------
-def pytest_itemstart(item, node=None):
-    """ (deprecated, use pytest_runtest_logstart). """
-
-def pytest_runtest_protocol(item, nextitem):
-    """ implements the runtest_setup/call/teardown protocol for
-    the given test item, including capturing exceptions and calling
-    reporting hooks.
-
-    :arg item: test item for which the runtest protocol is performed.
-
-    :arg nexitem: the scheduled-to-be-next test item (or None if this
-                  is the end my friend).  This argument is passed on to
-                  :py:func:`pytest_runtest_teardown`.
-
-    :return boolean: True if no further hook implementations should be invoked.
-    """
-pytest_runtest_protocol.firstresult = True
-
-def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of running a single test item. """
-
-def pytest_runtest_setup(item):
-    """ called before ``pytest_runtest_call(item)``. """
-
-def pytest_runtest_call(item):
-    """ called to execute the test ``item``. """
-
-def pytest_runtest_teardown(item, nextitem):
-    """ called after ``pytest_runtest_call``.
-
-    :arg nexitem: the scheduled-to-be-next test item (None if no further
-                  test item is scheduled).  This argument can be used to
-                  perform exact teardowns, i.e. calling just enough finalizers
-                  so that nextitem only needs to call setup-functions.
-    """
-
-def pytest_runtest_makereport(item, call):
-    """ return a :py:class:`_pytest.runner.TestReport` object
-    for the given :py:class:`pytest.Item` and
-    :py:class:`_pytest.runner.CallInfo`.
-    """
-pytest_runtest_makereport.firstresult = True
-
-def pytest_runtest_logreport(report):
-    """ process a test setup/call/teardown report relating to
-    the respective phase of executing a test. """
-
-# -------------------------------------------------------------------------
-# test session related hooks
-# -------------------------------------------------------------------------
-
-def pytest_sessionstart(session):
-    """ before session.main() is called. """
-
-def pytest_sessionfinish(session, exitstatus):
-    """ whole test run finishes. """
-
-
-# -------------------------------------------------------------------------
-# hooks for customising the assert methods
-# -------------------------------------------------------------------------
-
-def pytest_assertrepr_compare(config, op, left, right):
-    """return explanation for comparisons in failing assert expressions.
-
-    Return None for no custom explanation, otherwise return a list
-    of strings.  The strings will be joined by newlines but any newlines
-    *in* a string will be escaped.  Note that all but the first line will
-    be indented sligthly, the intention is for the first line to be a summary.
-    """
-
-# -------------------------------------------------------------------------
-# hooks for influencing reporting (invoked from _pytest_terminal)
-# -------------------------------------------------------------------------
-
-def pytest_report_header(config):
-    """ return a string to be displayed as header info for terminal reporting."""
-
-def pytest_report_teststatus(report):
-    """ return result-category, shortletter and verbose word for reporting."""
-pytest_report_teststatus.firstresult = True
-
-def pytest_terminal_summary(terminalreporter):
-    """ add additional section in terminal summary reporting. """
-
-# -------------------------------------------------------------------------
-# doctest hooks
-# -------------------------------------------------------------------------
-
-def pytest_doctest_prepare_content(content):
-    """ return processed content for a given doctest"""
-pytest_doctest_prepare_content.firstresult = True
-
-# -------------------------------------------------------------------------
-# error handling and internal debugging hooks
-# -------------------------------------------------------------------------
-
-def pytest_plugin_registered(plugin, manager):
-    """ a new py lib plugin got registered. """
-
-def pytest_plugin_unregistered(plugin):
-    """ a py lib plugin got unregistered. """
-
-def pytest_internalerror(excrepr):
-    """ called for internal errors. """
-
-def pytest_keyboard_interrupt(excinfo):
-    """ called for keyboard interrupt. """
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
deleted file mode 100644
--- a/_pytest/junitxml.py
+++ /dev/null
@@ -1,220 +0,0 @@
-""" report test results in JUnit-XML format, for use with Hudson and build integration servers.
-
-Based on initial code from Ross Lawley.
-"""
-
-import py
-import os
-import re
-import sys
-import time
-
-
-# Python 2.X and 3.X compatibility
-try:
-    unichr(65)
-except NameError:
-    unichr = chr
-try:
-    unicode('A')
-except NameError:
-    unicode = str
-try:
-    long(1)
-except NameError:
-    long = int
-
-
-class Junit(py.xml.Namespace):
-    pass
-
-
-# We need to get the subset of the invalid unicode ranges according to
-# XML 1.0 which are valid in this python build.  Hence we calculate
-# this dynamically instead of hardcoding it.  The spec range of valid
-# chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
-#                    | [#x10000-#x10FFFF]
-_legal_chars = (0x09, 0x0A, 0x0d)
-_legal_ranges = (
-    (0x20, 0xD7FF),
-    (0xE000, 0xFFFD),
-    (0x10000, 0x10FFFF),
-)
-_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _legal_ranges
-                  if low < sys.maxunicode]
-_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
-illegal_xml_re = re.compile(unicode('[^%s]') %
-                            unicode('').join(_legal_xml_re))
-del _legal_chars
-del _legal_ranges
-del _legal_xml_re
-
-def bin_xml_escape(arg):
-    def repl(matchobj):
-        i = ord(matchobj.group())
-        if i <= 0xFF:
-            return unicode('#x%02X') % i
-        else:
-            return unicode('#x%04X') % i
-    return illegal_xml_re.sub(repl, py.xml.escape(arg))
-
-def pytest_addoption(parser):
-    group = parser.getgroup("terminal reporting")
-    group.addoption('--junitxml', action="store", dest="xmlpath",
-           metavar="path", default=None,
-           help="create junit-xml style report file at given path.")
-    group.addoption('--junitprefix', action="store", dest="junitprefix",
-           metavar="str", default=None,
-           help="prepend prefix to classnames in junit-xml output")
-
-def pytest_configure(config):
-    xmlpath = config.option.xmlpath
-    if xmlpath:
-        config._xml = LogXML(xmlpath, config.option.junitprefix)
-        config.pluginmanager.register(config._xml)
-
-def pytest_unconfigure(config):
-    xml = getattr(config, '_xml', None)
-    if xml:
-        del config._xml
-        config.pluginmanager.unregister(xml)
-
-
-class LogXML(object):
-    def __init__(self, logfile, prefix):
-        logfile = os.path.expanduser(os.path.expandvars(logfile))
-        self.logfile = os.path.normpath(logfile)
-        self.prefix = prefix
-        self.tests = []
-        self.passed = self.skipped = 0
-        self.failed = self.errors = 0
-
-    def _opentestcase(self, report):
-        names = report.nodeid.split("::")
-        names[0] = names[0].replace("/", '.')
-        names = [x.replace(".py", "") for x in names if x != "()"]
-        classnames = names[:-1]
-        if self.prefix:
-            classnames.insert(0, self.prefix)
-        self.tests.append(Junit.testcase(
-            classname=".".join(classnames),
-            name=names[-1],
-            time=getattr(report, 'duration', 0)
-        ))
-
-    def append(self, obj):
-        self.tests[-1].append(obj)
-
-    def append_pass(self, report):
-        self.passed += 1
-
-    def append_failure(self, report):
-        #msg = str(report.longrepr.reprtraceback.extraline)
-        if "xfail" in report.keywords:
-            self.append(
-                Junit.skipped(message="xfail-marked test passes unexpectedly"))
-            self.skipped += 1
-        else:
-            sec = dict(report.sections)
-            fail = Junit.failure(message="test failure")
-            fail.append(str(report.longrepr))
-            self.append(fail)
-            for name in ('out', 'err'):
-                content = sec.get("Captured std%s" % name)
-                if content:
-                    tag = getattr(Junit, 'system-'+name)
-                    self.append(tag(bin_xml_escape(content)))
-            self.failed += 1
-
-    def append_collect_failure(self, report):
-        #msg = str(report.longrepr.reprtraceback.extraline)
-        self.append(Junit.failure(str(report.longrepr),
-                                  message="collection failure"))
-        self.errors += 1
-
-    def append_collect_skipped(self, report):
-        #msg = str(report.longrepr.reprtraceback.extraline)
-        self.append(Junit.skipped(str(report.longrepr),
-                                  message="collection skipped"))
-        self.skipped += 1
-
-    def append_error(self, report):
-        self.append(Junit.error(str(report.longrepr),
-                                message="test setup failure"))
-        self.errors += 1
-
-    def append_skipped(self, report):
-        if "xfail" in report.keywords:
-            self.append(Junit.skipped(str(report.keywords['xfail']),
-                                      message="expected test failure"))
-        else:
-            filename, lineno, skipreason = report.longrepr
-            if skipreason.startswith("Skipped: "):
-                skipreason = skipreason[9:]
-            self.append(
-                Junit.skipped("%s:%s: %s" % report.longrepr,
-                              type="pytest.skip",
-                              message=skipreason
-                ))
-        self.skipped += 1
-
-    def pytest_runtest_logreport(self, report):
-        if report.passed:
-            if report.when == "call": # ignore setup/teardown
-                self._opentestcase(report)
-                self.append_pass(report)
-        elif report.failed:
-            self._opentestcase(report)
-            if report.when != "call":
-                self.append_error(report)
-            else:
-                self.append_failure(report)
-        elif report.skipped:
-            self._opentestcase(report)
-            self.append_skipped(report)
-
-    def pytest_collectreport(self, report):
-        if not report.passed:
-            self._opentestcase(report)
-            if report.failed:
-                self.append_collect_failure(report)
-            else:
-                self.append_collect_skipped(report)
-
-    def pytest_internalerror(self, excrepr):
-        self.errors += 1
-        data = py.xml.escape(excrepr)
-        self.tests.append(
-            Junit.testcase(
-                    Junit.error(data, message="internal error"),
-                    classname="pytest",
-                    name="internal"))
-
-    def pytest_sessionstart(self, session):
-        self.suite_start_time = time.time()
-
-    def pytest_sessionfinish(self, session, exitstatus, __multicall__):
-        if py.std.sys.version_info[0] < 3:
-            logfile = py.std.codecs.open(self.logfile, 'w', encoding='utf-8')
-        else:
-            logfile = open(self.logfile, 'w', encoding='utf-8')
-
-        suite_stop_time = time.time()
-        suite_time_delta = suite_stop_time - self.suite_start_time
-        numtests = self.passed + self.failed
-
-        logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write(Junit.testsuite(
-            self.tests,
-            name="",
-            errors=self.errors,
-            failures=self.failed,
-            skips=self.skipped,
-            tests=numtests,
-            time="%.3f" % suite_time_delta,
-        ).unicode(indent=0))
-        logfile.close()
-
-    def pytest_terminal_summary(self, terminalreporter):
-        terminalreporter.write_sep("-", "generated xml file: %s" % (self.logfile))
diff --git a/_pytest/main.py b/_pytest/main.py
deleted file mode 100644
--- a/_pytest/main.py
+++ /dev/null
@@ -1,576 +0,0 @@
-""" core implementation of testing process: init, session, runtest loop. """
-
-import py
-import pytest, _pytest
-import os, sys, imp
-tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
-
-# exitcodes for the command line
-EXIT_OK = 0
-EXIT_TESTSFAILED = 1
-EXIT_INTERRUPTED = 2
-EXIT_INTERNALERROR = 3
-
-name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
-
-def pytest_addoption(parser):
-    parser.addini("norecursedirs", "directory patterns to avoid for recursion",
-        type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
-    #parser.addini("dirpatterns",
-    #    "patterns specifying possible locations of test files",
-    #    type="linelist", default=["**/test_*.txt",
-    #            "**/test_*.py", "**/*_test.py"]
-    #)
-    group = parser.getgroup("general", "running and selection options")
-    group._addoption('-x', '--exitfirst', action="store_true", default=False,
-               dest="exitfirst",
-               help="exit instantly on first error or failed test."),
-    group._addoption('--maxfail', metavar="num",
-               action="store", type="int", dest="maxfail", default=0,
-               help="exit after first num failures or errors.")
-
-    group._addoption('--strict', action="store_true",
-               help="run pytest in strict mode, warnings become errors.")
-
-    group = parser.getgroup("collect", "collection")
-    group.addoption('--collectonly',
-        action="store_true", dest="collectonly",
-        help="only collect tests, don't execute them."),
-    group.addoption('--pyargs', action="store_true",
-        help="try to interpret all arguments as python packages.")
-    group.addoption("--ignore", action="append", metavar="path",
-        help="ignore path during collection (multi-allowed).")
-    group.addoption('--confcutdir', dest="confcutdir", default=None,
-        metavar="dir",
-        help="only load conftest.py's relative to specified dir.")
-
-    group = parser.getgroup("debugconfig",
-        "test session debugging and configuration")
-    group.addoption('--basetemp', dest="basetemp", default=None, metavar="dir",
-               help="base temporary directory for this test run.")
-
-
-def pytest_namespace():
-    collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
-    return dict(collect=collect)
-
-def pytest_configure(config):
-    py.test.config = config # compatibiltiy
-    if config.option.exitfirst:
-        config.option.maxfail = 1
-
-def wrap_session(config, doit):
-    """Skeleton command line program"""
-    session = Session(config)
-    session.exitstatus = EXIT_OK
-    initstate = 0
-    try:
-        config.pluginmanager.do_configure(config)
-        initstate = 1
-        config.hook.pytest_sessionstart(session=session)
-        initstate = 2
-        doit(config, session)
-    except pytest.UsageError:
-        raise
-    except KeyboardInterrupt:
-        excinfo = py.code.ExceptionInfo()
-        config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
-        session.exitstatus = EXIT_INTERRUPTED
-    except:
-        excinfo = py.code.ExceptionInfo()
-        config.pluginmanager.notify_exception(excinfo, config.option)
-        session.exitstatus = EXIT_INTERNALERROR
-        if excinfo.errisinstance(SystemExit):
-            sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus or (session._testsfailed and 1))
-    if not session.exitstatus and session._testsfailed:
-        session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 1:
-        config.pluginmanager.do_unconfigure(config)
-    return session.exitstatus
-
-def pytest_cmdline_main(config):
-    return wrap_session(config, _main)
-
-def _main(config, session):
-    """ default command line protocol for initialization, session,
-    running tests and reporting. """
-    config.hook.pytest_collection(session=session)
-    config.hook.pytest_runtestloop(session=session)
-
-def pytest_collection(session):
-    return session.perform_collect()
-
-def pytest_runtestloop(session):
-    if session.config.option.collectonly:
-        return True
-    for i, item in enumerate(session.items):
-        try:
-            nextitem = session.items[i+1]
-        except IndexError:
-            nextitem = None
-        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
-        if session.shouldstop:
-            raise session.Interrupted(session.shouldstop)
-    return True
-
-def pytest_ignore_collect(path, config):
-    p = path.dirpath()
-    ignore_paths = config._getconftest_pathlist("collect_ignore", path=p)
-    ignore_paths = ignore_paths or []
-    excludeopt = config.getvalue("ignore")
-    if excludeopt:
-        ignore_paths.extend([py.path.local(x) for x in excludeopt])
-    return path in ignore_paths
-
-class HookProxy:
-    def __init__(self, fspath, config):
-        self.fspath = fspath
-        self.config = config
-    def __getattr__(self, name):
-        hookmethod = getattr(self.config.hook, name)
-        def call_matching_hooks(**kwargs):
-            plugins = self.config._getmatchingplugins(self.fspath)
-            return hookmethod.pcall(plugins, **kwargs)
-        return call_matching_hooks
-
-def compatproperty(name):
-    def fget(self):
-        return getattr(pytest, name)
-    return property(fget, None, None,
-        "deprecated attribute %r, use pytest.%s" % (name,name))
-
-class Node(object):
-    """ base class for all Nodes in the collection tree.
-    Collector subclasses have children, Items are terminal nodes."""
-
-    def __init__(self, name, parent=None, config=None, session=None):
-        #: a unique name with the scope of the parent
-        self.name = name
-
-        #: the parent collector node.
-        self.parent = parent
-
-        #: the test config object
-        self.config = config or parent.config
-
-        #: the collection this node is part of
-        self.session = session or parent.session
-
-        #: filesystem path where this node was collected from
-        self.fspath = getattr(parent, 'fspath', None)
-        self.ihook = self.session.gethookproxy(self.fspath)
-        self.keywords = {self.name: True}
-
-    Module = compatproperty("Module")
-    Class = compatproperty("Class")
-    Instance = compatproperty("Instance")
-    Function = compatproperty("Function")
-    File = compatproperty("File")
-    Item = compatproperty("Item")
-
-    def _getcustomclass(self, name):
-        cls = getattr(self, name)
-        if cls != getattr(pytest, name):
-            py.log._apiwarn("2.0", "use of node.%s is deprecated, "
-                "use pytest_pycollect_makeitem(...) to create custom "
-                "collection nodes" % name)
-        return cls
-
-    def __repr__(self):
-        return "<%s %r>" %(self.__class__.__name__, getattr(self, 'name', None))
-
-    # methods for ordering nodes
-    @property
-    def nodeid(self):
-        try:
-            return self._nodeid
-        except AttributeError:
-            self._nodeid = x = self._makeid()
-            return x
-
-    def _makeid(self):
-        return self.parent.nodeid + "::" + self.name
-
-    def __eq__(self, other):
-        if not isinstance(other, Node):
-            return False
-        return self.__class__ == other.__class__ and \
-               self.name == other.name and self.parent == other.parent
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __hash__(self):
-        return hash((self.name, self.parent))
-
-    def setup(self):
-        pass
-
-    def teardown(self):
-        pass
-
-    def _memoizedcall(self, attrname, function):
-        exattrname = "_ex_" + attrname
-        failure = getattr(self, exattrname, None)
-        if failure is not None:
-            py.builtin._reraise(failure[0], failure[1], failure[2])
-        if hasattr(self, attrname):
-            return getattr(self, attrname)
-        try:
-            res = function()
-        except py.builtin._sysex:
-            raise
-        except:
-            failure = py.std.sys.exc_info()
-            setattr(self, exattrname, failure)
-            raise
-        setattr(self, attrname, res)
-        return res
-
-    def listchain(self):
-        """ return list of all parent collectors up to self,
-            starting from root of collection tree. """
-        chain = []
-        item = self
-        while item is not None:
-            chain.append(item)
-            item = item.parent
-        chain.reverse()
-        return chain
-
-    def listnames(self):
-        return [x.name for x in self.listchain()]
-
-    def getplugins(self):
-        return self.config._getmatchingplugins(self.fspath)
-
-    def getparent(self, cls):
-        current = self
-        while current and not isinstance(current, cls):
-            current = current.parent
-        return current
-
-    def _prunetraceback(self, excinfo):
-        pass
-
-    def _repr_failure_py(self, excinfo, style=None):
-        if self.config.option.fulltrace:
-            style="long"
-        else:
-            self._prunetraceback(excinfo)
-        # XXX should excinfo.getrepr record all data and toterminal()
-        # process it?
-        if style is None:
-            if self.config.option.tbstyle == "short":
-                style = "short"
-            else:
-                style = "long"
-        return excinfo.getrepr(funcargs=True,
-                               showlocals=self.config.option.showlocals,
-                               style=style)
-
-    repr_failure = _repr_failure_py
-
-class Collector(Node):
-    """ Collector instances create children through collect()
-        and thus iteratively build a tree.
-    """
-    class CollectError(Exception):
-        """ an error during collection, contains a custom message. """
-
-    def collect(self):
-        """ returns a list of children (items and collectors)
-            for this collection node.
-        """
-        raise NotImplementedError("abstract")
-
-    def repr_failure(self, excinfo):
-        """ represent a collection failure. """
-        if excinfo.errisinstance(self.CollectError):
-            exc = excinfo.value
-            return str(exc.args[0])
-        return self._repr_failure_py(excinfo, style="short")
-
-    def _memocollect(self):
-        """ internal helper method to cache results of calling collect(). """
-        return self._memoizedcall('_collected', lambda: list(self.collect()))
-
-    def _prunetraceback(self, excinfo):
-        if hasattr(self, 'fspath'):
-            path = self.fspath
-            traceback = excinfo.traceback
-            ntraceback = traceback.cut(path=self.fspath)
-            if ntraceback == traceback:
-                ntraceback = ntraceback.cut(excludepath=tracebackcutdir)
-            excinfo.traceback = ntraceback.filter()
-
-class FSCollector(Collector):
-    def __init__(self, fspath, parent=None, config=None, session=None):
-        fspath = py.path.local(fspath) # xxx only for test_resultlog.py?
-        name = fspath.basename
-        if parent is not None:
-            rel = fspath.relto(parent.fspath)
-            if rel:
-                name = rel
-            name = name.replace(os.sep, "/")
-        super(FSCollector, self).__init__(name, parent, config, session)
-        self.fspath = fspath
-
-    def _makeid(self):
-        if self == self.session:
-            return "."
-        relpath = self.session.fspath.bestrelpath(self.fspath)
-        if os.sep != "/":
-            relpath = relpath.replace(os.sep, "/")
-        return relpath
-
-class File(FSCollector):
-    """ base class for collecting tests from a file. """
-
-class Item(Node):
-    """ a basic test invocation item. Note that for a single function
-    there might be multiple test invocation items.
-    """
-    nextitem = None
-
-    def reportinfo(self):
-        return self.fspath, None, ""
-
-    @property
-    def location(self):
-        try:
-            return self._location
-        except AttributeError:
-            location = self.reportinfo()
-            # bestrelpath is a quite slow function
-            cache = self.config.__dict__.setdefault("_bestrelpathcache", {})
-            try:
-                fspath = cache[location[0]]
-            except KeyError:
-                fspath = self.session.fspath.bestrelpath(location[0])
-                cache[location[0]] = fspath
-            location = (fspath, location[1], str(location[2]))
-            self._location = location
-            return location
-
-class NoMatch(Exception):
-    """ raised if matching cannot locate a matching names. """
-
-class Session(FSCollector):
-    class Interrupted(KeyboardInterrupt):
-        """ signals an interrupted test run. """
-        __module__ = 'builtins' # for py3
-
-    def __init__(self, config):
-        super(Session, self).__init__(py.path.local(), parent=None,
-            config=config, session=self)
-        assert self.config.pluginmanager.register(self, name="session", prepend=True)
-        self._testsfailed = 0
-        self.shouldstop = False
-        self.trace = config.trace.root.get("collection")
-        self._norecursepatterns = config.getini("norecursedirs")
-
-    def pytest_collectstart(self):
-        if self.shouldstop:
-            raise self.Interrupted(self.shouldstop)
-
-    def pytest_runtest_logreport(self, report):
-        if report.failed and 'xfail' not in getattr(report, 'keywords', []):
-            self._testsfailed += 1
-            maxfail = self.config.getvalue("maxfail")
-            if maxfail and self._testsfailed >= maxfail:
-                self.shouldstop = "stopping after %d failures" % (
-                    self._testsfailed)
-    pytest_collectreport = pytest_runtest_logreport
-
-    def isinitpath(self, path):
-        return path in self._initialpaths
-
-    def gethookproxy(self, fspath):
-        return HookProxy(fspath, self.config)
-
-    def perform_collect(self, args=None, genitems=True):
-        hook = self.config.hook
-        try:
-            items = self._perform_collect(args, genitems)
-            hook.pytest_collection_modifyitems(session=self,
-                config=self.config, items=items)
-        finally:
-            hook.pytest_collection_finish(session=self)
-        return items
-
-    def _perform_collect(self, args, genitems):
-        if args is None:
-            args = self.config.args
-        self.trace("perform_collect", self, args)
-        self.trace.root.indent += 1
-        self._notfound = []
-        self._initialpaths = set()
-        self._initialparts = []
-        self.items = items = []
-        for arg in args:
-            parts = self._parsearg(arg)
-            self._initialparts.append(parts)
-            self._initialpaths.add(parts[0])
-        self.ihook.pytest_collectstart(collector=self)
-        rep = self.ihook.pytest_make_collect_report(collector=self)
-        self.ihook.pytest_collectreport(report=rep)
-        self.trace.root.indent -= 1
-        if self._notfound:
-            for arg, exc in self._notfound:
-                line = "(no name %r in any of %r)" % (arg, exc.args[0])
-                raise pytest.UsageError("not found: %s\n%s" %(arg, line))
-        if not genitems:
-            return rep.result
-        else:
-            if rep.passed:
-                for node in rep.result:
-                    self.items.extend(self.genitems(node))
-            return items
-
-    def collect(self):
-        for parts in self._initialparts:
-            arg = "::".join(map(str, parts))
-            self.trace("processing argument", arg)
-            self.trace.root.indent += 1
-            try:
-                for x in self._collect(arg):
-                    yield x
-            except NoMatch:
-                # we are inside a make_report hook so
-                # we cannot directly pass through the exception
-                self._notfound.append((arg, sys.exc_info()[1]))
-                self.trace.root.indent -= 1
-                break
-            self.trace.root.indent -= 1
-
-    def _collect(self, arg):
-        names = self._parsearg(arg)
-        path = names.pop(0)
-        if path.check(dir=1):
-            assert not names, "invalid arg %r" %(arg,)
-            for path in path.visit(fil=lambda x: x.check(file=1),
-                rec=self._recurse, bf=True, sort=True):
-                for x in self._collectfile(path):
-                    yield x
-        else:
-            assert path.check(file=1)
-            for x in self.matchnodes(self._collectfile(path), names):
-                yield x
-
-    def _collectfile(self, path):
-        ihook = self.gethookproxy(path)
-        if not self.isinitpath(path):
-            if ihook.pytest_ignore_collect(path=path, config=self.config):
-               return ()
-        return ihook.pytest_collect_file(path=path, parent=self)
-
-    def _recurse(self, path):
-        ihook = self.gethookproxy(path.dirpath())
-        if ihook.pytest_ignore_collect(path=path, config=self.config):
-           return
-        for pat in self._norecursepatterns:
-            if path.check(fnmatch=pat):
-                return False
-        ihook = self.gethookproxy(path)
-        ihook.pytest_collect_directory(path=path, parent=self)
-        return True
-
-    def _tryconvertpyarg(self, x):
-        mod = None
-        path = [os.path.abspath('.')] + sys.path
-        for name in x.split('.'):
-            # ignore anything that's not a proper name here
-            # else something like --pyargs will mess up '.'
-            # since imp.find_module will actually sometimes work for it
-            # but it's supposed to be considered a filesystem path
-            # not a package
-            if name_re.match(name) is None:
-                return x
-            try:
-                fd, mod, type_ = imp.find_module(name, path)
-            except ImportError:
-                return x
-            else:
-                if fd is not None:
-                    fd.close()
-
-            if type_[2] != imp.PKG_DIRECTORY:
-                path = [os.path.dirname(mod)]
-            else:
-                path = [mod]
-        return mod
-
-    def _parsearg(self, arg):
-        """ return (fspath, names) tuple after checking the file exists. """
-        arg = str(arg)
-        if self.config.option.pyargs:
-            arg = self._tryconvertpyarg(arg)
-        parts = str(arg).split("::")
-        relpath = parts[0].replace("/", os.sep)
-        path = self.fspath.join(relpath, abs=True)
-        if not path.check():
-            if self.config.option.pyargs:
-                msg = "file or package not found: "
-            else:
-                msg = "file not found: "
-            raise pytest.UsageError(msg + arg)
-        parts[0] = path
-        return parts
-
-    def matchnodes(self, matching, names):
-        self.trace("matchnodes", matching, names)
-        self.trace.root.indent += 1
-        nodes = self._matchnodes(matching, names)
-        num = len(nodes)
-        self.trace("matchnodes finished -> ", num, "nodes")
-        self.trace.root.indent -= 1
-        if num == 0:
-            raise NoMatch(matching, names[:1])
-        return nodes
-
-    def _matchnodes(self, matching, names):
-        if not matching or not names:
-            return matching
-        name = names[0]
-        assert name
-        nextnames = names[1:]
-        resultnodes = []
-        for node in matching:
-            if isinstance(node, pytest.Item):
-                if not names:
-                    resultnodes.append(node)
-                continue
-            assert isinstance(node, pytest.Collector)
-            node.ihook.pytest_collectstart(collector=node)
-            rep = node.ihook.pytest_make_collect_report(collector=node)
-            if rep.passed:
-                has_matched = False
-                for x in rep.result:
-                    if x.name == name:
-                        resultnodes.extend(self.matchnodes([x], nextnames))
-                        has_matched = True
-                # XXX accept IDs that don't have "()" for class instances
-                if not has_matched and len(rep.result) == 1 and x.name == "()":
-                    nextnames.insert(0, name)
-                    resultnodes.extend(self.matchnodes([x], nextnames))
-            node.ihook.pytest_collectreport(report=rep)
-        return resultnodes
-
-    def genitems(self, node):
-        self.trace("genitems", node)
-        if isinstance(node, pytest.Item):
-            node.ihook.pytest_itemcollected(item=node)
-            yield node
-        else:
-            assert isinstance(node, pytest.Collector)
-            node.ihook.pytest_collectstart(collector=node)
-            rep = node.ihook.pytest_make_collect_report(collector=node)
-            if rep.passed:
-                for subnode in rep.result:
-                    for x in self.genitems(subnode):
-                        yield x
-            node.ihook.pytest_collectreport(report=rep)
diff --git a/_pytest/mark.py b/_pytest/mark.py
deleted file mode 100644
--- a/_pytest/mark.py
+++ /dev/null
@@ -1,226 +0,0 @@
-""" generic mechanism for marking and selecting python functions. """
-import pytest, py
-
-def pytest_namespace():
-    return {'mark': MarkGenerator()}
-
-def pytest_addoption(parser):
-    group = parser.getgroup("general")
-    group._addoption('-k',
-        action="store", dest="keyword", default='', metavar="KEYWORDEXPR",
-        help="only run tests which match given keyword expression.  "
-             "An expression consists of space-separated terms. "
-             "Each term must match. Precede a term with '-' to negate. "
-             "Terminate expression with ':' to make the first match match "
-             "all subsequent tests (usually file-order). ")
-
-    group._addoption("-m",
-        action="store", dest="markexpr", default="", metavar="MARKEXPR",
-        help="only run tests matching given mark expression.  "
-             "example: -m 'mark1 and not mark2'."
-             )
-
-    group.addoption("--markers", action="store_true", help=
-        "show markers (builtin, plugin and per-project ones).")
-
-    parser.addini("markers", "markers for test functions", 'linelist')
-
-def pytest_cmdline_main(config):
-    if config.option.markers:
-        config.pluginmanager.do_configure(config)
-        tw = py.io.TerminalWriter()
-        for line in config.getini("markers"):
-            name, rest = line.split(":", 1)
-            tw.write("@pytest.mark.%s:" %  name, bold=True)
-            tw.line(rest)
-            tw.line()
-        config.pluginmanager.do_unconfigure(config)
-        return 0
-pytest_cmdline_main.tryfirst = True
-
-def pytest_collection_modifyitems(items, config):
-    keywordexpr = config.option.keyword
-    matchexpr = config.option.markexpr
-    if not keywordexpr and not matchexpr:
-        return
-    selectuntil = False
-    if keywordexpr[-1:] == ":":
-        selectuntil = True
-        keywordexpr = keywordexpr[:-1]
-
-    remaining = []
-    deselected = []
-    for colitem in items:
-        if keywordexpr and skipbykeyword(colitem, keywordexpr):
-            deselected.append(colitem)
-        else:
-            if selectuntil:
-                keywordexpr = None
-            if matchexpr:
-                if not matchmark(colitem, matchexpr):
-                    deselected.append(colitem)
-                    continue
-            remaining.append(colitem)
-
-    if deselected:
-        config.hook.pytest_deselected(items=deselected)
-        items[:] = remaining
-
-class BoolDict:
-    def __init__(self, mydict):
-        self._mydict = mydict
-    def __getitem__(self, name):
-        return name in self._mydict
-
-def matchmark(colitem, matchexpr):
-    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
-
-def pytest_configure(config):
-    if config.option.strict:
-        pytest.mark._config = config
-
-def skipbykeyword(colitem, keywordexpr):
-    """ return True if they given keyword expression means to
-        skip this collector/item.
-    """
-    if not keywordexpr:
-        return
-
-    itemkeywords = getkeywords(colitem)
-    for key in filter(None, keywordexpr.split()):
-        eor = key[:1] == '-'
-        if eor:
-            key = key[1:]
-        if not (eor ^ matchonekeyword(key, itemkeywords)):
-            return True
-
-def getkeywords(node):
-    keywords = {}
-    while node is not None:
-        keywords.update(node.keywords)
-        node = node.parent
-    return keywords
-
-
-def matchonekeyword(key, itemkeywords):
-    for elem in key.split("."):
-        for kw in itemkeywords:
-            if elem in kw:
-                break
-        else:
-            return False
-    return True
-
-class MarkGenerator:
-    """ Factory for :class:`MarkDecorator` objects - exposed as
-    a ``py.test.mark`` singleton instance.  Example::
-
-         import py
-         @py.test.mark.slowtest
-         def test_function():
-            pass
-
-    will set a 'slowtest' :class:`MarkInfo` object
-    on the ``test_function`` object. """
-
-    def __getattr__(self, name):
-        if name[0] == "_":
-            raise AttributeError(name)
-        if hasattr(self, '_config'):
-            self._check(name)
-        return MarkDecorator(name)
-
-    def _check(self, name):
-        try:
-            if name in self._markers:
-                return
-        except AttributeError:
-            pass
-        self._markers = l = set()
-        for line in self._config.getini("markers"):
-            beginning = line.split(":", 1)
-            x = beginning[0].split("(", 1)[0]
-            l.add(x)
-        if name not in self._markers:
-            raise AttributeError("%r not a registered marker" % (name,))
-
-class MarkDecorator:
-    """ A decorator for test functions and test classes.  When applied
-    it will create :class:`MarkInfo` objects which may be
-    :ref:`retrieved by hooks as item keywords <excontrolskip>`.
-    MarkDecorator instances are often created like this::
-
-        mark1 = py.test.mark.NAME              # simple MarkDecorator
-        mark2 = py.test.mark.NAME(name1=value) # parametrized MarkDecorator
-
-    and can then be applied as decorators to test functions::
-
-        @mark2
-        def test_function():
-            pass
-    """
-    def __init__(self, name, args=None, kwargs=None):
-        self.markname = name
-        self.args = args or ()
-        self.kwargs = kwargs or {}
-
-    def __repr__(self):
-        d = self.__dict__.copy()
-        name = d.pop('markname')
-        return "<MarkDecorator %r %r>" %(name, d)
-
-    def __call__(self, *args, **kwargs):
-        """ if passed a single callable argument: decorate it with mark info.
-            otherwise add *args/**kwargs in-place to mark information. """
-        if args:
-            func = args[0]
-            if len(args) == 1 and hasattr(func, '__call__') or \
-               hasattr(func, '__bases__'):
-                if hasattr(func, '__bases__'):
-                    if hasattr(func, 'pytestmark'):
-                        l = func.pytestmark
-                        if not isinstance(l, list):
-                           func.pytestmark = [l, self]
-                        else:
-                           l.append(self)
-                    else:
-                       func.pytestmark = [self]
-                else:
-                    holder = getattr(func, self.markname, None)
-                    if holder is None:
-                        holder = MarkInfo(self.markname, self.args, self.kwargs)
-                        setattr(func, self.markname, holder)
-                    else:
-                        holder.add(self.args, self.kwargs)
-                return func
-        kw = self.kwargs.copy()
-        kw.update(kwargs)
-        args = self.args + args
-        return self.__class__(self.markname, args=args, kwargs=kw)
-
-class MarkInfo:
-    """ Marking object created by :class:`MarkDecorator` instances. """
-    def __init__(self, name, args, kwargs):
-        #: name of attribute
-        self.name = name
-        #: positional argument list, empty if none specified
-        self.args = args
-        #: keyword argument dictionary, empty if nothing specified
-        self.kwargs = kwargs
-        self._arglist = [(args, kwargs.copy())]
-
-    def __repr__(self):
-        return "<MarkInfo %r args=%r kwargs=%r>" % (
-                self.name, self.args, self.kwargs)
-
-    def add(self, args, kwargs):
-        """ add a MarkInfo with the given args and kwargs. """
-        self._arglist.append((args, kwargs))
-        self.args += args
-        self.kwargs.update(kwargs)
-
-    def __iter__(self):
-        """ yield MarkInfo objects each relating to a marking-call. """
-        for args, kwargs in self._arglist:
-            yield MarkInfo(self.name, args, kwargs)
-
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
deleted file mode 100644
--- a/_pytest/monkeypatch.py
+++ /dev/null
@@ -1,124 +0,0 @@
-""" monkeypatching and mocking functionality.  """
-
-import os, sys
-
-def pytest_funcarg__monkeypatch(request):
-    """The returned ``monkeypatch`` funcarg provides these
-    helper methods to modify objects, dictionaries or os.environ::
-
-        monkeypatch.setattr(obj, name, value, raising=True)
-        monkeypatch.delattr(obj, name, raising=True)
-        monkeypatch.setitem(mapping, name, value)
-        monkeypatch.delitem(obj, name, raising=True)
-        monkeypatch.setenv(name, value, prepend=False)
-        monkeypatch.delenv(name, value, raising=True)
-        monkeypatch.syspath_prepend(path)
-        monkeypatch.chdir(path)
-
-    All modifications will be undone after the requesting
-    test function has finished. The ``raising``
-    parameter determines if a KeyError or AttributeError
-    will be raised if the set/deletion operation has no target.
-    """
-    mpatch = monkeypatch()
-    request.addfinalizer(mpatch.undo)
-    return mpatch
-
-notset = object()
-
-class monkeypatch:
-    """ object keeping a record of setattr/item/env/syspath changes. """
-    def __init__(self):
-        self._setattr = []
-        self._setitem = []
-        self._cwd = None
-
-    def setattr(self, obj, name, value, raising=True):
-        """ set attribute ``name`` on ``obj`` to ``value``, by default
-        raise AttributeEror if the attribute did not exist. """
-        oldval = getattr(obj, name, notset)
-        if raising and oldval is notset:
-            raise AttributeError("%r has no attribute %r" %(obj, name))
-        self._setattr.insert(0, (obj, name, oldval))
-        setattr(obj, name, value)
-
-    def delattr(self, obj, name, raising=True):
-        """ delete attribute ``name`` from ``obj``, by default raise
-        AttributeError it the attribute did not previously exist. """
-        if not hasattr(obj, name):
-            if raising:
-                raise AttributeError(name)
-        else:
-            self._setattr.insert(0, (obj, name, getattr(obj, name, notset)))
-            delattr(obj, name)
-
-    def setitem(self, dic, name, value):
-        """ set dictionary entry ``name`` to value. """
-        self._setitem.insert(0, (dic, name, dic.get(name, notset)))
-        dic[name] = value
-
-    def delitem(self, dic, name, raising=True):
-        """ delete ``name`` from dict, raise KeyError if it doesn't exist."""
-        if name not in dic:
-            if raising:
-                raise KeyError(name)
-        else:
-            self._setitem.insert(0, (dic, name, dic.get(name, notset)))
-            del dic[name]
-
-    def setenv(self, name, value, prepend=None):
-        """ set environment variable ``name`` to ``value``.  if ``prepend``
-        is a character, read the current environment variable value
-        and prepend the ``value`` adjoined with the ``prepend`` character."""
-        value = str(value)
-        if prepend and name in os.environ:
-            value = value + prepend + os.environ[name]
-        self.setitem(os.environ, name, value)
-
-    def delenv(self, name, raising=True):
-        """ delete ``name`` from environment, raise KeyError it not exists."""
-        self.delitem(os.environ, name, raising=raising)
-
-    def syspath_prepend(self, path):
-        """ prepend ``path`` to ``sys.path`` list of import locations. """
-        if not hasattr(self, '_savesyspath'):
-            self._savesyspath = sys.path[:]
-        sys.path.insert(0, str(path))
-
-    def chdir(self, path):
-        """ change the current working directory to the specified path
-        path can be a string or a py.path.local object
-        """
-        if self._cwd is None:
-            self._cwd = os.getcwd()
-        if hasattr(path, "chdir"):
-            path.chdir()
-        else:
-            os.chdir(path)
-
-    def undo(self):
-        """ undo previous changes.  This call consumes the
-        undo stack.  Calling it a second time has no effect unless
-        you  do more monkeypatching after the undo call."""
-        for obj, name, value in self._setattr:
-            if value is not notset:
-                setattr(obj, name, value)
-            else:
-                delattr(obj, name)
-        self._setattr[:] = []
-        for dictionary, name, value in self._setitem:
-            if value is notset:
-                try:
-                    del dictionary[name]
-                except KeyError:
-                    pass # was already deleted, so we have the desired state
-            else:
-                dictionary[name] = value
-        self._setitem[:] = []
-        if hasattr(self, '_savesyspath'):
-            sys.path[:] = self._savesyspath
-            del self._savesyspath
-
-        if self._cwd is not None:
-            os.chdir(self._cwd)
-            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
deleted file mode 100644
--- a/_pytest/nose.py
+++ /dev/null
@@ -1,48 +0,0 @@
-""" run test suites written for nose. """
-
-import pytest, py
-import inspect
-import sys
-
-def pytest_runtest_makereport(__multicall__, item, call):
-    SkipTest = getattr(sys.modules.get('nose', None), 'SkipTest', None)
-    if SkipTest:
-        if call.excinfo and call.excinfo.errisinstance(SkipTest):
-            # let's substitute the excinfo with a py.test.skip one
-            call2 = call.__class__(lambda: py.test.skip(str(call.excinfo.value)), call.when)
-            call.excinfo = call2.excinfo
-
-
- at pytest.mark.trylast
-def pytest_runtest_setup(item):
-    if isinstance(item, (pytest.Function)):
-        if isinstance(item.parent, pytest.Generator):
-            gen = item.parent
-            if not hasattr(gen, '_nosegensetup'):
-                call_optional(gen.obj, 'setup')
-                if isinstance(gen.parent, pytest.Instance):
-                    call_optional(gen.parent.obj, 'setup')
-                gen._nosegensetup = True
-        if not call_optional(item.obj, 'setup'):
-            # call module level setup if there is no object level one
-            call_optional(item.parent.obj, 'setup')
-
-def pytest_runtest_teardown(item):
-    if isinstance(item, pytest.Function):
-        if not call_optional(item.obj, 'teardown'):
-            call_optional(item.parent.obj, 'teardown')
-        #if hasattr(item.parent, '_nosegensetup'):
-        #    #call_optional(item._nosegensetup, 'teardown')
-        #    del item.parent._nosegensetup
-
-def pytest_make_collect_report(collector):
-    if isinstance(collector, pytest.Generator):
-        call_optional(collector.obj, 'setup')
-
-def call_optional(obj, name):
-    method = getattr(obj, name, None)
-    if method:
-        # If there's any problems allow the exception to raise rather than
-        # silently ignoring them
-        method()
-        return True
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
deleted file mode 100644
--- a/_pytest/pastebin.py
+++ /dev/null
@@ -1,67 +0,0 @@
-""" submit failure or test session information to a pastebin service. """
-import py, sys
-
-class url:
-    base = "http://paste.pocoo.org"
-    xmlrpc = base + "/xmlrpc/"
-    show = base + "/show/"
-
-def pytest_addoption(parser):
-    group = parser.getgroup("terminal reporting")
-    group._addoption('--pastebin', metavar="mode",
-        action='store', dest="pastebin", default=None,
-        type="choice", choices=['failed', 'all'],
-        help="send failed|all info to Pocoo pastebin service.")
-
-def pytest_configure(__multicall__, config):
-    import tempfile
-    __multicall__.execute()
-    if config.option.pastebin == "all":
-        config._pastebinfile = tempfile.TemporaryFile('w+')
-        tr = config.pluginmanager.getplugin('terminalreporter')
-        oldwrite = tr._tw.write
-        def tee_write(s, **kwargs):
-            oldwrite(s, **kwargs)
-            config._pastebinfile.write(str(s))
-        tr._tw.write = tee_write
-
-def pytest_unconfigure(config):
-    if hasattr(config, '_pastebinfile'):
-        config._pastebinfile.seek(0)
-        sessionlog = config._pastebinfile.read()
-        config._pastebinfile.close()
-        del config._pastebinfile
-        proxyid = getproxy().newPaste("python", sessionlog)
-        pastebinurl = "%s%s" % (url.show, proxyid)
-        sys.stderr.write("pastebin session-log: %s\n" % pastebinurl)
-        tr = config.pluginmanager.getplugin('terminalreporter')
-        del tr._tw.__dict__['write']
-
-def getproxy():
-    if sys.version_info < (3, 0):
-        from xmlrpclib import ServerProxy
-    else:
-        from xmlrpc.client import ServerProxy
-    return ServerProxy(url.xmlrpc).pastes
-
-def pytest_terminal_summary(terminalreporter):
-    if terminalreporter.config.option.pastebin != "failed":
-        return
-    tr = terminalreporter
-    if 'failed' in tr.stats:
-        terminalreporter.write_sep("=", "Sending information to Paste Service")
-        if tr.config.option.debug:
-            terminalreporter.write_line("xmlrpcurl: %s" %(url.xmlrpc,))
-        serverproxy = getproxy()
-        for rep in terminalreporter.stats.get('failed'):
-            try:
-                msg = rep.longrepr.reprtraceback.reprentries[-1].reprfileloc
-            except AttributeError:
-                msg = tr._getfailureheadline(rep)
-            tw = py.io.TerminalWriter(stringio=True)
-            rep.toterminal(tw)
-            s = tw.stringio.getvalue()
-            assert len(s)
-            proxyid = serverproxy.newPaste("python", s)
-            pastebinurl = "%s%s" % (url.show, proxyid)
-            tr.write_line("%s --> %s" %(msg, pastebinurl))
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
deleted file mode 100644
--- a/_pytest/pdb.py
+++ /dev/null
@@ -1,95 +0,0 @@
-""" interactive debugging with PDB, the Python Debugger. """
-
-import pytest, py
-import sys
-
-def pytest_addoption(parser):
-    group = parser.getgroup("general")
-    group._addoption('--pdb',
-               action="store_true", dest="usepdb", default=False,
-               help="start the interactive Python debugger on errors.")
-
-def pytest_namespace():
-    return {'set_trace': pytestPDB().set_trace}
-
-def pytest_configure(config):
-    if config.getvalue("usepdb"):
-        config.pluginmanager.register(PdbInvoke(), 'pdbinvoke')
-
-class pytestPDB:
-    """ Pseudo PDB that defers to the real pdb. """
-    item = None
-    collector = None
-
-    def set_trace(self):
-        """ invoke PDB set_trace debugging, dropping any IO capturing. """
-        frame = sys._getframe().f_back
-        item = self.item or self.collector
-
-        if item is not None:
-            capman = item.config.pluginmanager.getplugin("capturemanager")
-            out, err = capman.suspendcapture()
-            if hasattr(item, 'outerr'):
-                item.outerr = (item.outerr[0] + out, item.outerr[1] + err)
-            tw = py.io.TerminalWriter()
-            tw.line()
-            tw.sep(">", "PDB set_trace (IO-capturing turned off)")
-        py.std.pdb.Pdb().set_trace(frame)
-
-def pdbitem(item):
-    pytestPDB.item = item
-pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
-
- at pytest.mark.tryfirst
-def pytest_make_collect_report(__multicall__, collector):
-    try:
-        pytestPDB.collector = collector
-        return __multicall__.execute()
-    finally:
-        pytestPDB.collector = None
-
-def pytest_runtest_makereport():
-    pytestPDB.item = None
-    
-class PdbInvoke:
-    @pytest.mark.tryfirst
-    def pytest_runtest_makereport(self, item, call, __multicall__):
-        rep = __multicall__.execute()
-        if not call.excinfo or \
-            call.excinfo.errisinstance(pytest.skip.Exception) or \
-            call.excinfo.errisinstance(py.std.bdb.BdbQuit):
-            return rep
-        if "xfail" in rep.keywords:
-            return rep
-        # we assume that the above execute() suspended capturing
-        # XXX we re-use the TerminalReporter's terminalwriter
-        # because this seems to avoid some encoding related troubles
-        # for not completely clear reasons.
-        tw = item.config.pluginmanager.getplugin("terminalreporter")._tw
-        tw.line()
-        tw.sep(">", "traceback")
-        rep.toterminal(tw)
-        tw.sep(">", "entering PDB")
-        # A doctest.UnexpectedException is not useful for post_mortem.
-        # Use the underlying exception instead:
-        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
-            tb = call.excinfo.value.exc_info[2]
-        else:
-            tb = call.excinfo._excinfo[2]
-        post_mortem(tb)
-        rep._pdbshown = True
-        return rep
-
-def post_mortem(t):
-    pdb = py.std.pdb
-    class Pdb(pdb.Pdb):
-        def get_stack(self, f, t):
-            stack, i = pdb.Pdb.get_stack(self, f, t)
-            if f is None:
-                i = max(0, len(stack) - 1)
-                while i and stack[i][0].f_locals.get("__tracebackhide__", False):
-                    i-=1
-            return stack, i
-    p = Pdb()
-    p.reset()
-    p.interaction(None, t)
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
deleted file mode 100644
--- a/_pytest/pytester.py
+++ /dev/null
@@ -1,680 +0,0 @@
-""" (disabled by default) support for testing py.test and py.test plugins. """
-
-import py, pytest
-import sys, os
-import re
-import inspect
-import time
-from fnmatch import fnmatch
-from _pytest.main import Session, EXIT_OK
-from py.builtin import print_
-from _pytest.core import HookRelay
-
-def pytest_addoption(parser):
-    group = parser.getgroup("pylib")
-    group.addoption('--no-tools-on-path',
-           action="store_true", dest="notoolsonpath", default=False,
-           help=("discover tools on PATH instead of going through py.cmdline.")
-    )
-
-def pytest_configure(config):
-    # This might be called multiple times. Only take the first.
-    global _pytest_fullpath
-    import pytest
-    try:
-        _pytest_fullpath
-    except NameError:
-        _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
-        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
-
-def pytest_funcarg___pytest(request):
-    return PytestArg(request)
-
-class PytestArg:
-    def __init__(self, request):
-        self.request = request
-
-    def gethookrecorder(self, hook):
-        hookrecorder = HookRecorder(hook._pm)
-        hookrecorder.start_recording(hook._hookspecs)
-        self.request.addfinalizer(hookrecorder.finish_recording)
-        return hookrecorder
-
-class ParsedCall:
-    def __init__(self, name, locals):
-        assert '_name' not in locals
-        self.__dict__.update(locals)
-        self.__dict__.pop('self')
-        self._name = name
-
-    def __repr__(self):
-        d = self.__dict__.copy()
-        del d['_name']
-        return "<ParsedCall %r(**%r)>" %(self._name, d)
-
-class HookRecorder:
-    def __init__(self, pluginmanager):
-        self._pluginmanager = pluginmanager
-        self.calls = []
-        self._recorders = {}
-
-    def start_recording(self, hookspecs):
-        if not isinstance(hookspecs, (list, tuple)):
-            hookspecs = [hookspecs]
-        for hookspec in hookspecs:
-            assert hookspec not in self._recorders
-            class RecordCalls:
-                _recorder = self
-            for name, method in vars(hookspec).items():
-                if name[0] != "_":
-                    setattr(RecordCalls, name, self._makecallparser(method))
-            recorder = RecordCalls()
-            self._recorders[hookspec] = recorder
-            self._pluginmanager.register(recorder)
-        self.hook = HookRelay(hookspecs, pm=self._pluginmanager,
-            prefix="pytest_")
-
-    def finish_recording(self):
-        for recorder in self._recorders.values():
-            self._pluginmanager.unregister(recorder)
-        self._recorders.clear()
-
-    def _makecallparser(self, method):
-        name = method.__name__
-        args, varargs, varkw, default = py.std.inspect.getargspec(method)
-        if not args or args[0] != "self":
-            args.insert(0, 'self')
-        fspec = py.std.inspect.formatargspec(args, varargs, varkw, default)
-        # we use exec because we want to have early type
-        # errors on wrong input arguments, using
-        # *args/**kwargs delays this and gives errors
-        # elsewhere
-        exec (py.code.compile("""
-            def %(name)s%(fspec)s:
-                        self._recorder.calls.append(
-                            ParsedCall(%(name)r, locals()))
-        """ % locals()))
-        return locals()[name]
-
-    def getcalls(self, names):
-        if isinstance(names, str):
-            names = names.split()
-        for name in names:
-            for cls in self._recorders:
-                if name in vars(cls):
-                    break
-            else:
-                raise ValueError("callname %r not found in %r" %(
-                name, self._recorders.keys()))
-        l = []
-        for call in self.calls:
-            if call._name in names:
-                l.append(call)
-        return l
-
-    def contains(self, entries):
-        __tracebackhide__ = True
-        from py.builtin import print_
-        i = 0
-        entries = list(entries)
-        backlocals = py.std.sys._getframe(1).f_locals
-        while entries:
-            name, check = entries.pop(0)
-            for ind, call in enumerate(self.calls[i:]):
-                if call._name == name:
-                    print_("NAMEMATCH", name, call)
-                    if eval(check, backlocals, call.__dict__):
-                        print_("CHECKERMATCH", repr(check), "->", call)
-                    else:
-                        print_("NOCHECKERMATCH", repr(check), "-", call)
-                        continue
-                    i += ind + 1
-                    break
-                print_("NONAMEMATCH", name, "with", call)
-            else:
-                py.test.fail("could not find %r check %r" % (name, check))
-
-    def popcall(self, name):
-        __tracebackhide__ = True
-        for i, call in enumerate(self.calls):
-            if call._name == name:
-                del self.calls[i]
-                return call
-        lines = ["could not find call %r, in:" % (name,)]
-        lines.extend(["  %s" % str(x) for x in self.calls])
-        py.test.fail("\n".join(lines))
-
-    def getcall(self, name):
-        l = self.getcalls(name)
-        assert len(l) == 1, (name, l)
-        return l[0]
-
-
-def pytest_funcarg__linecomp(request):
-    return LineComp()
-
-def pytest_funcarg__LineMatcher(request):
-    return LineMatcher
-
-def pytest_funcarg__testdir(request):
-    tmptestdir = TmpTestdir(request)
-    return tmptestdir
-
-rex_outcome = re.compile("(\d+) (\w+)")
-class RunResult:
-    def __init__(self, ret, outlines, errlines, duration):
-        self.ret = ret
-        self.outlines = outlines
-        self.errlines = errlines
-        self.stdout = LineMatcher(outlines)
-        self.stderr = LineMatcher(errlines)
-        self.duration = duration
-
-    def parseoutcomes(self):
-        for line in reversed(self.outlines):
-            if 'seconds' in line:
-                outcomes = rex_outcome.findall(line)
-                if outcomes:
-                    d = {}
-                    for num, cat in outcomes:
-                        d[cat] = int(num)
-                    return d
-
-class TmpTestdir:
-    def __init__(self, request):
-        self.request = request
-        self.Config = request.config.__class__
-        self._pytest = request.getfuncargvalue("_pytest")
-        # XXX remove duplication with tmpdir plugin
-        basetmp = request.config._tmpdirhandler.ensuretemp("testdir")
-        name = request.function.__name__
-        for i in range(100):
-            try:
-                tmpdir = basetmp.mkdir(name + str(i))
-            except py.error.EEXIST:
-                continue
-            break
-        # we need to create another subdir
-        # because Directory.collect() currently loads
-        # conftest.py from sibling directories
-        self.tmpdir = tmpdir.mkdir(name)
-        self.plugins = []
-        self._syspathremove = []
-        self.chdir() # always chdir
-        self.request.addfinalizer(self.finalize)
-
-    def __repr__(self):
-        return "<TmpTestdir %r>" % (self.tmpdir,)
-
-    def finalize(self):
-        for p in self._syspathremove:
-            py.std.sys.path.remove(p)
-        if hasattr(self, '_olddir'):
-            self._olddir.chdir()
-        # delete modules that have been loaded from tmpdir
-        for name, mod in list(sys.modules.items()):
-            if mod:
-                fn = getattr(mod, '__file__', None)
-                if fn and fn.startswith(str(self.tmpdir)):
-                    del sys.modules[name]
-
-    def getreportrecorder(self, obj):
-        if hasattr(obj, 'config'):
-            obj = obj.config
-        if hasattr(obj, 'hook'):
-            obj = obj.hook
-        assert hasattr(obj, '_hookspecs'), obj
-        reprec = ReportRecorder(obj)
-        reprec.hookrecorder = self._pytest.gethookrecorder(obj)
-        reprec.hook = reprec.hookrecorder.hook
-        return reprec
-
-    def chdir(self):
-        old = self.tmpdir.chdir()
-        if not hasattr(self, '_olddir'):
-            self._olddir = old
-
-    def _makefile(self, ext, args, kwargs):
-        items = list(kwargs.items())
-        if args:
-            source = py.builtin._totext("\n").join(
-                map(py.builtin._totext, args)) + py.builtin._totext("\n")
-            basename = self.request.function.__name__
-            items.insert(0, (basename, source))
-        ret = None
-        for name, value in items:
-            p = self.tmpdir.join(name).new(ext=ext)
-            source = py.builtin._totext(py.code.Source(value)).lstrip()
-            p.write(source.encode("utf-8"), "wb")
-            if ret is None:
-                ret = p
-        return ret
-
-
-    def makefile(self, ext, *args, **kwargs):
-        return self._makefile(ext, args, kwargs)
-
-    def makeini(self, source):
-        return self.makefile('cfg', setup=source)
-
-    def makeconftest(self, source):
-        return self.makepyfile(conftest=source)
-
-    def makeini(self, source):
-        return self.makefile('.ini', tox=source)
-
-    def getinicfg(self, source):
-        p = self.makeini(source)
-        return py.iniconfig.IniConfig(p)['pytest']
-
-    def makepyfile(self, *args, **kwargs):
-        return self._makefile('.py', args, kwargs)
-
-    def maketxtfile(self, *args, **kwargs):
-        return self._makefile('.txt', args, kwargs)
-
-    def syspathinsert(self, path=None):
-        if path is None:
-            path = self.tmpdir
-        py.std.sys.path.insert(0, str(path))
-        self._syspathremove.append(str(path))
-
-    def mkdir(self, name):
-        return self.tmpdir.mkdir(name)
-
-    def mkpydir(self, name):
-        p = self.mkdir(name)
-        p.ensure("__init__.py")
-        return p
-
-    Session = Session
-    def getnode(self, config, arg):
-        session = Session(config)
-        assert '::' not in str(arg)
-        p = py.path.local(arg)
-        x = session.fspath.bestrelpath(p)
-        config.hook.pytest_sessionstart(session=session)
-        res = session.perform_collect([x], genitems=False)[0]
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return res
-
-    def getpathnode(self, path):
-        config = self.parseconfigure(path)
-        session = Session(config)
-        x = session.fspath.bestrelpath(path)
-        config.hook.pytest_sessionstart(session=session)
-        res = session.perform_collect([x], genitems=False)[0]
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return res
-
-    def genitems(self, colitems):
-        session = colitems[0].session
-        result = []
-        for colitem in colitems:
-            result.extend(session.genitems(colitem))
-        return result
-
-    def runitem(self, source):
-        # used from runner functional tests
-        item = self.getitem(source)
-        # the test class where we are called from wants to provide the runner
-        testclassinstance = py.builtin._getimself(self.request.function)
-        runner = testclassinstance.getrunner()
-        return runner(item)
-
-    def inline_runsource(self, source, *cmdlineargs):
-        p = self.makepyfile(source)
-        l = list(cmdlineargs) + [p]
-        return self.inline_run(*l)
-
-    def inline_runsource1(self, *args):
-        args = list(args)
-        source = args.pop()
-        p = self.makepyfile(source)
-        l = list(args) + [p]
-        reprec = self.inline_run(*l)
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3, reports # setup/call/teardown
-        return reports[1]
-
-    def inline_genitems(self, *args):
-        return self.inprocess_run(list(args) + ['--collectonly'])
-
-    def inline_run(self, *args):
-        items, rec = self.inprocess_run(args)
-        return rec
-
-    def inprocess_run(self, args, plugins=None):
-        rec = []
-        items = []
-        class Collect:
-            def pytest_configure(x, config):
-                rec.append(self.getreportrecorder(config))
-            def pytest_itemcollected(self, item):
-                items.append(item)
-        if not plugins:
-            plugins = []
-        plugins.append(Collect())
-        ret = self.pytestmain(list(args), plugins=[Collect()])
-        reprec = rec[0]
-        reprec.ret = ret
-        assert len(rec) == 1
-        return items, reprec
-
-    def parseconfig(self, *args):
-        args = [str(x) for x in args]
-        for x in args:
-            if str(x).startswith('--basetemp'):
-                break
-        else:
-            args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        import _pytest.core
-        config = _pytest.core._prepareconfig(args, self.plugins)
-        # the in-process pytest invocation needs to avoid leaking FDs
-        # so we register a "reset_capturings" callmon the capturing manager
-        # and make sure it gets called
-        config._cleanup.append(
-            config.pluginmanager.getplugin("capturemanager").reset_capturings)
-        import _pytest.config
-        self.request.addfinalizer(
-            lambda: _pytest.config.pytest_unconfigure(config))
-        return config
-
-    def parseconfigure(self, *args):
-        config = self.parseconfig(*args)
-        config.pluginmanager.do_configure(config)
-        self.request.addfinalizer(lambda:
-        config.pluginmanager.do_unconfigure(config))
-        return config
-
-    def getitem(self,  source, funcname="test_func"):
-        for item in self.getitems(source):
-            if item.name == funcname:
-                return item
-        assert 0, "%r item not found in module:\n%s" %(funcname, source)
-
-    def getitems(self,  source):
-        modcol = self.getmodulecol(source)
-        return self.genitems([modcol])
-
-    def getmodulecol(self,  source, configargs=(), withinit=False):
-        kw = {self.request.function.__name__: py.code.Source(source).strip()}
-        path = self.makepyfile(**kw)
-        if withinit:
-            self.makepyfile(__init__ = "#")
-        self.config = config = self.parseconfigure(path, *configargs)
-        node = self.getnode(config, path)
-        return node
-
-    def collect_by_name(self, modcol, name):
-        for colitem in modcol._memocollect():
-            if colitem.name == name:
-                return colitem
-
-    def popen(self, cmdargs, stdout, stderr, **kw):
-        env = os.environ.copy()
-        env['PYTHONPATH'] = os.pathsep.join(filter(None, [
-            str(os.getcwd()), env.get('PYTHONPATH', '')]))
-        kw['env'] = env
-        #print "env", env
-        return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
-
-    def pytestmain(self, *args, **kwargs):
-        class ResetCapturing:
-            @pytest.mark.trylast
-            def pytest_unconfigure(self, config):
-                capman = config.pluginmanager.getplugin("capturemanager")
-                capman.reset_capturings()
-        plugins = kwargs.setdefault("plugins", [])
-        rc = ResetCapturing()
-        plugins.append(rc)
-        return pytest.main(*args, **kwargs)
-
-    def run(self, *cmdargs):
-        return self._run(*cmdargs)
-
-    def _run(self, *cmdargs):
-        cmdargs = [str(x) for x in cmdargs]
-        p1 = self.tmpdir.join("stdout")
-        p2 = self.tmpdir.join("stderr")
-        print_("running", cmdargs, "curdir=", py.path.local())
-        f1 = p1.open("wb")
-        f2 = p2.open("wb")
-        now = time.time()
-        popen = self.popen(cmdargs, stdout=f1, stderr=f2,
-            close_fds=(sys.platform != "win32"))
-        ret = popen.wait()
-        f1.close()
-        f2.close()
-        out = p1.read("rb")
-        out = getdecoded(out).splitlines()
-        err = p2.read("rb")
-        err = getdecoded(err).splitlines()
-        def dump_lines(lines, fp):
-            try:
-                for line in lines:
-                    py.builtin.print_(line, file=fp)
-            except UnicodeEncodeError:
-                print("couldn't print to %s because of encoding" % (fp,))
-        dump_lines(out, sys.stdout)
-        dump_lines(err, sys.stderr)
-        return RunResult(ret, out, err, time.time()-now)
-
-    def runpybin(self, scriptname, *args):
-        fullargs = self._getpybinargs(scriptname) + args
-        return self.run(*fullargs)
-
-    def _getpybinargs(self, scriptname):
-        if not self.request.config.getvalue("notoolsonpath"):
-            # XXX we rely on script refering to the correct environment
-            # we cannot use "(py.std.sys.executable,script)"
-            # becaue on windows the script is e.g. a py.test.exe
-            return (py.std.sys.executable, _pytest_fullpath,)
-        else:
-            py.test.skip("cannot run %r with --no-tools-on-path" % scriptname)
-
-    def runpython(self, script, prepend=True):
-        if prepend:
-            s = self._getsysprepend()
-            if s:
-                script.write(s + "\n" + script.read())
-        return self.run(sys.executable, script)
-
-    def _getsysprepend(self):
-        if self.request.config.getvalue("notoolsonpath"):
-            s = "import sys;sys.path.insert(0,%r);" % str(py._pydir.dirpath())
-        else:
-            s = ""
-        return s
-
-    def runpython_c(self, command):
-        command = self._getsysprepend() + command
-        return self.run(py.std.sys.executable, "-c", command)
-
-    def runpytest(self, *args):
-        p = py.path.local.make_numbered_dir(prefix="runpytest-",
-            keep=None, rootdir=self.tmpdir)
-        args = ('--basetemp=%s' % p, ) + args
-        #for x in args:
-        #    if '--confcutdir' in str(x):
-        #        break
-        #else:
-        #    pass
-        #    args = ('--confcutdir=.',) + args
-        plugins = [x for x in self.plugins if isinstance(x, str)]
-        if plugins:
-            args = ('-p', plugins[0]) + args
-        return self.runpybin("py.test", *args)
-
-    def spawn_pytest(self, string, expect_timeout=10.0):
-        if self.request.config.getvalue("notoolsonpath"):
-            py.test.skip("--no-tools-on-path prevents running pexpect-spawn tests")
-        basetemp = self.tmpdir.mkdir("pexpect")
-        invoke = " ".join(map(str, self._getpybinargs("py.test")))
-        cmd = "%s --basetemp=%s %s" % (invoke, basetemp, string)
-        return self.spawn(cmd, expect_timeout=expect_timeout)
-
-    def spawn(self, cmd, expect_timeout=10.0):
-        pexpect = py.test.importorskip("pexpect", "2.4")
-        if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
-            pytest.skip("pypy-64 bit not supported")
-        if sys.platform == "darwin":
-            pytest.xfail("pexpect does not work reliably on darwin?!")
-        logfile = self.tmpdir.join("spawn.out")
-        child = pexpect.spawn(cmd, logfile=logfile.open("w"))
-        child.timeout = expect_timeout
-        return child
-
-def getdecoded(out):
-        try:
-            return out.decode("utf-8")
-        except UnicodeDecodeError:
-            return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
-                    py.io.saferepr(out),)
-
-class ReportRecorder(object):
-    def __init__(self, hook):
-        self.hook = hook
-        self.pluginmanager = hook._pm
-        self.pluginmanager.register(self)
-
-    def getcall(self, name):
-        return self.hookrecorder.getcall(name)
-
-    def popcall(self, name):
-        return self.hookrecorder.popcall(name)
-
-    def getcalls(self, names):
-        """ return list of ParsedCall instances matching the given eventname. """
-        return self.hookrecorder.getcalls(names)
-
-    # functionality for test reports
-
-    def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
-        return [x.report for x in self.getcalls(names)]
-
-    def matchreport(self, inamepart="",
-        names="pytest_runtest_logreport pytest_collectreport", when=None):
-        """ return a testreport whose dotted import path matches """
-        l = []
-        for rep in self.getreports(names=names):
-            try:
-                if not when and rep.when != "call" and rep.passed:
-                    # setup/teardown passing reports - let's ignore those
-                    continue
-            except AttributeError:
-                pass
-            if when and getattr(rep, 'when', None) != when:
-                continue
-            if not inamepart or inamepart in rep.nodeid.split("::"):
-                l.append(rep)
-        if not l:
-            raise ValueError("could not find test report matching %r: no test reports at all!" %
-                (inamepart,))
-        if len(l) > 1:
-            raise ValueError("found more than one testreport matching %r: %s" %(
-                             inamepart, l))
-        return l[0]
-
-    def getfailures(self, names='pytest_runtest_logreport pytest_collectreport'):
-        return [rep for rep in self.getreports(names) if rep.failed]
-
-    def getfailedcollections(self):
-        return self.getfailures('pytest_collectreport')
-
-    def listoutcomes(self):
-        passed = []
-        skipped = []
-        failed = []
-        for rep in self.getreports("pytest_runtest_logreport"):
-            if rep.passed:
-                if rep.when == "call":
-                    passed.append(rep)
-            elif rep.skipped:
-                skipped.append(rep)
-            elif rep.failed:
-                failed.append(rep)
-        return passed, skipped, failed
-
-    def countoutcomes(self):
-        return [len(x) for x in self.listoutcomes()]
-
-    def assertoutcome(self, passed=0, skipped=0, failed=0):
-        realpassed, realskipped, realfailed = self.listoutcomes()
-        assert passed == len(realpassed)
-        assert skipped == len(realskipped)
-        assert failed == len(realfailed)
-
-    def clear(self):
-        self.hookrecorder.calls[:] = []
-
-    def unregister(self):
-        self.pluginmanager.unregister(self)
-        self.hookrecorder.finish_recording()
-
-class LineComp:
-    def __init__(self):
-        self.stringio = py.io.TextIO()
-
-    def assert_contains_lines(self, lines2):
-        """ assert that lines2 are contained (linearly) in lines1.
-            return a list of extralines found.
-        """
-        __tracebackhide__ = True
-        val = self.stringio.getvalue()
-        self.stringio.truncate(0)
-        self.stringio.seek(0)
-        lines1 = val.split("\n")
-        return LineMatcher(lines1).fnmatch_lines(lines2)
-
-class LineMatcher:
-    def __init__(self,  lines):
-        self.lines = lines
-
-    def str(self):
-        return "\n".join(self.lines)
-
-    def _getlines(self, lines2):
-        if isinstance(lines2, str):
-            lines2 = py.code.Source(lines2)
-        if isinstance(lines2, py.code.Source):
-            lines2 = lines2.strip().lines
-        return lines2
-
-    def fnmatch_lines_random(self, lines2):
-        lines2 = self._getlines(lines2)
-        for line in lines2:
-            for x in self.lines:
-                if line == x or fnmatch(x, line):
-                    print_("matched: ", repr(line))
-                    break
-            else:
-                raise ValueError("line %r not found in output" % line)
-
-    def fnmatch_lines(self, lines2):
-        def show(arg1, arg2):
-            py.builtin.print_(arg1, arg2, file=py.std.sys.stderr)
-        lines2 = self._getlines(lines2)
-        lines1 = self.lines[:]
-        nextline = None
-        extralines = []
-        __tracebackhide__ = True
-        for line in lines2:
-            nomatchprinted = False
-            while lines1:
-                nextline = lines1.pop(0)
-                if line == nextline:
-                    show("exact match:", repr(line))
-                    break
-                elif fnmatch(nextline, line):
-                    show("fnmatch:", repr(line))
-                    show("   with:", repr(nextline))
-                    break
-                else:
-                    if not nomatchprinted:
-                        show("nomatch:", repr(line))
-                        nomatchprinted = True
-                    show("    and:", repr(nextline))
-                extralines.append(nextline)
-            else:
-                py.test.fail("remains unmatched: %r, see stderr" % (line,))
diff --git a/_pytest/python.py b/_pytest/python.py
deleted file mode 100644
--- a/_pytest/python.py
+++ /dev/null
@@ -1,1009 +0,0 @@
-""" Python test discovery, setup and run of test functions. """
-import py
-import inspect
-import sys
-import pytest
-from py._code.code import TerminalRepr
-from _pytest.monkeypatch import monkeypatch
-
-import _pytest
-cutdir = py.path.local(_pytest.__file__).dirpath()
-
-def pytest_addoption(parser):
-    group = parser.getgroup("general")
-    group.addoption('--funcargs',
-               action="store_true", dest="showfuncargs", default=False,
-               help="show available function arguments, sorted by plugin")
-    parser.addini("python_files", type="args",
-        default=('test_*.py', '*_test.py'),
-        help="glob-style file patterns for Python test module discovery")
-    parser.addini("python_classes", type="args", default=("Test",),
-        help="prefixes for Python test class discovery")
-    parser.addini("python_functions", type="args", default=("test",),
-        help="prefixes for Python test function and method discovery")
-
-def pytest_cmdline_main(config):
-    if config.option.showfuncargs:
-        showfuncargs(config)
-        return 0
-
-
-def pytest_generate_tests(metafunc):
-    try:
-        param = metafunc.function.parametrize
-    except AttributeError:
-        return
-    for p in param:
-        metafunc.parametrize(*p.args, **p.kwargs)
-
-def pytest_configure(config):
-    config.addinivalue_line("markers",
-        "parametrize(argnames, argvalues): call a test function multiple "
-        "times passing in multiple different argument value sets. Example: "
-        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
-        "test function, one with arg1=1 and another with arg1=2."
-    )
-
-
- at pytest.mark.trylast
-def pytest_namespace():
-    raises.Exception = pytest.fail.Exception
-    return {
-        'raises' : raises,
-        'collect': {
-        'Module': Module, 'Class': Class, 'Instance': Instance,
-        'Function': Function, 'Generator': Generator,
-        '_fillfuncargs': fillfuncargs}
-    }
-
-def pytest_funcarg__pytestconfig(request):
-    """ the pytest config object with access to command line opts."""
-    return request.config
-
-def pytest_pyfunc_call(__multicall__, pyfuncitem):
-    if not __multicall__.execute():
-        testfunction = pyfuncitem.obj
-        if pyfuncitem._isyieldedfunction():
-            testfunction(*pyfuncitem._args)
-        else:
-            funcargs = pyfuncitem.funcargs
-            testfunction(**funcargs)
-
-def pytest_collect_file(path, parent):
-    ext = path.ext
-    pb = path.purebasename
-    if ext == ".py":
-        if not parent.session.isinitpath(path):
-            for pat in parent.config.getini('python_files'):
-                if path.fnmatch(pat):
-                    break
-            else:
-               return
-        return parent.ihook.pytest_pycollect_makemodule(
-            path=path, parent=parent)
-
-def pytest_pycollect_makemodule(path, parent):
-    return Module(path, parent)
-
-def pytest_pycollect_makeitem(__multicall__, collector, name, obj):
-    res = __multicall__.execute()
-    if res is not None:
-        return res
-    if inspect.isclass(obj):
-        #if hasattr(collector.obj, 'unittest'):
-        #    return # we assume it's a mixin class for a TestCase derived one
-        if collector.classnamefilter(name):
-            if not hasinit(obj):
-                Class = collector._getcustomclass("Class")
-                return Class(name, parent=collector)
-    elif collector.funcnamefilter(name) and hasattr(obj, '__call__'):
-        if is_generator(obj):
-            return Generator(name, parent=collector)
-        else:
-            return collector._genfunctions(name, obj)
-
-def is_generator(func):
-    try:
-        return py.code.getrawcode(func).co_flags & 32 # generator function
-    except AttributeError: # builtin functions have no bytecode
-        # assume them to not be generators
-        return False
-
-class PyobjMixin(object):
-    def obj():
-        def fget(self):
-            try:
-                return self._obj
-            except AttributeError:
-                self._obj = obj = self._getobj()
-                return obj
-        def fset(self, value):
-            self._obj = value
-        return property(fget, fset, None, "underlying python object")
-    obj = obj()
-
-    def _getobj(self):
-        return getattr(self.parent.obj, self.name)
-
-    def getmodpath(self, stopatmodule=True, includemodule=False):
-        """ return python path relative to the containing module. """
-        chain = self.listchain()
-        chain.reverse()
-        parts = []
-        for node in chain:
-            if isinstance(node, Instance):
-                continue
-            name = node.name
-            if isinstance(node, Module):
-                assert name.endswith(".py")
-                name = name[:-3]
-                if stopatmodule:
-                    if includemodule:
-                        parts.append(name)
-                    break
-            parts.append(name)
-        parts.reverse()
-        s = ".".join(parts)
-        return s.replace(".[", "[")
-
-    def _getfslineno(self):
-        try:
-            return self._fslineno
-        except AttributeError:
-            pass
-        obj = self.obj
-        # xxx let decorators etc specify a sane ordering
-        if hasattr(obj, 'place_as'):
-            obj = obj.place_as
-
-        self._fslineno = py.code.getfslineno(obj)
-        assert isinstance(self._fslineno[1], int), obj
-        return self._fslineno
-
-    def reportinfo(self):
-        # XXX caching?
-        obj = self.obj
-        if hasattr(obj, 'compat_co_firstlineno'):
-            # nose compatibility
-            fspath = sys.modules[obj.__module__].__file__
-            if fspath.endswith(".pyc"):
-                fspath = fspath[:-1]
-            #assert 0
-            #fn = inspect.getsourcefile(obj) or inspect.getfile(obj)
-            lineno = obj.compat_co_firstlineno
-            modpath = obj.__module__
-        else:
-            fspath, lineno = self._getfslineno()
-            modpath = self.getmodpath()
-        assert isinstance(lineno, int)
-        return fspath, lineno, modpath
-
-class PyCollectorMixin(PyobjMixin, pytest.Collector):
-
-    def funcnamefilter(self, name):
-        for prefix in self.config.getini("python_functions"):
-            if name.startswith(prefix):
-                return True
-
-    def classnamefilter(self, name):
-        for prefix in self.config.getini("python_classes"):
-            if name.startswith(prefix):
-                return True
-
-    def collect(self):
-        # NB. we avoid random getattrs and peek in the __dict__ instead
-        # (XXX originally introduced from a PyPy need, still true?)
-        dicts = [getattr(self.obj, '__dict__', {})]
-        for basecls in inspect.getmro(self.obj.__class__):
-            dicts.append(basecls.__dict__)
-        seen = {}
-        l = []
-        for dic in dicts:
-            for name, obj in dic.items():
-                if name in seen:
-                    continue
-                seen[name] = True
-                if name[0] != "_":
-                    res = self.makeitem(name, obj)
-                    if res is None:
-                        continue
-                    if not isinstance(res, list):
-                        res = [res]
-                    l.extend(res)
-        l.sort(key=lambda item: item.reportinfo()[:2])
-        return l
-
-    def makeitem(self, name, obj):
-        return self.ihook.pytest_pycollect_makeitem(
-            collector=self, name=name, obj=obj)
-
-    def _genfunctions(self, name, funcobj):
-        module = self.getparent(Module).obj
-        clscol = self.getparent(Class)
-        cls = clscol and clscol.obj or None
-        transfer_markers(funcobj, cls, module)
-        metafunc = Metafunc(funcobj, config=self.config,
-            cls=cls, module=module)
-        gentesthook = self.config.hook.pytest_generate_tests
-        extra = [module]
-        if cls is not None:
-            extra.append(cls())
-        plugins = self.getplugins() + extra
-        gentesthook.pcall(plugins, metafunc=metafunc)
-        Function = self._getcustomclass("Function")
-        if not metafunc._calls:
-            return Function(name, parent=self)
-        l = []
-        for callspec in metafunc._calls:
-            subname = "%s[%s]" %(name, callspec.id)
-            function = Function(name=subname, parent=self,
-                callspec=callspec, callobj=funcobj, keywords={callspec.id:True})
-            l.append(function)
-        return l
-
-def transfer_markers(funcobj, cls, mod):
-    # XXX this should rather be code in the mark plugin or the mark
-    # plugin should merge with the python plugin.
-    for holder in (cls, mod):
-        try:
-            pytestmark = holder.pytestmark
-        except AttributeError:
-            continue
-        if isinstance(pytestmark, list):
-            for mark in pytestmark:
-                mark(funcobj)
-        else:
-            pytestmark(funcobj)
-
-class Module(pytest.File, PyCollectorMixin):
-    def _getobj(self):
-        return self._memoizedcall('_obj', self._importtestmodule)
-
-    def _importtestmodule(self):
-        # we assume we are only called once per module
-        try:
-            mod = self.fspath.pyimport(ensuresyspath=True)
-        except SyntaxError:
-            excinfo = py.code.ExceptionInfo()
-            raise self.CollectError(excinfo.getrepr(style="short"))
-        except self.fspath.ImportMismatchError:
-            e = sys.exc_info()[1]
-            raise self.CollectError(
-                "import file mismatch:\n"
-                "imported module %r has this __file__ attribute:\n"
-                "  %s\n"
-                "which is not the same as the test file we want to collect:\n"
-                "  %s\n"
-                "HINT: remove __pycache__ / .pyc files and/or use a "
-                "unique basename for your test file modules"
-                 % e.args
-            )
-        #print "imported test module", mod
-        self.config.pluginmanager.consider_module(mod)
-        return mod
-
-    def setup(self):
-        if hasattr(self.obj, 'setup_module'):
-            #XXX: nose compat hack, move to nose plugin
-            # if it takes a positional arg, its probably a pytest style one
-            # so we pass the current module object
-            if inspect.getargspec(self.obj.setup_module)[0]:
-                self.obj.setup_module(self.obj)
-            else:
-                self.obj.setup_module()
-
-    def teardown(self):
-        if hasattr(self.obj, 'teardown_module'):
-            #XXX: nose compat hack, move to nose plugin
-            # if it takes a positional arg, its probably a py.test style one
-            # so we pass the current module object
-            if inspect.getargspec(self.obj.teardown_module)[0]:
-                self.obj.teardown_module(self.obj)
-            else:
-                self.obj.teardown_module()
-
-class Class(PyCollectorMixin, pytest.Collector):
-
-    def collect(self):
-        return [self._getcustomclass("Instance")(name="()", parent=self)]
-
-    def setup(self):
-        setup_class = getattr(self.obj, 'setup_class', None)
-        if setup_class is not None:
-            setup_class = getattr(setup_class, 'im_func', setup_class)
-            setup_class(self.obj)
-
-    def teardown(self):
-        teardown_class = getattr(self.obj, 'teardown_class', None)
-        if teardown_class is not None:
-            teardown_class = getattr(teardown_class, 'im_func', teardown_class)
-            teardown_class(self.obj)
-
-class Instance(PyCollectorMixin, pytest.Collector):
-    def _getobj(self):
-        return self.parent.obj()
-
-    def newinstance(self):
-        self.obj = self._getobj()
-        return self.obj
-
-class FunctionMixin(PyobjMixin):
-    """ mixin for the code common to Function and Generator.
-    """
-    def setup(self):
-        """ perform setup for this test function. """
-        if hasattr(self, '_preservedparent'):
-            obj = self._preservedparent
-        elif isinstance(self.parent, Instance):
-            obj = self.parent.newinstance()
-            self.obj = self._getobj()
-        else:
-            obj = self.parent.obj
-        if inspect.ismethod(self.obj):
-            name = 'setup_method'
-        else:
-            name = 'setup_function'
-        setup_func_or_method = getattr(obj, name, None)
-        if setup_func_or_method is not None:
-            setup_func_or_method(self.obj)
-
-    def teardown(self):
-        """ perform teardown for this test function. """
-        if inspect.ismethod(self.obj):
-            name = 'teardown_method'
-        else:
-            name = 'teardown_function'
-        obj = self.parent.obj
-        teardown_func_or_meth = getattr(obj, name, None)
-        if teardown_func_or_meth is not None:
-            teardown_func_or_meth(self.obj)
-
-    def _prunetraceback(self, excinfo):
-        if hasattr(self, '_obj') and not self.config.option.fulltrace:
-            code = py.code.Code(self.obj)
-            path, firstlineno = code.path, code.firstlineno
-            traceback = excinfo.traceback
-            ntraceback = traceback.cut(path=path, firstlineno=firstlineno)
-            if ntraceback == traceback:
-                ntraceback = ntraceback.cut(path=path)
-                if ntraceback == traceback:
-                    ntraceback = ntraceback.cut(excludepath=cutdir)
-            excinfo.traceback = ntraceback.filter()
-
-    def _repr_failure_py(self, excinfo, style="long"):
-        if excinfo.errisinstance(FuncargRequest.LookupError):
-            fspath, lineno, msg = self.reportinfo()
-            lines, _ = inspect.getsourcelines(self.obj)
-            for i, line in enumerate(lines):
-                if line.strip().startswith('def'):
-                    return FuncargLookupErrorRepr(fspath, lineno,
-            lines[:i+1], str(excinfo.value))
-        if excinfo.errisinstance(pytest.fail.Exception):
-            if not excinfo.value.pytrace:
-                return str(excinfo.value)
-        return super(FunctionMixin, self)._repr_failure_py(excinfo,
-            style=style)
-
-    def repr_failure(self, excinfo, outerr=None):
-        assert outerr is None, "XXX outerr usage is deprecated"
-        return self._repr_failure_py(excinfo,
-            style=self.config.option.tbstyle)
-
-class FuncargLookupErrorRepr(TerminalRepr):
-    def __init__(self, filename, firstlineno, deflines, errorstring):
-        self.deflines = deflines
-        self.errorstring = errorstring
-        self.filename = filename
-        self.firstlineno = firstlineno
-
-    def toterminal(self, tw):
-        tw.line()
-        for line in self.deflines:
-            tw.line("    " + line.strip())
-        for line in self.errorstring.split("\n"):
-            tw.line("        " + line.strip(), red=True)
-        tw.line()
-        tw.line("%s:%d" % (self.filename, self.firstlineno+1))
-
-
-class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
-    def collect(self):
-        # test generators are seen as collectors but they also
-        # invoke setup/teardown on popular request
-        # (induced by the common "test_*" naming shared with normal tests)
-        self.session._setupstate.prepare(self)
-        # see FunctionMixin.setup and test_setupstate_is_preserved_134
-        self._preservedparent = self.parent.obj
-        l = []
-        seen = {}
-        for i, x in enumerate(self.obj()):
-            name, call, args = self.getcallargs(x)
-            if not py.builtin.callable(call):
-                raise TypeError("%r yielded non callable test %r" %(self.obj, call,))
-            if name is None:
-                name = "[%d]" % i
-            else:
-                name = "['%s']" % name
-            if name in seen:
-                raise ValueError("%r generated tests with non-unique name %r" %(self, name))
-            seen[name] = True
-            l.append(self.Function(name, self, args=args, callobj=call))
-        return l
-
-    def getcallargs(self, obj):
-        if not isinstance(obj, (tuple, list)):
-            obj = (obj,)
-        # explict naming
-        if isinstance(obj[0], py.builtin._basestring):
-            name = obj[0]
-            obj = obj[1:]
-        else:
-            name = None
-        call, args = obj[0], obj[1:]
-        return name, call, args
-
-
-#
-#  Test Items
-#
-_dummy = object()
-class Function(FunctionMixin, pytest.Item):
-    """ a Function Item is responsible for setting up
-        and executing a Python callable test object.
-    """
-    _genid = None
-    def __init__(self, name, parent=None, args=None, config=None,
-                 callspec=None, callobj=_dummy, keywords=None, session=None):
-        super(Function, self).__init__(name, parent,
-            config=config, session=session)
-        self._args = args
-        if self._isyieldedfunction():
-            assert not callspec, (
-                "yielded functions (deprecated) cannot have funcargs")
-        else:
-            if callspec is not None:
-                self.callspec = callspec
-                self.funcargs = callspec.funcargs or {}
-                self._genid = callspec.id
-                if hasattr(callspec, "param"):
-                    self._requestparam = callspec.param
-            else:
-                self.funcargs = {}
-        if callobj is not _dummy:
-            self._obj = callobj
-        self.function = getattr(self.obj, 'im_func', self.obj)
-        self.keywords.update(py.builtin._getfuncdict(self.obj) or {})
-        if keywords:
-            self.keywords.update(keywords)
-
-    def _getobj(self):
-        name = self.name
-        i = name.find("[") # parametrization
-        if i != -1:
-            name = name[:i]
-        return getattr(self.parent.obj, name)
-
-    def _isyieldedfunction(self):
-        return self._args is not None
-
-    def runtest(self):
-        """ execute the underlying test function. """
-        self.ihook.pytest_pyfunc_call(pyfuncitem=self)
-
-    def setup(self):
-        super(Function, self).setup()
-        if hasattr(self, 'funcargs'):
-            fillfuncargs(self)
-
-    def __eq__(self, other):
-        try:
-            return (self.name == other.name and
-                    self._args == other._args and
-                    self.parent == other.parent and
-                    self.obj == other.obj and
-                    getattr(self, '_genid', None) ==
-                    getattr(other, '_genid', None)
-            )
-        except AttributeError:
-            pass
-        return False
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __hash__(self):
-        return hash((self.parent, self.name))
-
-def hasinit(obj):
-    init = getattr(obj, '__init__', None)
-    if init:
-        if init != object.__init__:
-            return True
-
-
-def getfuncargnames(function, startindex=None):
-    # XXX merge with main.py's varnames
-    argnames = py.std.inspect.getargs(py.code.getrawcode(function))[0]
-    if startindex is None:
-        startindex = py.std.inspect.ismethod(function) and 1 or 0
-    defaults = getattr(function, 'func_defaults',
-                       getattr(function, '__defaults__', None)) or ()
-    numdefaults = len(defaults)
-    if numdefaults:
-        return argnames[startindex:-numdefaults]
-    return argnames[startindex:]
-
-def fillfuncargs(function):
-    """ fill missing funcargs. """
-    request = FuncargRequest(pyfuncitem=function)
-    request._fillfuncargs()
-
-_notexists = object()
-
-class CallSpec2(object):
-    def __init__(self, metafunc):
-        self.metafunc = metafunc
-        self.funcargs = {}
-        self._idlist = []
-        self.params = {}
-        self._globalid = _notexists
-        self._globalid_args = set()
-        self._globalparam = _notexists
-
-    def copy(self, metafunc):
-        cs = CallSpec2(self.metafunc)
-        cs.funcargs.update(self.funcargs)
-        cs.params.update(self.params)
-        cs._idlist = list(self._idlist)
-        cs._globalid = self._globalid
-        cs._globalid_args = self._globalid_args
-        cs._globalparam = self._globalparam
-        return cs
-
-    def _checkargnotcontained(self, arg):
-        if arg in self.params or arg in self.funcargs:
-            raise ValueError("duplicate %r" %(arg,))
-
-    def getparam(self, name):
-        try:
-            return self.params[name]
-        except KeyError:
-            if self._globalparam is _notexists:
-                raise ValueError(name)
-            return self._globalparam
-
-    @property
-    def id(self):
-        return "-".join(map(str, filter(None, self._idlist)))
-
-    def setmulti(self, valtype, argnames, valset, id):
-        for arg,val in zip(argnames, valset):
-            self._checkargnotcontained(arg)
-            getattr(self, valtype)[arg] = val
-        self._idlist.append(id)
-
-    def setall(self, funcargs, id, param):
-        for x in funcargs:
-            self._checkargnotcontained(x)
-        self.funcargs.update(funcargs)
-        if id is not _notexists:
-            self._idlist.append(id)
-        if param is not _notexists:
-            assert self._globalparam is _notexists
-            self._globalparam = param
-
-
-class Metafunc:
-    def __init__(self, function, config=None, cls=None, module=None):
-        self.config = config
-        self.module = module
-        self.function = function
-        self.funcargnames = getfuncargnames(function,
-                                            startindex=int(cls is not None))
-        self.cls = cls
-        self.module = module
-        self._calls = []
-        self._ids = py.builtin.set()
-
-    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
-        """ Add new invocations to the underlying test function using the list
-        of argvalues for the given argnames.  Parametrization is performed
-        during the collection phase.  If you need to setup expensive resources
-        you may pass indirect=True and implement a funcarg factory which can
-        perform the expensive setup just before a test is actually run.
-
-        :arg argnames: an argument name or a list of argument names
-
-        :arg argvalues: a list of values for the argname or a list of tuples of
-            values for the list of argument names.
-
-        :arg indirect: if True each argvalue corresponding to an argument will
-            be passed as request.param to its respective funcarg factory so
-            that it can perform more expensive setups during the setup phase of
-            a test rather than at collection time.
-
-        :arg ids: list of string ids each corresponding to the argvalues so
-            that they are part of the test id. If no ids are provided they will
-            be generated automatically from the argvalues.
-        """
-        if not isinstance(argnames, (tuple, list)):
-            argnames = (argnames,)
-            argvalues = [(val,) for val in argvalues]
-        if not indirect:
-            #XXX should we also check for the opposite case?
-            for arg in argnames:
-                if arg not in self.funcargnames:
-                    raise ValueError("%r has no argument %r" %(self.function, arg))
-        valtype = indirect and "params" or "funcargs"
-        if not ids:
-            idmaker = IDMaker()
-            ids = list(map(idmaker, argvalues))
-        newcalls = []
-        for callspec in self._calls or [CallSpec2(self)]:
-            for i, valset in enumerate(argvalues):
-                assert len(valset) == len(argnames)
-                newcallspec = callspec.copy(self)
-                newcallspec.setmulti(valtype, argnames, valset, ids[i])
-                newcalls.append(newcallspec)
-        self._calls = newcalls
-
-    def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ (deprecated, use parametrize) Add a new call to the underlying
-        test function during the collection phase of a test run.  Note that
-        request.addcall() is called during the test collection phase prior and
-        independently to actual test execution.  You should only use addcall()
-        if you need to specify multiple arguments of a test function.
-
-        :arg funcargs: argument keyword dictionary used when invoking
-            the test function.
-
-        :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` an automatic unique id will be generated.
-
-        :arg param: a parameter which will be exposed to a later funcarg factory
-            invocation through the ``request.param`` attribute.
-        """
-        assert funcargs is None or isinstance(funcargs, dict)
-        if funcargs is not None:
-            for name in funcargs:
-                if name not in self.funcargnames:
-                    pytest.fail("funcarg %r not used in this function." % name)
-        else:
-            funcargs = {}
-        if id is None:
-            raise ValueError("id=None not allowed")
-        if id is _notexists:
-            id = len(self._calls)
-        id = str(id)
-        if id in self._ids:
-            raise ValueError("duplicate id %r" % id)
-        self._ids.add(id)
-
-        cs = CallSpec2(self)
-        cs.setall(funcargs, id, param)
-        self._calls.append(cs)
-
-class IDMaker:
-    def __init__(self):
-        self.counter = 0
-    def __call__(self, valset):
-        l = []
-        for val in valset:
-            if not isinstance(val, (int, str)):
-                val = "."+str(self.counter)
-            self.counter += 1
-            l.append(str(val))
-        return "-".join(l)
-
-class FuncargRequest:
-    """ A request for function arguments from a test function.
-
-        Note that there is an optional ``param`` attribute in case
-        there was an invocation to metafunc.addcall(param=...).
-        If no such call was done in a ``pytest_generate_tests``
-        hook, the attribute will not be present.
-    """
-    _argprefix = "pytest_funcarg__"
-    _argname = None
-
-    class LookupError(LookupError):
-        """ error on performing funcarg request. """
-
-    def __init__(self, pyfuncitem):
-        self._pyfuncitem = pyfuncitem
-        if hasattr(pyfuncitem, '_requestparam'):
-            self.param = pyfuncitem._requestparam
-        extra = [obj for obj in (self.module, self.instance) if obj]
-        self._plugins = pyfuncitem.getplugins() + extra
-        self._funcargs  = self._pyfuncitem.funcargs.copy()
-        self._name2factory = {}
-        self._currentarg = None
-
-    @property
-    def function(self):
-        """ function object of the test invocation. """
-        return self._pyfuncitem.obj
-
-    @property
-    def keywords(self):
-        """ keywords of the test function item.
-
-        .. versionadded:: 2.0
-        """
-        return self._pyfuncitem.keywords
-
-    @property
-    def module(self):
-        """ module where the test function was collected. """
-        return self._pyfuncitem.getparent(pytest.Module).obj
-
-    @property
-    def cls(self):
-        """ class (can be None) where the test function was collected. """
-        clscol = self._pyfuncitem.getparent(pytest.Class)
-        if clscol:
-            return clscol.obj
-    @property
-    def instance(self):
-        """ instance (can be None) on which test function was collected. """
-        return py.builtin._getimself(self.function)
-
-    @property
-    def config(self):
-        """ the pytest config object associated with this request. """
-        return self._pyfuncitem.config
-
-    @property
-    def fspath(self):
-        """ the file system path of the test module which collected this test. """
-        return self._pyfuncitem.fspath
-
-    def _fillfuncargs(self):
-        argnames = getfuncargnames(self.function)
-        if argnames:
-            assert not getattr(self._pyfuncitem, '_args', None), (
-                "yielded functions cannot have funcargs")
-        for argname in argnames:
-            if argname not in self._pyfuncitem.funcargs:
-                self._pyfuncitem.funcargs[argname] = self.getfuncargvalue(argname)
-
-
-    def applymarker(self, marker):
-        """ Apply a marker to a single test function invocation.
-        This method is useful if you don't want to have a keyword/marker
-        on all function invocations.
-
-        :arg marker: a :py:class:`_pytest.mark.MarkDecorator` object
-            created by a call to ``py.test.mark.NAME(...)``.
-        """
-        if not isinstance(marker, py.test.mark.XYZ.__class__):
-            raise ValueError("%r is not a py.test.mark.* object")
-        self._pyfuncitem.keywords[marker.markname] = marker
-
-    def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ Return a testing resource managed by ``setup`` &
-        ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
-        ``teardown`` function will be called so that subsequent calls to
-        ``setup`` would recreate the resource.
-
-        :arg teardown: function receiving a previously setup resource.
-        :arg setup: a no-argument function creating a resource.
-        :arg scope: a string value out of ``function``, ``class``, ``module``
-            or ``session`` indicating the caching lifecycle of the resource.
-        :arg extrakey: added to internal caching key of (funcargname, scope).
-        """
-        if not hasattr(self.config, '_setupcache'):
-            self.config._setupcache = {} # XXX weakref?
-        cachekey = (self._currentarg, self._getscopeitem(scope), extrakey)
-        cache = self.config._setupcache
-        try:
-            val = cache[cachekey]
-        except KeyError:
-            val = setup()
-            cache[cachekey] = val
-            if teardown is not None:
-                def finalizer():
-                    del cache[cachekey]
-                    teardown(val)
-                self._addfinalizer(finalizer, scope=scope)
-        return val
-
-    def getfuncargvalue(self, argname):
-        """ Retrieve a function argument by name for this test
-        function invocation.  This allows one function argument factory
-        to call another function argument factory.  If there are two
-        funcarg factories for the same test function argument the first
-        factory may use ``getfuncargvalue`` to call the second one and
-        do something additional with the resource.
-        """
-        try:
-            return self._funcargs[argname]
-        except KeyError:
-            pass
-        if argname not in self._name2factory:
-            self._name2factory[argname] = self.config.pluginmanager.listattr(
-                    plugins=self._plugins,
-                    attrname=self._argprefix + str(argname)
-            )
-        #else: we are called recursively
-        if not self._name2factory[argname]:
-            self._raiselookupfailed(argname)
-        funcargfactory = self._name2factory[argname].pop()
-        oldarg = self._currentarg
-        mp = monkeypatch()
-        mp.setattr(self, '_currentarg', argname)
-        try:
-            param = self._pyfuncitem.callspec.getparam(argname)
-        except (AttributeError, ValueError):
-            pass
-        else:
-            mp.setattr(self, 'param', param, raising=False)
-        try:
-            self._funcargs[argname] = res = funcargfactory(request=self)
-        finally:
-            mp.undo()
-        return res
-
-    def _getscopeitem(self, scope):
-        if scope == "function":
-            return self._pyfuncitem
-        elif scope == "session":
-            return None
-        elif scope == "class":
-            x = self._pyfuncitem.getparent(pytest.Class)
-            if x is not None:
-                return x
-            scope = "module"
-        if scope == "module":
-            return self._pyfuncitem.getparent(pytest.Module)
-        raise ValueError("unknown finalization scope %r" %(scope,))
-
-    def addfinalizer(self, finalizer):
-        """add finalizer function to be called after test function
-        finished execution. """
-        self._addfinalizer(finalizer, scope="function")
-
-    def _addfinalizer(self, finalizer, scope):
-        colitem = self._getscopeitem(scope)
-        self._pyfuncitem.session._setupstate.addfinalizer(
-            finalizer=finalizer, colitem=colitem)
-
-    def __repr__(self):
-        return "<FuncargRequest for %r>" %(self._pyfuncitem)
-
-    def _raiselookupfailed(self, argname):
-        available = []
-        for plugin in self._plugins:
-            for name in vars(plugin):
-                if name.startswith(self._argprefix):
-                    name = name[len(self._argprefix):]
-                    if name not in available:
-                        available.append(name)
-        fspath, lineno, msg = self._pyfuncitem.reportinfo()
-        msg = "LookupError: no factory found for function argument %r" % (argname,)
-        msg += "\n available funcargs: %s" %(", ".join(available),)
-        msg += "\n use 'py.test --funcargs [testpath]' for help on them."
-        raise self.LookupError(msg)
-
-def showfuncargs(config):
-    from _pytest.main import wrap_session
-    return wrap_session(config, _showfuncargs_main)
-
-def _showfuncargs_main(config, session):
-    session.perform_collect()
-    if session.items:
-        plugins = session.items[0].getplugins()
-    else:
-        plugins = session.getplugins()
-    curdir = py.path.local()
-    tw = py.io.TerminalWriter()
-    verbose = config.getvalue("verbose")
-    for plugin in plugins:
-        available = []
-        for name, factory in vars(plugin).items():
-            if name.startswith(FuncargRequest._argprefix):
-                name = name[len(FuncargRequest._argprefix):]
-                if name not in available:
-                    available.append([name, factory])
-        if available:
-            pluginname = plugin.__name__
-            for name, factory in available:
-                loc = getlocation(factory, curdir)
-                if verbose:
-                    funcargspec = "%s -- %s" %(name, loc,)
-                else:
-                    funcargspec = name
-                tw.line(funcargspec, green=True)
-                doc = factory.__doc__ or ""
-                if doc:
-                    for line in doc.split("\n"):
-                        tw.line("    " + line.strip())
-                else:
-                    tw.line("    %s: no docstring available" %(loc,),
-                        red=True)
-
-def getlocation(function, curdir):
-    import inspect
-    fn = py.path.local(inspect.getfile(function))
-    lineno = py.builtin._getcode(function).co_firstlineno
-    if fn.relto(curdir):
-        fn = fn.relto(curdir)
-    return "%s:%d" %(fn, lineno+1)
-
-# builtin pytest.raises helper
-
-def raises(ExpectedException, *args, **kwargs):
-    """ assert that a code block/function call raises @ExpectedException
-    and raise a failure exception otherwise.
-
-    If using Python 2.5 or above, you may use this function as a
-    context manager::
-
-        >>> with raises(ZeroDivisionError):
-        ...    1/0
-
-    Or you can specify a callable by passing a to-be-called lambda::
-
-        >>> raises(ZeroDivisionError, lambda: 1/0)
-        <ExceptionInfo ...>
-
-    or you can specify an arbitrary callable with arguments::
-
-        >>> def f(x): return 1/x
-        ...
-        >>> raises(ZeroDivisionError, f, 0)
-        <ExceptionInfo ...>
-        >>> raises(ZeroDivisionError, f, x=0)
-        <ExceptionInfo ...>
-
-    A third possibility is to use a string to be executed::
-
-        >>> raises(ZeroDivisionError, "f(0)")
-        <ExceptionInfo ...>
-    """
-    __tracebackhide__ = True
-
-    if not args:
-        return RaisesContext(ExpectedException)
-    elif isinstance(args[0], str):
-        code, = args
-        assert isinstance(code, str)
-        frame = sys._getframe(1)
-        loc = frame.f_locals.copy()
-        loc.update(kwargs)
-        #print "raises frame scope: %r" % frame.f_locals
-        try:
-            code = py.code.Source(code).compile()
-            py.builtin.exec_(code, frame.f_globals, loc)
-            # XXX didn'T mean f_globals == f_locals something special?
-            #     this is destroyed here ...
-        except ExpectedException:
-            return py.code.ExceptionInfo()
-    else:
-        func = args[0]
-        try:
-            func(*args[1:], **kwargs)
-        except ExpectedException:
-            return py.code.ExceptionInfo()
-        k = ", ".join(["%s=%r" % x for x in kwargs.items()])
-        if k:
-            k = ', ' + k
-        expr = '%s(%r%s)' %(getattr(func, '__name__', func), args, k)
-    pytest.fail("DID NOT RAISE")
-
-class RaisesContext(object):
-    def __init__(self, ExpectedException):
-        self.ExpectedException = ExpectedException
-        self.excinfo = None
-
-    def __enter__(self):
-        self.excinfo = object.__new__(py.code.ExceptionInfo)
-        return self.excinfo
-
-    def __exit__(self, *tp):
-        __tracebackhide__ = True
-        if tp[0] is None:
-            pytest.fail("DID NOT RAISE")
-        self.excinfo.__init__(tp)
-        return issubclass(self.excinfo.type, self.ExpectedException)
-
diff --git a/_pytest/recwarn.py b/_pytest/recwarn.py
deleted file mode 100644
--- a/_pytest/recwarn.py
+++ /dev/null
@@ -1,99 +0,0 @@
-""" recording warnings during test function execution. """
-
-import py
-import sys, os
-
-def pytest_funcarg__recwarn(request):
-    """Return a WarningsRecorder instance that provides these methods:
-
-    * ``pop(category=None)``: return last warning matching the category.
-    * ``clear()``: clear list of warnings
-   
-    See http://docs.python.org/library/warnings.html for information
-    on warning categories.
-    """
-    if sys.version_info >= (2,7):
-        import warnings
-        oldfilters = warnings.filters[:]
-        warnings.simplefilter('default')
-        def reset_filters():
-            warnings.filters[:] = oldfilters
-        request.addfinalizer(reset_filters)
-    wrec = WarningsRecorder()
-    request.addfinalizer(wrec.finalize)
-    return wrec
-
-def pytest_namespace():
-    return {'deprecated_call': deprecated_call}
-
-def deprecated_call(func, *args, **kwargs):
-    """ assert that calling ``func(*args, **kwargs)``
-    triggers a DeprecationWarning.
-    """
-    warningmodule = py.std.warnings
-    l = []
-    oldwarn_explicit = getattr(warningmodule, 'warn_explicit')
-    def warn_explicit(*args, **kwargs):
-        l.append(args)
-        oldwarn_explicit(*args, **kwargs)
-    oldwarn = getattr(warningmodule, 'warn')
-    def warn(*args, **kwargs):
-        l.append(args)
-        oldwarn(*args, **kwargs)
-
-    warningmodule.warn_explicit = warn_explicit
-    warningmodule.warn = warn
-    try:
-        ret = func(*args, **kwargs)
-    finally:
-        warningmodule.warn_explicit = warn_explicit
-        warningmodule.warn = warn
-    if not l:
-        #print warningmodule
-        __tracebackhide__ = True
-        raise AssertionError("%r did not produce DeprecationWarning" %(func,))
-    return ret
-
-
-class RecordedWarning:
-    def __init__(self, message, category, filename, lineno, line):
-        self.message = message
-        self.category = category
-        self.filename = filename
-        self.lineno = lineno
-        self.line = line
-
-class WarningsRecorder:
-    def __init__(self):
-        warningmodule = py.std.warnings
-        self.list = []
-        def showwarning(message, category, filename, lineno, line=0):
-            self.list.append(RecordedWarning(
-                message, category, filename, lineno, line))
-            try:
-                self.old_showwarning(message, category,
-                    filename, lineno, line=line)
-            except TypeError:
-                # < python2.6
-                self.old_showwarning(message, category, filename, lineno)
-        self.old_showwarning = warningmodule.showwarning
-        warningmodule.showwarning = showwarning
-
-    def pop(self, cls=Warning):
-        """ pop the first recorded warning, raise exception if not exists."""
-        for i, w in enumerate(self.list):
-            if issubclass(w.category, cls):
-                return self.list.pop(i)
-        __tracebackhide__ = True
-        assert 0, "%r not found in %r" %(cls, self.list)
-
-    #def resetregistry(self):
-    #    import warnings
-    #    warnings.onceregistry.clear()
-    #    warnings.__warningregistry__.clear()
-
-    def clear(self):
-        self.list[:] = []
-
-    def finalize(self):
-        py.std.warnings.showwarning = self.old_showwarning
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
deleted file mode 100644
--- a/_pytest/resultlog.py
+++ /dev/null
@@ -1,98 +0,0 @@
-""" (disabled by default) create result information in a plain text file. """
-
-import py
-
-def pytest_addoption(parser):
-    group = parser.getgroup("terminal reporting", "resultlog plugin options")
-    group.addoption('--resultlog', action="store", dest="resultlog",
-        metavar="path", default=None,
-        help="path for machine-readable result log.")
-
-def pytest_configure(config):
-    resultlog = config.option.resultlog
-    # prevent opening resultlog on slave nodes (xdist)
-    if resultlog and not hasattr(config, 'slaveinput'):
-        logfile = open(resultlog, 'w', 1) # line buffered
-        config._resultlog = ResultLog(config, logfile)
-        config.pluginmanager.register(config._resultlog)
-
-def pytest_unconfigure(config):
-    resultlog = getattr(config, '_resultlog', None)
-    if resultlog:
-        resultlog.logfile.close()
-        del config._resultlog
-        config.pluginmanager.unregister(resultlog)
-
-def generic_path(item):
-    chain = item.listchain()
-    gpath = [chain[0].name]
-    fspath = chain[0].fspath
-    fspart = False
-    for node in chain[1:]:
-        newfspath = node.fspath
-        if newfspath == fspath:
-            if fspart:
-                gpath.append(':')
-                fspart = False
-            else:
-                gpath.append('.')
-        else:
-            gpath.append('/')
-            fspart = True
-        name = node.name
-        if name[0] in '([':
-            gpath.pop()
-        gpath.append(name)
-        fspath = newfspath
-    return ''.join(gpath)
-
-class ResultLog(object):
-    def __init__(self, config, logfile):
-        self.config = config
-        self.logfile = logfile # preferably line buffered
-
-    def write_log_entry(self, testpath, lettercode, longrepr):
-        py.builtin.print_("%s %s" % (lettercode, testpath), file=self.logfile)
-        for line in longrepr.splitlines():
-            py.builtin.print_(" %s" % line, file=self.logfile)
-
-    def log_outcome(self, report, lettercode, longrepr):
-        testpath = getattr(report, 'nodeid', None)
-        if testpath is None:
-            testpath = report.fspath
-        self.write_log_entry(testpath, lettercode, longrepr)
-
-    def pytest_runtest_logreport(self, report):
-        if report.when != "call" and report.passed:
-            return
-        res = self.config.hook.pytest_report_teststatus(report=report)
-        code = res[1]
-        if code == 'x':
-            longrepr = str(report.longrepr)
-        elif code == 'X':
-            longrepr = ''
-        elif report.passed:
-            longrepr = ""
-        elif report.failed:
-            longrepr = str(report.longrepr)
-        elif report.skipped:
-            longrepr = str(report.longrepr[2])
-        self.log_outcome(report, code, longrepr)
-
-    def pytest_collectreport(self, report):
-        if not report.passed:
-            if report.failed:
-                code = "F"
-                longrepr = str(report.longrepr.reprcrash)
-            else:
-                assert report.skipped
-                code = "S"
-                longrepr = "%s:%d: %s" % report.longrepr
-            self.log_outcome(report, code, longrepr)
-
-    def pytest_internalerror(self, excrepr):
-        reprcrash = getattr(excrepr, 'reprcrash', None)
-        path = getattr(reprcrash, "path", None)
-        if path is None:
-            path = "cwd:%s" % py.path.local()
-        self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
deleted file mode 100644
--- a/_pytest/runner.py
+++ /dev/null
@@ -1,432 +0,0 @@
-""" basic collect and runtest protocol implementations """
-
-import py, sys, time
-from py._code.code import TerminalRepr
-
-def pytest_namespace():
-    return {
-        'fail'         : fail,
-        'skip'         : skip,
-        'importorskip' : importorskip,
-        'exit'         : exit,
-    }
-
-#
-# pytest plugin hooks
-
-def pytest_addoption(parser):
-    group = parser.getgroup("terminal reporting", "reporting", after="general")
-    group.addoption('--durations',
-         action="store", type="int", default=None, metavar="N",
-         help="show N slowest setup/test durations (N=0 for all)."),
-
-def pytest_terminal_summary(terminalreporter):
-    durations = terminalreporter.config.option.durations
-    if durations is None:
-        return
-    tr = terminalreporter
-    dlist = []
-    for replist in tr.stats.values():
-        for rep in replist:
-            if hasattr(rep, 'duration'):
-                dlist.append(rep)
-    if not dlist:
-        return
-    dlist.sort(key=lambda x: x.duration)
-    dlist.reverse()
-    if not durations:
-        tr.write_sep("=", "slowest test durations")
-    else:
-        tr.write_sep("=", "slowest %s test durations" % durations)
-        dlist = dlist[:durations]
-
-    for rep in dlist:
-        nodeid = rep.nodeid.replace("::()::", "::")
-        tr.write_line("%02.2fs %-8s %s" %
-            (rep.duration, rep.when, nodeid))
-
-def pytest_sessionstart(session):
-    session._setupstate = SetupState()
-def pytest_sessionfinish(session):
-    session._setupstate.teardown_all()
-
-class NodeInfo:
-    def __init__(self, location):
-        self.location = location
-
-def pytest_runtest_protocol(item, nextitem):
-    item.ihook.pytest_runtest_logstart(
-        nodeid=item.nodeid, location=item.location,
-    )
-    runtestprotocol(item, nextitem=nextitem)
-    return True
-
-def runtestprotocol(item, log=True, nextitem=None):
-    rep = call_and_report(item, "setup", log)
-    reports = [rep]
-    if rep.passed:
-        reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log,
-        nextitem=nextitem))
-    return reports
-
-def pytest_runtest_setup(item):
-    item.session._setupstate.prepare(item)
-
-def pytest_runtest_call(item):
-    item.runtest()
-
-def pytest_runtest_teardown(item, nextitem):
-    item.session._setupstate.teardown_exact(item, nextitem)
-
-def pytest_report_teststatus(report):
-    if report.when in ("setup", "teardown"):
-        if report.failed:
-            #      category, shortletter, verbose-word
-            return "error", "E", "ERROR"
-        elif report.skipped:
-            return "skipped", "s", "SKIPPED"
-        else:
-            return "", "", ""
-
-
-#
-# Implementation
-
-def call_and_report(item, when, log=True, **kwds):
-    call = call_runtest_hook(item, when, **kwds)
-    hook = item.ihook
-    report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log:
-        hook.pytest_runtest_logreport(report=report)
-    return report
-
-def call_runtest_hook(item, when, **kwds):
-    hookname = "pytest_runtest_" + when
-    ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
-
-class CallInfo:
-    """ Result/Exception info a function invocation. """
-    #: None or ExceptionInfo object.
-    excinfo = None
-    def __init__(self, func, when):
-        #: context of invocation: one of "setup", "call",
-        #: "teardown", "memocollect"
-        self.when = when
-        self.start = time.time()
-        try:
-            try:
-                self.result = func()
-            except KeyboardInterrupt:
-                raise
-            except:
-                self.excinfo = py.code.ExceptionInfo()
-        finally:
-            self.stop = time.time()
-
-    def __repr__(self):
-        if self.excinfo:
-            status = "exception: %s" % str(self.excinfo.value)
-        else:
-            status = "result: %r" % (self.result,)
-        return "<CallInfo when=%r %s>" % (self.when, status)
-
-def getslaveinfoline(node):
-    try:
-        return node._slaveinfocache
-    except AttributeError:
-        d = node.slaveinfo
-        ver = "%s.%s.%s" % d['version_info'][:3]
-        node._slaveinfocache = s = "[%s] %s -- Python %s %s" % (
-            d['id'], d['sysplatform'], ver, d['executable'])
-        return s
-
-class BaseReport(object):
-
-    def __init__(self, **kw):
-        self.__dict__.update(kw)
-
-    def toterminal(self, out):
-        longrepr = self.longrepr
-        if hasattr(self, 'node'):
-            out.line(getslaveinfoline(self.node))
-        if hasattr(longrepr, 'toterminal'):
-            longrepr.toterminal(out)
-        else:
-            out.line(str(longrepr))
-
-    passed = property(lambda x: x.outcome == "passed")
-    failed = property(lambda x: x.outcome == "failed")
-    skipped = property(lambda x: x.outcome == "skipped")
-
-    @property
-    def fspath(self):
-        return self.nodeid.split("::")[0]
-
-def pytest_runtest_makereport(item, call):
-    when = call.when
-    duration = call.stop-call.start
-    keywords = dict([(x,1) for x in item.keywords])
-    excinfo = call.excinfo
-    if not call.excinfo:
-        outcome = "passed"
-        longrepr = None
-    else:
-        excinfo = call.excinfo
-        if not isinstance(excinfo, py.code.ExceptionInfo):
-            outcome = "failed"
-            longrepr = excinfo
-        elif excinfo.errisinstance(py.test.skip.Exception):
-            outcome = "skipped"
-            r = excinfo._getreprcrash()
-            longrepr = (str(r.path), r.lineno, r.message)
-        else:
-            outcome = "failed"
-            if call.when == "call":
-                longrepr = item.repr_failure(excinfo)
-            else: # exception in setup or teardown
-                longrepr = item._repr_failure_py(excinfo)
-    return TestReport(item.nodeid, item.location,
-                      keywords, outcome, longrepr, when,
-                      duration=duration)
-
-class TestReport(BaseReport):
-    """ Basic test report object (also used for setup and teardown calls if
-    they fail).
-    """
-    def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
-        #: normalized collection node id
-        self.nodeid = nodeid
-
-        #: a (filesystempath, lineno, domaininfo) tuple indicating the
-        #: actual location of a test item - it might be different from the
-        #: collected one e.g. if a method is inherited from a different module.
-        self.location = location
-
-        #: a name -> value dictionary containing all keywords and
-        #: markers associated with a test invocation.
-        self.keywords = keywords
-
-        #: test outcome, always one of "passed", "failed", "skipped".
-        self.outcome = outcome
-
-        #: None or a failure representation.
-        self.longrepr = longrepr
-
-        #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
-        self.when = when
-
-        #: list of (secname, data) extra information which needs to
-        #: marshallable
-        self.sections = list(sections)
-
-        #: time it took to run just the test
-        self.duration = duration
-
-        self.__dict__.update(extra)
-
-    def __repr__(self):
-        return "<TestReport %r when=%r outcome=%r>" % (
-            self.nodeid, self.when, self.outcome)
-
-class TeardownErrorReport(BaseReport):
-    outcome = "failed"
-    when = "teardown"
-    def __init__(self, longrepr, **extra):
-        self.longrepr = longrepr
-        self.sections = []
-        self.__dict__.update(extra)
-
-def pytest_make_collect_report(collector):
-    call = CallInfo(collector._memocollect, "memocollect")
-    longrepr = None
-    if not call.excinfo:
-        outcome = "passed"
-    else:
-        if call.excinfo.errisinstance(py.test.skip.Exception):
-            outcome = "skipped"
-            r = collector._repr_failure_py(call.excinfo, "line").reprcrash
-            longrepr = (str(r.path), r.lineno, r.message)
-        else:
-            outcome = "failed"
-            errorinfo = collector.repr_failure(call.excinfo)
-            if not hasattr(errorinfo, "toterminal"):
-                errorinfo = CollectErrorRepr(errorinfo)
-            longrepr = errorinfo
-    return CollectReport(collector.nodeid, outcome, longrepr,
-        getattr(call, 'result', None))
-
-class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
-        self.nodeid = nodeid
-        self.outcome = outcome
-        self.longrepr = longrepr
-        self.result = result or []
-        self.sections = list(sections)
-        self.__dict__.update(extra)
-
-    @property
-    def location(self):
-        return (self.fspath, None, self.fspath)
-
-    def __repr__(self):
-        return "<CollectReport %r lenresult=%s outcome=%r>" % (
-                self.nodeid, len(self.result), self.outcome)
-
-class CollectErrorRepr(TerminalRepr):
-    def __init__(self, msg):
-        self.longrepr = msg
-    def toterminal(self, out):
-        out.line(str(self.longrepr), red=True)
-
-class SetupState(object):
-    """ shared state for setting up/tearing down test items or collectors. """
-    def __init__(self):
-        self.stack = []
-        self._finalizers = {}
-
-    def addfinalizer(self, finalizer, colitem):
-        """ attach a finalizer to the given colitem.
-        if colitem is None, this will add a finalizer that
-        is called at the end of teardown_all().
-        """
-        assert hasattr(finalizer, '__call__')
-        #assert colitem in self.stack
-        self._finalizers.setdefault(colitem, []).append(finalizer)
-
-    def _pop_and_teardown(self):
-        colitem = self.stack.pop()
-        self._teardown_with_finalization(colitem)
-
-    def _callfinalizers(self, colitem):
-        finalizers = self._finalizers.pop(colitem, None)
-        while finalizers:
-            fin = finalizers.pop()
-            fin()
-
-    def _teardown_with_finalization(self, colitem):
-        self._callfinalizers(colitem)
-        if colitem:
-            colitem.teardown()
-        for colitem in self._finalizers:
-            assert colitem is None or colitem in self.stack
-
-    def teardown_all(self):
-        while self.stack:
-            self._pop_and_teardown()
-        self._teardown_with_finalization(None)
-        assert not self._finalizers
-
-    def teardown_exact(self, item, nextitem):
-        needed_collectors = nextitem and nextitem.listchain() or []
-        self._teardown_towards(needed_collectors)
-
-    def _teardown_towards(self, needed_collectors):
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
-
-    def prepare(self, colitem):
-        """ setup objects along the collector chain to the test-method
-            and teardown previously setup objects."""
-        needed_collectors = colitem.listchain()
-        self._teardown_towards(needed_collectors)
-
-        # check if the last collection node has raised an error
-        for col in self.stack:
-            if hasattr(col, '_prepare_exc'):
-                py.builtin._reraise(*col._prepare_exc)
-        for col in needed_collectors[len(self.stack):]:
-            self.stack.append(col)
-            try:
-                col.setup()
-            except Exception:
-                col._prepare_exc = sys.exc_info()
-                raise
-
-# =============================================================
-# Test OutcomeExceptions and helpers for creating them.
-
-
-class OutcomeException(Exception):
-    """ OutcomeException and its subclass instances indicate and
-        contain info about test and collection outcomes.
-    """
-    def __init__(self, msg=None, pytrace=True):
-        self.msg = msg
-        self.pytrace = pytrace
-
-    def __repr__(self):
-        if self.msg:
-            return str(self.msg)
-        return "<%s instance>" %(self.__class__.__name__,)
-    __str__ = __repr__
-
-class Skipped(OutcomeException):
-    # XXX hackish: on 3k we fake to live in the builtins
-    # in order to have Skipped exception printing shorter/nicer
-    __module__ = 'builtins'
-
-class Failed(OutcomeException):
-    """ raised from an explicit call to py.test.fail() """
-    __module__ = 'builtins'
-
-class Exit(KeyboardInterrupt):
-    """ raised for immediate program exits (no tracebacks/summaries)"""
-    def __init__(self, msg="unknown reason"):
-        self.msg = msg
-        KeyboardInterrupt.__init__(self, msg)
-
-# exposed helper methods
-
-def exit(msg):
-    """ exit testing process as if KeyboardInterrupt was triggered. """
-    __tracebackhide__ = True
-    raise Exit(msg)
-
-exit.Exception = Exit
-
-def skip(msg=""):
-    """ skip an executing test with the given message.  Note: it's usually
-    better to use the py.test.mark.skipif marker to declare a test to be
-    skipped under certain conditions like mismatching platforms or
-    dependencies.  See the pytest_skipping plugin for details.
-    """
-    __tracebackhide__ = True
-    raise Skipped(msg=msg)
-skip.Exception = Skipped
-
-def fail(msg="", pytrace=True):
-    """ explicitely fail an currently-executing test with the given Message.
-    if @pytrace is not True the msg represents the full failure information.
-    """
-    __tracebackhide__ = True
-    raise Failed(msg=msg, pytrace=pytrace)
-fail.Exception = Failed
-
-
-def importorskip(modname, minversion=None):
-    """ return imported module if it has a higher __version__ than the
-    optionally specified 'minversion' - otherwise call py.test.skip()
-    with a message detailing the mismatch.
-    """
-    __tracebackhide__ = True
-    compile(modname, '', 'eval') # to catch syntaxerrors
-    try:
-        mod = __import__(modname, None, None, ['__doc__'])
-    except ImportError:
-        py.test.skip("could not import %r" %(modname,))
-    if minversion is None:
-        return mod
-    verattr = getattr(mod, '__version__', None)
-    if isinstance(minversion, str):
-        minver = minversion.split(".")
-    else:
-        minver = list(minversion)
-    if verattr is None or verattr.split(".") < minver:
-        py.test.skip("module %r has __version__ %r, required is: %r" %(
-                     modname, verattr, minversion))
-    return mod
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
deleted file mode 100644
--- a/_pytest/skipping.py
+++ /dev/null
@@ -1,271 +0,0 @@
-""" support for skip/xfail functions and markers. """
-
-import py, pytest
-import sys
-
-def pytest_addoption(parser):
-    group = parser.getgroup("general")
-    group.addoption('--runxfail',
-           action="store_true", dest="runxfail", default=False,
-           help="run tests even if they are marked xfail")
-
-def pytest_configure(config):
-    config.addinivalue_line("markers",
-        "skipif(*conditions): skip the given test function if evaluation "
-        "of all conditions has a True value.  Evaluation happens within the "
-        "module global context. Example: skipif('sys.platform == \"win32\"') "
-        "skips the test if we are on the win32 platform. "
-    )
-    config.addinivalue_line("markers",
-        "xfail(*conditions, reason=None, run=True): mark the the test function "
-        "as an expected failure. Optionally specify a reason and run=False "
-        "if you don't even want to execute the test function. Any positional "
-        "condition strings will be evaluated (like with skipif) and if one is "
-        "False the marker will not be applied."
-    )
-
-def pytest_namespace():
-    return dict(xfail=xfail)
-
-class XFailed(pytest.fail.Exception):
-    """ raised from an explicit call to py.test.xfail() """
-
-def xfail(reason=""):
-    """ xfail an executing test or setup functions with the given reason."""
-    __tracebackhide__ = True
-    raise XFailed(reason)
-xfail.Exception = XFailed
-
-class MarkEvaluator:
-    def __init__(self, item, name):
-        self.item = item
-        self.name = name
-
-    @property
-    def holder(self):
-        return self.item.keywords.get(self.name, None)
-    def __bool__(self):
-        return bool(self.holder)
-    __nonzero__ = __bool__
-
-    def wasvalid(self):
-        return not hasattr(self, 'exc')
-
-    def istrue(self):
-        try:
-            return self._istrue()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.exc = sys.exc_info()
-            if isinstance(self.exc[1], SyntaxError):
-                msg = [" " * (self.exc[1].offset + 4) + "^",]
-                msg.append("SyntaxError: invalid syntax")
-            else:
-                msg = py.std.traceback.format_exception_only(*self.exc[:2])
-            pytest.fail("Error evaluating %r expression\n"
-                        "    %s\n"
-                        "%s"
-                        %(self.name, self.expr, "\n".join(msg)),
-                        pytrace=False)
-
-    def _getglobals(self):
-        d = {'os': py.std.os, 'sys': py.std.sys, 'config': self.item.config}
-        func = self.item.obj
-        try:
-            d.update(func.__globals__)
-        except AttributeError:
-            d.update(func.func_globals)
-        return d
-
-    def _istrue(self):
-        if self.holder:
-            d = self._getglobals()
-            if self.holder.args:
-                self.result = False
-                for expr in self.holder.args:
-                    self.expr = expr
-                    if isinstance(expr, str):
-                        result = cached_eval(self.item.config, expr, d)
-                    else:
-                        pytest.fail("expression is not a string")
-                    if result:
-                        self.result = True
-                        self.expr = expr
-                        break
-            else:
-                self.result = True
-        return getattr(self, 'result', False)
-
-    def get(self, attr, default=None):
-        return self.holder.kwargs.get(attr, default)
-
-    def getexplanation(self):
-        expl = self.get('reason', None)
-        if not expl:
-            if not hasattr(self, 'expr'):
-                return ""
-            else:
-                return "condition: " + str(self.expr)
-        return expl
-
-
-def pytest_runtest_setup(item):
-    if not isinstance(item, pytest.Function):
-        return
-    evalskip = MarkEvaluator(item, 'skipif')
-    if evalskip.istrue():
-        py.test.skip(evalskip.getexplanation())
-    item._evalxfail = MarkEvaluator(item, 'xfail')
-    check_xfail_no_run(item)
-
-def pytest_pyfunc_call(pyfuncitem):
-    check_xfail_no_run(pyfuncitem)
-
-def check_xfail_no_run(item):
-    if not item.config.option.runxfail:
-        evalxfail = item._evalxfail
-        if evalxfail.istrue():
-            if not evalxfail.get('run', True):
-                py.test.xfail("[NOTRUN] " + evalxfail.getexplanation())
-
-def pytest_runtest_makereport(__multicall__, item, call):
-    if not isinstance(item, pytest.Function):
-        return
-    # unitttest special case, see setting of _unexpectedsuccess
-    if hasattr(item, '_unexpectedsuccess'):
-        rep = __multicall__.execute()
-        if rep.when == "call":
-            # we need to translate into how py.test encodes xpass
-            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
-            rep.outcome = "failed"
-        return rep
-    if not (call.excinfo and
-        call.excinfo.errisinstance(py.test.xfail.Exception)):
-        evalxfail = getattr(item, '_evalxfail', None)
-        if not evalxfail:
-            return
-    if call.excinfo and call.excinfo.errisinstance(py.test.xfail.Exception):
-        if not item.config.getvalue("runxfail"):
-            rep = __multicall__.execute()
-            rep.keywords['xfail'] = "reason: " + call.excinfo.value.msg
-            rep.outcome = "skipped"
-            return rep
-    rep = __multicall__.execute()
-    evalxfail = item._evalxfail
-    if not item.config.option.runxfail:
-        if evalxfail.wasvalid() and evalxfail.istrue():
-            if call.excinfo:
-                rep.outcome = "skipped"
-                rep.keywords['xfail'] = evalxfail.getexplanation()
-            elif call.when == "call":
-                rep.outcome = "failed"
-                rep.keywords['xfail'] = evalxfail.getexplanation()
-            return rep
-    if 'xfail' in rep.keywords:
-        del rep.keywords['xfail']
-    return rep
-
-# called by terminalreporter progress reporting
-def pytest_report_teststatus(report):
-    if 'xfail' in report.keywords:
-        if report.skipped:
-            return "xfailed", "x", "xfail"
-        elif report.failed:
-            return "xpassed", "X", "XPASS"
-
-# called by the terminalreporter instance/plugin
-def pytest_terminal_summary(terminalreporter):
-    tr = terminalreporter
-    if not tr.reportchars:
-        #for name in "xfailed skipped failed xpassed":
-        #    if not tr.stats.get(name, 0):
-        #        tr.write_line("HINT: use '-r' option to see extra "
-        #              "summary info about tests")
-        #        break
-        return
-
-    lines = []
-    for char in tr.reportchars:
-        if char == "x":
-            show_xfailed(terminalreporter, lines)
-        elif char == "X":
-            show_xpassed(terminalreporter, lines)
-        elif char in "fF":
-            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
-        elif char in "sS":
-            show_skipped(terminalreporter, lines)
-        elif char == "E":
-            show_simple(terminalreporter, lines, 'error', "ERROR %s")
-    if lines:
-        tr._tw.sep("=", "short test summary info")
-        for line in lines:
-            tr._tw.line(line)
-
-def show_simple(terminalreporter, lines, stat, format):
-    tw = terminalreporter._tw
-    failed = terminalreporter.stats.get(stat)
-    if failed:
-        for rep in failed:
-            pos = rep.nodeid
-            lines.append(format %(pos, ))
-
-def show_xfailed(terminalreporter, lines):
-    xfailed = terminalreporter.stats.get("xfailed")
-    if xfailed:
-        for rep in xfailed:
-            pos = rep.nodeid
-            reason = rep.keywords['xfail']
-            lines.append("XFAIL %s" % (pos,))
-            if reason:
-                lines.append("  " + str(reason))
-
-def show_xpassed(terminalreporter, lines):
-    xpassed = terminalreporter.stats.get("xpassed")
-    if xpassed:
-        for rep in xpassed:
-            pos = rep.nodeid
-            reason = rep.keywords['xfail']
-            lines.append("XPASS %s %s" %(pos, reason))
-
-def cached_eval(config, expr, d):
-    if not hasattr(config, '_evalcache'):
-        config._evalcache = {}
-    try:
-        return config._evalcache[expr]
-    except KeyError:
-        #import sys
-        #print >>sys.stderr, ("cache-miss: %r" % expr)
-        exprcode = py.code.compile(expr, mode="eval")
-        config._evalcache[expr] = x = eval(exprcode, d)
-        return x
-
-
-def folded_skips(skipped):
-    d = {}
-    for event in skipped:
-        key = event.longrepr
-        assert len(key) == 3, (event, key)
-        d.setdefault(key, []).append(event)
-    l = []
-    for key, events in d.items():
-        l.append((len(events),) + key)
-    return l
-
-def show_skipped(terminalreporter, lines):
-    tr = terminalreporter
-    skipped = tr.stats.get('skipped', [])
-    if skipped:
-        #if not tr.hasopt('skipped'):
-        #    tr.write_line(
-        #        "%d skipped tests, specify -rs for more info" %
-        #        len(skipped))
-        #    return
-        fskips = folded_skips(skipped)
-        if fskips:
-            #tr.write_sep("_", "skipped test summary")
-            for num, fspath, lineno, reason in fskips:
-                if reason.startswith("Skipped: "):
-                    reason = reason[9:]
-                lines.append("SKIP [%d] %s:%d: %s" %
-                    (num, fspath, lineno, reason))
diff --git a/_pytest/standalonetemplate.py b/_pytest/standalonetemplate.py
deleted file mode 100755
--- a/_pytest/standalonetemplate.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#! /usr/bin/env python
-
-sources = """
- at SOURCES@"""
-
-import sys
-import base64
-import zlib
-import imp
-
-class DictImporter(object):
-    def __init__(self, sources):
-        self.sources = sources
-
-    def find_module(self, fullname, path=None):
-        if fullname in self.sources:
-            return self
-        if fullname + '.__init__' in self.sources:
-            return self
-        return None
-
-    def load_module(self, fullname):
-        # print "load_module:",  fullname
-        from types import ModuleType
-        try:
-            s = self.sources[fullname]
-            is_pkg = False
-        except KeyError:
-            s = self.sources[fullname + '.__init__']
-            is_pkg = True
-
-        co = compile(s, fullname, 'exec')
-        module = sys.modules.setdefault(fullname, ModuleType(fullname))
-        module.__file__ = "%s/%s" % (__file__, fullname)
-        module.__loader__ = self
-        if is_pkg:
-            module.__path__ = [fullname]
-
-        do_exec(co, module.__dict__)
-        return sys.modules[fullname]
-
-    def get_source(self, name):
-        res = self.sources.get(name)
-        if res is None:
-            res = self.sources.get(name + '.__init__')
-        return res
-
-if __name__ == "__main__":
-    if sys.version_info >= (3, 0):
-        exec("def do_exec(co, loc): exec(co, loc)\n")
-        import pickle
-        sources = sources.encode("ascii") # ensure bytes
-        sources = pickle.loads(zlib.decompress(base64.decodebytes(sources)))
-    else:
-        import cPickle as pickle
-        exec("def do_exec(co, loc): exec co in loc\n")
-        sources = pickle.loads(zlib.decompress(base64.decodestring(sources)))
-
-    importer = DictImporter(sources)
-    sys.meta_path.append(importer)
-
-    entry = "@ENTRY@"
-    do_exec(entry, locals())
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
deleted file mode 100644
--- a/_pytest/terminal.py
+++ /dev/null
@@ -1,473 +0,0 @@
-""" terminal reporting of the full testing process.
-
-This is a good source for looking at the various reporting hooks.
-"""
-import pytest, py
-import sys
-import os
-
-def pytest_addoption(parser):
-    group = parser.getgroup("terminal reporting", "reporting", after="general")
-    group._addoption('-v', '--verbose', action="count",
-               dest="verbose", default=0, help="increase verbosity."),
-    group._addoption('-q', '--quiet', action="count",
-               dest="quiet", default=0, help="decreate verbosity."),
-    group._addoption('-r',
-         action="store", dest="reportchars", default=None, metavar="chars",
-         help="show extra test summary info as specified by chars (f)ailed, "
-              "(E)error, (s)skipped, (x)failed, (X)passed.")
-    group._addoption('-l', '--showlocals',
-         action="store_true", dest="showlocals", default=False,
-         help="show locals in tracebacks (disabled by default).")
-    group._addoption('--report',
-         action="store", dest="report", default=None, metavar="opts",
-         help="(deprecated, use -r)")
-    group._addoption('--tb', metavar="style",
-               action="store", dest="tbstyle", default='long',
-               type="choice", choices=['long', 'short', 'no', 'line', 'native'],
-               help="traceback print mode (long/short/line/native/no).")
-    group._addoption('--fulltrace',
-               action="store_true", dest="fulltrace", default=False,
-               help="don't cut any tracebacks (default is to cut).")
-
-def pytest_configure(config):
-    config.option.verbose -= config.option.quiet
-    # we try hard to make printing resilient against
-    # later changes on FD level.
-    stdout = py.std.sys.stdout
-    if hasattr(os, 'dup') and hasattr(stdout, 'fileno'):
-        try:
-            newfd = os.dup(stdout.fileno())
-            #print "got newfd", newfd
-        except ValueError:
-            pass
-        else:
-            stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._cleanup.append(lambda: stdout.close())
-
-    reporter = TerminalReporter(config, stdout)
-    config.pluginmanager.register(reporter, 'terminalreporter')
-    if config.option.debug or config.option.traceconfig:
-        def mywriter(tags, args):
-            msg = " ".join(map(str, args))
-            reporter.write_line("[traceconfig] " + msg)
-        config.trace.root.setprocessor("pytest:config", mywriter)
-
-def getreportopt(config):
-    reportopts = ""
-    optvalue = config.option.report
-    if optvalue:
-        py.builtin.print_("DEPRECATED: use -r instead of --report option.",
-            file=py.std.sys.stderr)
-        if optvalue:
-            for setting in optvalue.split(","):
-                setting = setting.strip()
-                if setting == "skipped":
-                    reportopts += "s"
-                elif setting == "xfailed":
-                    reportopts += "x"
-    reportchars = config.option.reportchars
-    if reportchars:
-        for char in reportchars:
-            if char not in reportopts:
-                reportopts += char
-    return reportopts
-
-def pytest_report_teststatus(report):
-    if report.passed:
-        letter = "."
-    elif report.skipped:
-        letter = "s"
-    elif report.failed:
-        letter = "F"
-        if report.when != "call":
-            letter = "f"
-    return report.outcome, letter, report.outcome.upper()
-
-class TerminalReporter:
-    def __init__(self, config, file=None):
-        self.config = config
-        self.verbosity = self.config.option.verbose
-        self.showheader = self.verbosity >= 0
-        self.showfspath = self.verbosity >= 0
-        self.showlongtestinfo = self.verbosity > 0
-        self._numcollected = 0
-
-        self.stats = {}
-        self.curdir = py.path.local()
-        if file is None:
-            file = py.std.sys.stdout
-        self._tw = py.io.TerminalWriter(file)
-        self.currentfspath = None
-        self.reportchars = getreportopt(config)
-        self.hasmarkup = self._tw.hasmarkup
-
-    def hasopt(self, char):
-        char = {'xfailed': 'x', 'skipped': 's'}.get(char,char)
-        return char in self.reportchars
-
-    def write_fspath_result(self, fspath, res):
-        if fspath != self.currentfspath:
-            self.currentfspath = fspath
-            #fspath = self.curdir.bestrelpath(fspath)
-            self._tw.line()
-            #relpath = self.curdir.bestrelpath(fspath)
-            self._tw.write(fspath + " ")
-        self._tw.write(res)
-
-    def write_ensure_prefix(self, prefix, extra="", **kwargs):
-        if self.currentfspath != prefix:
-            self._tw.line()
-            self.currentfspath = prefix
-            self._tw.write(prefix)
-        if extra:
-            self._tw.write(extra, **kwargs)
-            self.currentfspath = -2
-
-    def ensure_newline(self):
-        if self.currentfspath:
-            self._tw.line()
-            self.currentfspath = None
-
-    def write(self, content, **markup):
-        self._tw.write(content, **markup)
-
-    def write_line(self, line, **markup):
-        line = str(line)
-        self.ensure_newline()
-        self._tw.line(line, **markup)
-
-    def rewrite(self, line, **markup):
-        line = str(line)
-        self._tw.write("\r" + line, **markup)
-
-    def write_sep(self, sep, title=None, **markup):
-        self.ensure_newline()
-        self._tw.sep(sep, title, **markup)
-
-    def pytest_internalerror(self, excrepr):
-        for line in str(excrepr).split("\n"):
-            self.write_line("INTERNALERROR> " + line)
-        return 1
-
-    def pytest_plugin_registered(self, plugin):
-        if self.config.option.traceconfig:
-            msg = "PLUGIN registered: %s" %(plugin,)
-            # XXX this event may happen during setup/teardown time
-            #     which unfortunately captures our output here
-            #     which garbles our output if we use self.write_line
-            self.write_line(msg)
-
-    def pytest_deselected(self, items):
-        self.stats.setdefault('deselected', []).extend(items)
-
-    def pytest_runtest_logstart(self, nodeid, location):
-        # ensure that the path is printed before the
-        # 1st test of a module starts running
-        fspath = nodeid.split("::")[0]
-        if self.showlongtestinfo:
-            line = self._locationline(fspath, *location)
-            self.write_ensure_prefix(line, "")
-        elif self.showfspath:
-            self.write_fspath_result(fspath, "")
-
-    def pytest_runtest_logreport(self, report):
-        rep = report
-        res = self.config.hook.pytest_report_teststatus(report=rep)
-        cat, letter, word = res
-        self.stats.setdefault(cat, []).append(rep)
-        if not letter and not word:
-            # probably passed setup/teardown
-            return
-        if self.verbosity <= 0:
-            if not hasattr(rep, 'node') and self.showfspath:
-                self.write_fspath_result(rep.fspath, letter)
-            else:
-                self._tw.write(letter)
-        else:
-            if isinstance(word, tuple):
-                word, markup = word
-            else:
-                if rep.passed:
-                    markup = {'green':True}
-                elif rep.failed:
-                    markup = {'red':True}
-                elif rep.skipped:
-                    markup = {'yellow':True}
-            line = self._locationline(str(rep.fspath), *rep.location)
-            if not hasattr(rep, 'node'):
-                self.write_ensure_prefix(line, word, **markup)
-                #self._tw.write(word, **markup)
-            else:
-                self.ensure_newline()
-                if hasattr(rep, 'node'):
-                    self._tw.write("[%s] " % rep.node.gateway.id)
-                self._tw.write(word, **markup)
-                self._tw.write(" " + line)
-                self.currentfspath = -2
-
-    def pytest_collection(self):
-        if not self.hasmarkup:
-            self.write("collecting ... ", bold=True)
-
-    def pytest_collectreport(self, report):
-        if report.failed:
-            self.stats.setdefault("error", []).append(report)
-        elif report.skipped:
-            self.stats.setdefault("skipped", []).append(report)
-        items = [x for x in report.result if isinstance(x, pytest.Item)]
-        self._numcollected += len(items)
-        if self.hasmarkup:
-            #self.write_fspath_result(report.fspath, 'E')
-            self.report_collect()
-
-    def report_collect(self, final=False):
-        errors = len(self.stats.get('error', []))
-        skipped = len(self.stats.get('skipped', []))
-        if final:
-            line = "collected "
-        else:
-            line = "collecting "
-        line += str(self._numcollected) + " items"
-        if errors:
-            line += " / %d errors" % errors
-        if skipped:
-            line += " / %d skipped" % skipped
-        if self.hasmarkup:
-            if final:
-                line += " \n"
-            self.rewrite(line, bold=True)
-        else:
-            self.write_line(line)
-
-    def pytest_collection_modifyitems(self):
-        self.report_collect(True)
-
-    def pytest_sessionstart(self, session):
-        self._sessionstarttime = py.std.time.time()
-        if not self.showheader:
-            return
-        self.write_sep("=", "test session starts", bold=True)
-        verinfo = ".".join(map(str, sys.version_info[:3]))
-        msg = "platform %s -- Python %s" % (sys.platform, verinfo)
-        if hasattr(sys, 'pypy_version_info'):
-            verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
-        msg += " -- pytest-%s" % (py.test.__version__)
-        if self.verbosity > 0 or self.config.option.debug or \
-           getattr(self.config.option, 'pastebin', None):
-            msg += " -- " + str(sys.executable)
-        self.write_line(msg)
-        lines = self.config.hook.pytest_report_header(config=self.config)
-        lines.reverse()
-        for line in flatten(lines):
-            self.write_line(line)
-
-    def pytest_collection_finish(self, session):
-        if self.config.option.collectonly:
-            self._printcollecteditems(session.items)
-            if self.stats.get('failed'):
-                self._tw.sep("!", "collection failures")
-                for rep in self.stats.get('failed'):
-                    rep.toterminal(self._tw)
-                return 1
-            return 0
-        if not self.showheader:
-            return
-        #for i, testarg in enumerate(self.config.args):
-        #    self.write_line("test path %d: %s" %(i+1, testarg))
-
-    def _printcollecteditems(self, items):
-        # to print out items and their parent collectors
-        # we take care to leave out Instances aka ()
-        # because later versions are going to get rid of them anyway
-        if self.config.option.verbose < 0:
-            if self.config.option.verbose < -1:
-                counts = {}
-                for item in items:
-                    name = item.nodeid.split('::', 1)[0]
-                    counts[name] = counts.get(name, 0) + 1
-                for name, count in sorted(counts.items()):
-                    self._tw.line("%s: %d" % (name, count))
-            else:
-                for item in items:
-                    nodeid = item.nodeid
-                    nodeid = nodeid.replace("::()::", "::")
-                    self._tw.line(nodeid)
-            return
-        stack = []
-        indent = ""
-        for item in items:
-            needed_collectors = item.listchain()[1:] # strip root node
-            while stack:
-                if stack == needed_collectors[:len(stack)]:
-                    break
-                stack.pop()
-            for col in needed_collectors[len(stack):]:
-                stack.append(col)
-                #if col.name == "()":
-                #    continue
-                indent = (len(stack)-1) * "  "
-                self._tw.line("%s%s" %(indent, col))
-
-    def pytest_sessionfinish(self, exitstatus, __multicall__):
-        __multicall__.execute()
-        self._tw.line("")
-        if exitstatus in (0, 1, 2):
-            self.summary_errors()
-            self.summary_failures()
-            self.config.hook.pytest_terminal_summary(terminalreporter=self)
-        if exitstatus == 2:
-            self._report_keyboardinterrupt()
-            del self._keyboardinterrupt_memo
-        self.summary_deselected()
-        self.summary_stats()
-
-    def pytest_keyboard_interrupt(self, excinfo):
-        self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
-
-    def pytest_unconfigure(self):
-        if hasattr(self, '_keyboardinterrupt_memo'):
-            self._report_keyboardinterrupt()
-
-    def _report_keyboardinterrupt(self):
-        excrepr = self._keyboardinterrupt_memo
-        msg = excrepr.reprcrash.message
-        self.write_sep("!", msg)
-        if "KeyboardInterrupt" in msg:
-            if self.config.option.fulltrace:
-                excrepr.toterminal(self._tw)
-            else:
-                excrepr.reprcrash.toterminal(self._tw)
-
-    def _locationline(self, collect_fspath, fspath, lineno, domain):
-        # collect_fspath comes from testid which has a "/"-normalized path
-        if fspath and fspath.replace("\\", "/") != collect_fspath:
-            fspath = "%s <- %s" % (collect_fspath, fspath)
-        if fspath:
-            line = str(fspath)
-            if lineno is not None:
-                lineno += 1
-                line += ":" + str(lineno)
-            if domain:
-                line += ": " + str(domain)
-        else:
-            line = "[location]"
-        return line + " "
-
-    def _getfailureheadline(self, rep):
-        if hasattr(rep, 'location'):
-            fspath, lineno, domain = rep.location
-            return domain
-        else:
-            return "test session" # XXX?
-
-    def _getcrashline(self, rep):
-        try:
-            return str(rep.longrepr.reprcrash)
-        except AttributeError:
-            try:
-                return str(rep.longrepr)[:50]
-            except AttributeError:
-                return ""
-
-    #
-    # summaries for sessionfinish
-    #
-    def getreports(self, name):
-        l = []
-        for x in self.stats.get(name, []):
-            if not hasattr(x, '_pdbshown'):
-                l.append(x)
-        return l
-
-    def summary_failures(self):
-        if self.config.option.tbstyle != "no":
-            reports = self.getreports('failed')
-            if not reports:
-                return
-            self.write_sep("=", "FAILURES")
-            for rep in reports:
-                if self.config.option.tbstyle == "line":
-                    line = self._getcrashline(rep)
-                    self.write_line(line)
-                else:
-                    msg = self._getfailureheadline(rep)
-                    self.write_sep("_", msg)
-                    self._outrep_summary(rep)
-
-    def summary_errors(self):
-        if self.config.option.tbstyle != "no":
-            reports = self.getreports('error')
-            if not reports:
-                return
-            self.write_sep("=", "ERRORS")
-            for rep in self.stats['error']:
-                msg = self._getfailureheadline(rep)
-                if not hasattr(rep, 'when'):
-                    # collect
-                    msg = "ERROR collecting " + msg
-                elif rep.when == "setup":
-                    msg = "ERROR at setup of " + msg
-                elif rep.when == "teardown":
-                    msg = "ERROR at teardown of " + msg
-                self.write_sep("_", msg)
-                self._outrep_summary(rep)
-
-    def _outrep_summary(self, rep):
-        rep.toterminal(self._tw)
-        for secname, content in rep.sections:
-            self._tw.sep("-", secname)
-            if content[-1:] == "\n":
-                content = content[:-1]
-            self._tw.line(content)
-
-    def summary_stats(self):
-        session_duration = py.std.time.time() - self._sessionstarttime
-
-        keys = "failed passed skipped deselected".split()
-        for key in self.stats.keys():
-            if key not in keys:
-                keys.append(key)
-        parts = []
-        for key in keys:
-            if key: # setup/teardown reports have an empty key, ignore them
-                val = self.stats.get(key, None)
-                if val:
-                    parts.append("%d %s" %(len(val), key))
-        line = ", ".join(parts)
-        # XXX coloring
-        msg = "%s in %.2f seconds" %(line, session_duration)
-        if self.verbosity >= 0:
-            self.write_sep("=", msg, bold=True)
-        else:
-            self.write_line(msg, bold=True)
-
-    def summary_deselected(self):
-        if 'deselected' in self.stats:
-            l = []
-            k = self.config.option.keyword
-            if k:
-                l.append("-k%s" % k)
-            m = self.config.option.markexpr
-            if m:
-                l.append("-m %r" % m)
-            self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), " ".join(l)), bold=True)
-
-def repr_pythonversion(v=None):
-    if v is None:
-        v = sys.version_info
-    try:
-        return "%s.%s.%s-%s-%s" % v
-    except (TypeError, ValueError):
-        return str(v)
-
-def flatten(l):
-    for x in l:
-        if isinstance(x, (list, tuple)):
-            for y in flatten(x):
-                yield y
-        else:
-            yield x
-
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
deleted file mode 100644
--- a/_pytest/tmpdir.py
+++ /dev/null
@@ -1,68 +0,0 @@
-""" support for providing temporary directories to test functions.  """
-import pytest, py
-from _pytest.monkeypatch import monkeypatch
-
-class TempdirHandler:
-    def __init__(self, config):
-        self.config = config
-        self.trace = config.trace.get("tmpdir")
-
-    def ensuretemp(self, string, dir=1):
-        """ (deprecated) return temporary directory path with
-            the given string as the trailing part.  It is usually
-            better to use the 'tmpdir' function argument which
-            provides an empty unique-per-test-invocation directory
-            and is guaranteed to be empty.
-        """
-        #py.log._apiwarn(">1.1", "use tmpdir function argument")
-        return self.getbasetemp().ensure(string, dir=dir)
-
-    def mktemp(self, basename, numbered=True):
-        basetemp = self.getbasetemp()
-        if not numbered:
-            p = basetemp.mkdir(basename)
-        else:
-            p = py.path.local.make_numbered_dir(prefix=basename,
-                keep=0, rootdir=basetemp, lock_timeout=None)
-        self.trace("mktemp", p)
-        return p
-
-    def getbasetemp(self):
-        """ return base temporary directory. """
-        try:
-            return self._basetemp
-        except AttributeError:
-            basetemp = self.config.option.basetemp
-            if basetemp:
-                basetemp = py.path.local(basetemp)
-                if basetemp.check():
-                    basetemp.remove()
-                basetemp.mkdir()
-            else:
-                basetemp = py.path.local.make_numbered_dir(prefix='pytest-')
-            self._basetemp = t = basetemp
-            self.trace("new basetemp", t)
-            return t
-
-    def finish(self):
-        self.trace("finish")
-
-def pytest_configure(config):
-    mp = monkeypatch()
-    t = TempdirHandler(config)
-    config._cleanup.extend([mp.undo, t.finish])
-    mp.setattr(config, '_tmpdirhandler', t, raising=False)
-    mp.setattr(pytest, 'ensuretemp', t.ensuretemp, raising=False)
-
-def pytest_funcarg__tmpdir(request):
-    """return a temporary directory path object
-    which is unique to each test function invocation,
-    created as a sub directory of the base temporary
-    directory.  The returned object is a `py.path.local`_
-    path object.
-    """
-    name = request._pyfuncitem.name
-    name = py.std.re.sub("[\W]", "_", name)
-    x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x
-
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
deleted file mode 100644
--- a/_pytest/unittest.py
+++ /dev/null
@@ -1,165 +0,0 @@
-""" discovery and running of std-library "unittest" style tests. """
-import pytest, py
-import sys, pdb
-
-# for transfering markers
-from _pytest.python import transfer_markers
-
-def pytest_pycollect_makeitem(collector, name, obj):
-    unittest = sys.modules.get('unittest')
-    if unittest is None:
-        return # nobody can have derived unittest.TestCase
-    try:
-        isunit = issubclass(obj, unittest.TestCase)
-    except KeyboardInterrupt:
-        raise
-    except Exception:
-        pass
-    else:
-        if isunit:
-            return UnitTestCase(name, parent=collector)
-
-class UnitTestCase(pytest.Class):
-    def collect(self):
-        loader = py.std.unittest.TestLoader()
-        module = self.getparent(pytest.Module).obj
-        cls = self.obj
-        for name in loader.getTestCaseNames(self.obj):
-            x = getattr(self.obj, name)
-            funcobj = getattr(x, 'im_func', x)
-            transfer_markers(funcobj, cls, module)
-            if hasattr(funcobj, 'todo'):
-                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
-            yield TestCaseFunction(name, parent=self)
-
-    def setup(self):
-        meth = getattr(self.obj, 'setUpClass', None)
-        if meth is not None:
-            meth()
-        super(UnitTestCase, self).setup()
-
-    def teardown(self):
-        meth = getattr(self.obj, 'tearDownClass', None)
-        if meth is not None:
-            meth()
-        super(UnitTestCase, self).teardown()
-
-class TestCaseFunction(pytest.Function):
-    _excinfo = None
-
-    def setup(self):
-        self._testcase = self.parent.obj(self.name)
-        self._obj = getattr(self._testcase, self.name)
-        if hasattr(self._testcase, 'skip'):
-            pytest.skip(self._testcase.skip)
-        if hasattr(self._obj, 'skip'):
-            pytest.skip(self._obj.skip)
-        if hasattr(self._testcase, 'setup_method'):
-            self._testcase.setup_method(self._obj)
-
-    def teardown(self):
-        if hasattr(self._testcase, 'teardown_method'):
-            self._testcase.teardown_method(self._obj)
-
-    def startTest(self, testcase):
-        pass
-
-    def _addexcinfo(self, rawexcinfo):
-        # unwrap potential exception info (see twisted trial support below)
-        rawexcinfo = getattr(rawexcinfo, '_rawexcinfo', rawexcinfo)
-        try:
-            excinfo = py.code.ExceptionInfo(rawexcinfo)
-        except TypeError:
-            try:
-                try:
-                    l = py.std.traceback.format_exception(*rawexcinfo)
-                    l.insert(0, "NOTE: Incompatible Exception Representation, "
-                        "displaying natively:\n\n")
-                    pytest.fail("".join(l), pytrace=False)
-                except (pytest.fail.Exception, KeyboardInterrupt):
-                    raise
-                except:
-                    pytest.fail("ERROR: Unknown Incompatible Exception "
-                        "representation:\n%r" %(rawexcinfo,), pytrace=False)
-            except KeyboardInterrupt:
-                raise
-            except pytest.fail.Exception:
-                excinfo = py.code.ExceptionInfo()
-        self.__dict__.setdefault('_excinfo', []).append(excinfo)
-
-    def addError(self, testcase, rawexcinfo):
-        self._addexcinfo(rawexcinfo)
-    def addFailure(self, testcase, rawexcinfo):
-        self._addexcinfo(rawexcinfo)
-
-    def addSkip(self, testcase, reason):
-        try:
-            pytest.skip(reason)
-        except pytest.skip.Exception:
-            self._addexcinfo(sys.exc_info())
-
-    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
-        try:
-            pytest.xfail(str(reason))
-        except pytest.xfail.Exception:
-            self._addexcinfo(sys.exc_info())
-
-    def addUnexpectedSuccess(self, testcase, reason=""):
-        self._unexpectedsuccess = reason
-
-    def addSuccess(self, testcase):
-        pass
-
-    def stopTest(self, testcase):
-        pass
-
-    def runtest(self):
-        self._testcase(result=self)
-
-    def _prunetraceback(self, excinfo):
-        pytest.Function._prunetraceback(self, excinfo)
-        traceback = excinfo.traceback.filter(
-            lambda x:not x.frame.f_globals.get('__unittest'))
-        if traceback:
-            excinfo.traceback = traceback
-
- at pytest.mark.tryfirst
-def pytest_runtest_makereport(item, call):
-    if isinstance(item, TestCaseFunction):
-        if item._excinfo:
-            call.excinfo = item._excinfo.pop(0)
-            del call.result
-
-# twisted trial support
-def pytest_runtest_protocol(item, __multicall__):
-    if isinstance(item, TestCaseFunction):
-        if 'twisted.trial.unittest' in sys.modules:
-            ut = sys.modules['twisted.python.failure']
-            Failure__init__ = ut.Failure.__init__.im_func
-            check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
-                captureVars=None):
-                if exc_value is None:
-                    self._rawexcinfo = sys.exc_info()
-                else:
-                    if exc_type is None:
-                        exc_type = type(exc_value)
-                    self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                try:
-                    Failure__init__(self, exc_value, exc_type, exc_tb,
-                        captureVars=captureVars)
-                except TypeError:
-                    Failure__init__(self, exc_value, exc_type, exc_tb)
-            ut.Failure.__init__ = excstore
-            try:
-                return __multicall__.execute()
-            finally:
-                ut.Failure.__init__ = Failure__init__
-
-def check_testcase_implements_trial_reporter(done=[]):
-    if done:
-        return
-    from zope.interface import classImplements
-    from twisted.trial.itrial import IReporter
-    classImplements(TestCaseFunction, IReporter)
-    done.append(1)
diff --git a/ctypes_configure/__init__.py b/ctypes_configure/__init__.py
deleted file mode 100644
diff --git a/ctypes_configure/cbuild.py b/ctypes_configure/cbuild.py
deleted file mode 100644
--- a/ctypes_configure/cbuild.py
+++ /dev/null
@@ -1,456 +0,0 @@
-
-import os, sys, inspect, re, imp, py
-from ctypes_configure import stdoutcapture
-import distutils
-
-debug = 0
-
-configdir = py.path.local.make_numbered_dir(prefix='ctypes_configure-')
-
-class ExternalCompilationInfo(object):
-
-    _ATTRIBUTES = ['pre_include_lines', 'includes', 'include_dirs',
-                   'post_include_lines', 'libraries', 'library_dirs',
-                   'separate_module_sources', 'separate_module_files']
-    _AVOID_DUPLICATES = ['separate_module_files', 'libraries', 'includes',
-                         'include_dirs', 'library_dirs', 'separate_module_sources']
-
-    def __init__(self,
-                 pre_include_lines       = [],
-                 includes                = [],
-                 include_dirs            = [],
-                 post_include_lines      = [],
-                 libraries               = [],
-                 library_dirs            = [],
-                 separate_module_sources = [],
-                 separate_module_files   = []):
-        """
-        pre_include_lines: list of lines that should be put at the top
-        of the generated .c files, before any #include.  They shouldn't
-        contain an #include themselves.
-
-        includes: list of .h file names to be #include'd from the
-        generated .c files.
-
-        include_dirs: list of dir names that is passed to the C compiler
-
-        post_include_lines: list of lines that should be put at the top
-        of the generated .c files, after the #includes.
-
-        libraries: list of library names that is passed to the linker
-
-        library_dirs: list of dir names that is passed to the linker
-
-        separate_module_sources: list of multiline strings that are
-        each written to a .c file and compiled separately and linked
-        later on.  (If function prototypes are needed for other .c files
-        to access this, they can be put in post_include_lines.)
-
-        separate_module_files: list of .c file names that are compiled
-        separately and linked later on.  (If an .h file is needed for
-        other .c files to access this, it can be put in includes.)
-        """
-        for name in self._ATTRIBUTES:
-            value = locals()[name]
-            assert isinstance(value, (list, tuple))
-            setattr(self, name, tuple(value))
-
-    def _value(self):
-        return tuple([getattr(self, x) for x in self._ATTRIBUTES])
-
-    def __hash__(self):
-        return hash(self._value())
-
-    def __eq__(self, other):
-        return self.__class__ is other.__class__ and \
-               self._value() == other._value()
-
-    def __ne__(self, other):
-        return not self == other
-
-    def __repr__(self):
-        info = []
-        for attr in self._ATTRIBUTES:
-            val = getattr(self, attr)
-            info.append("%s=%s" % (attr, repr(val)))
-        return "<ExternalCompilationInfo (%s)>" % ", ".join(info)
-
-    def merge(self, *others):
-        others = list(others)
-        attrs = {}
-        for name in self._ATTRIBUTES:
-            if name not in self._AVOID_DUPLICATES:
-                s = []
-                for i in [self] + others:
-                    s += getattr(i, name)
-                attrs[name] = s
-            else:
-                s = set()
-                attr = []
-                for one in [self] + others:
-                    for elem in getattr(one, name):
-                        if elem not in s:
-                            s.add(elem)
-                            attr.append(elem)
-                attrs[name] = attr
-        return ExternalCompilationInfo(**attrs)
-
-    def write_c_header(self, fileobj):
-        for line in self.pre_include_lines:
-            print >> fileobj, line
-        for path in self.includes:
-            print >> fileobj, '#include <%s>' % (path,)
-        for line in self.post_include_lines:
-            print >> fileobj, line
-
-    def _copy_attributes(self):
-        d = {}
-        for attr in self._ATTRIBUTES:
-            d[attr] = getattr(self, attr)
-        return d
-
-    def convert_sources_to_files(self, cache_dir=None, being_main=False):
-        if not self.separate_module_sources:
-            return self
-        if cache_dir is None:
-            cache_dir = configdir.join('module_cache').ensure(dir=1)
-        num = 0
-        files = []
-        for source in self.separate_module_sources:
-            while 1:
-                filename = cache_dir.join('module_%d.c' % num)
-                num += 1
-                if not filename.check():
-                    break
-            f = filename.open("w")
-            if being_main:
-                f.write("#define PYPY_NOT_MAIN_FILE\n")
-            self.write_c_header(f)
-            source = str(source)
-            f.write(source)
-            if not source.endswith('\n'):
-                f.write('\n')
-            f.close()
-            files.append(str(filename))
-        d = self._copy_attributes()
-        d['separate_module_sources'] = ()
-        d['separate_module_files'] += tuple(files)
-        return ExternalCompilationInfo(**d)
-
-    def compile_shared_lib(self):
-        self = self.convert_sources_to_files()
-        if not self.separate_module_files:
-            return self
-        lib = compile_c_module([], 'externmod', self)
-        d = self._copy_attributes()
-        d['libraries'] += (lib,)
-        d['separate_module_files'] = ()
-        d['separate_module_sources'] = ()
-        return ExternalCompilationInfo(**d)
-
-if sys.platform == 'win32':
-    so_ext = '.dll'
-else:
-    so_ext = '.so'
-
-def compiler_command():
-    # e.g. for tcc, you might set this to
-    #    "tcc -shared -o %s.so %s.c"
-    return os.getenv('PYPY_CC')
-
-def enable_fast_compilation():
-    if sys.platform == 'win32':
-        dash = '/'
-    else:
-        dash = '-'
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt:
-        opt = re.sub('%sO\d+' % dash, '%sO0' % dash, opt)
-    else:
-        opt = '%sO0' % dash
-    gcv['OPT'] = opt
-
-def ensure_correct_math():
-    if sys.platform != 'win32':
-        return # so far
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    opt = gcv.get('OPT') # not always existent
-    if opt and '/Op' not in opt:
-        opt += '/Op'
-    gcv['OPT'] = opt
-
-
-def try_compile(c_files, eci):
-    try:
-        build_executable(c_files, eci)
-        result = True
-    except (distutils.errors.CompileError,
-            distutils.errors.LinkError):
-        result = False
-    return result
-
-def compile_c_module(cfiles, modbasename, eci, tmpdir=None):
-    #try:
-    #    from distutils.log import set_threshold
-    #    set_threshold(10000)
-    #except ImportError:
-    #    print "ERROR IMPORTING"
-    #    pass
-    cfiles = [py.path.local(f) for f in cfiles]
-    if tmpdir is None:
-        tmpdir = configdir.join("module_cache").ensure(dir=1)
-    num = 0
-    cfiles += eci.separate_module_files
-    include_dirs = list(eci.include_dirs)
-    library_dirs = list(eci.library_dirs)
-    if (sys.platform == 'darwin' or    # support Fink & Darwinports
-            sys.platform.startswith('freebsd')):
-        for s in ('/sw/', '/opt/local/', '/usr/local/'):
-            if s + 'include' not in include_dirs and \
-               os.path.exists(s + 'include'):
-                include_dirs.append(s + 'include')
-            if s + 'lib' not in library_dirs and \
-               os.path.exists(s + 'lib'):
-                library_dirs.append(s + 'lib')
-
-    num = 0
-    modname = modbasename
-    while 1:
-        if not tmpdir.join(modname + so_ext).check():
-            break
-        num += 1
-        modname = '%s_%d' % (modbasename, num)
-
-    lastdir = tmpdir.chdir()
-    libraries = eci.libraries
-    ensure_correct_math()
-    try:
-        if debug: print "modname", modname
-        c = stdoutcapture.Capture(mixed_out_err = True)
-        try:
-            try:
-                if compiler_command():
-                    # GCC-ish options only
-                    from distutils import sysconfig
-                    gcv = sysconfig.get_config_vars()
-                    cmd = compiler_command().replace('%s',
-                                                     str(tmpdir.join(modname)))
-                    for dir in [gcv['INCLUDEPY']] + list(include_dirs):
-                        cmd += ' -I%s' % dir
-                    for dir in library_dirs:
-                        cmd += ' -L%s' % dir
-                    os.system(cmd)
-                else:
-                    from distutils.dist import Distribution
-                    from distutils.extension import Extension
-                    from distutils.ccompiler import get_default_compiler
-                    saved_environ = os.environ.items()
-                    try:
-                        # distutils.core.setup() is really meant for end-user
-                        # interactive usage, because it eats most exceptions and
-                        # turn them into SystemExits.  Instead, we directly
-                        # instantiate a Distribution, which also allows us to
-                        # ignore unwanted features like config files.
-                        extra_compile_args = []
-                        # ensure correct math on windows
-                        if sys.platform == 'win32':
-                            extra_compile_args.append('/Op') # get extra precision
-                        if get_default_compiler() == 'unix':
-                            old_version = False
-                            try:
-                                g = os.popen('gcc --version', 'r')
-                                verinfo = g.read()
-                                g.close()
-                            except (OSError, IOError):
-                                pass
-                            else:
-                                old_version = verinfo.startswith('2')
-                            if not old_version:
-                                extra_compile_args.extend(["-Wno-unused-label",
-                                                        "-Wno-unused-variable"])
-                        attrs = {
-                            'name': "testmodule",
-                            'ext_modules': [
-                                Extension(modname, [str(cfile) for cfile in cfiles],
-                                    include_dirs=include_dirs,
-                                    library_dirs=library_dirs,
-                                    extra_compile_args=extra_compile_args,
-                                    libraries=list(libraries),)
-                                ],
-                            'script_name': 'setup.py',
-                            'script_args': ['-q', 'build_ext', '--inplace', '--force'],
-                            }
-                        dist = Distribution(attrs)
-                        if not dist.parse_command_line():
-                            raise ValueError, "distutils cmdline parse error"
-                        dist.run_commands()
-                    finally:
-                        for key, value in saved_environ:
-                            if os.environ.get(key) != value:
-                                os.environ[key] = value
-            finally:
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = open("%s.errors" % modname, "w")
-                    fdump.write(data)
-                    fdump.close()
-            # XXX do we need to do some check on fout/ferr?
-            # XXX not a nice way to import a module
-        except:
-            print >>sys.stderr, data
-            raise
-    finally:
-        lastdir.chdir()
-    return str(tmpdir.join(modname) + so_ext)
-
-def make_module_from_c(cfile, eci):
-    cfile = py.path.local(cfile)
-    modname = cfile.purebasename
-    compile_c_module([cfile], modname, eci)
-    return import_module_from_directory(cfile.dirpath(), modname)
-
-def import_module_from_directory(dir, modname):
-    file, pathname, description = imp.find_module(modname, [str(dir)])
-    try:
-        mod = imp.load_module(modname, file, pathname, description)
-    finally:
-        if file:
-            file.close()
-    return mod
-
-
-def log_spawned_cmd(spawn):
-    def spawn_and_log(cmd, *args, **kwds):
-        if debug:
-            print ' '.join(cmd)
-        return spawn(cmd, *args, **kwds)
-    return spawn_and_log
-
-
-class ProfOpt(object):
-    #XXX assuming gcc style flags for now
-    name = "profopt"
-    
-    def __init__(self, compiler):
-        self.compiler = compiler
-
-    def first(self):
-        self.build('-fprofile-generate')
-
-    def probe(self, exe, args):
-        # 'args' is a single string typically containing spaces
-        # and quotes, which represents several arguments.
-        os.system("'%s' %s" % (exe, args))
-
-    def after(self):
-        self.build('-fprofile-use')
-
-    def build(self, option):
-        compiler = self.compiler
-        compiler.compile_extra.append(option)
-        compiler.link_extra.append(option)
-        try:
-            compiler._build()
-        finally:
-            compiler.compile_extra.pop()
-            compiler.link_extra.pop()
-            
-class CCompiler:
-
-    def __init__(self, cfilenames, eci, outputfilename=None,
-                 compiler_exe=None, profbased=None):
-        self.cfilenames = cfilenames
-        ext = ''
-        self.compile_extra = []
-        self.link_extra = []
-        self.libraries = list(eci.libraries)
-        self.include_dirs = list(eci.include_dirs)
-        self.library_dirs = list(eci.library_dirs)
-        self.compiler_exe = compiler_exe
-        self.profbased = profbased
-        if not sys.platform in ('win32', 'darwin'): # xxx
-            if 'm' not in self.libraries:
-                self.libraries.append('m')
-            if 'pthread' not in self.libraries:
-                self.libraries.append('pthread')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer', '-pthread']
-            self.link_extra += ['-pthread']
-        if sys.platform == 'win32':
-            self.link_extra += ['/DEBUG'] # generate .pdb file
-        if (sys.platform == 'darwin' or    # support Fink & Darwinports
-                sys.platform.startswith('freebsd')):
-            for s in ('/sw/', '/opt/local/', '/usr/local/'):
-                if s + 'include' not in self.include_dirs and \
-                   os.path.exists(s + 'include'):
-                    self.include_dirs.append(s + 'include')
-                if s + 'lib' not in self.library_dirs and \
-                   os.path.exists(s + 'lib'):
-                    self.library_dirs.append(s + 'lib')
-            self.compile_extra += ['-O3', '-fomit-frame-pointer']
-
-        if outputfilename is None:
-            self.outputfilename = py.path.local(cfilenames[0]).new(ext=ext)
-        else: 
-            self.outputfilename = py.path.local(outputfilename)
-
-    def build(self, noerr=False):
-        basename = self.outputfilename.new(ext='')
-        data = ''
-        try:
-            saved_environ = os.environ.copy()
-            c = stdoutcapture.Capture(mixed_out_err = True)
-            try:
-                self._build()
-            finally:
-                # workaround for a distutils bugs where some env vars can
-                # become longer and longer every time it is used
-                for key, value in saved_environ.items():
-                    if os.environ.get(key) != value:
-                        os.environ[key] = value
-                foutput, foutput = c.done()
-                data = foutput.read()
-                if data:
-                    fdump = basename.new(ext='errors').open("w")
-                    fdump.write(data)
-                    fdump.close()
-        except:
-            if not noerr:
-                print >>sys.stderr, data
-            raise
- 
-    def _build(self):
-        from distutils.ccompiler import new_compiler 
-        compiler = new_compiler(force=1)
-        if self.compiler_exe is not None:
-            for c in '''compiler compiler_so compiler_cxx
-                        linker_exe linker_so'''.split():
-                compiler.executables[c][0] = self.compiler_exe
-        compiler.spawn = log_spawned_cmd(compiler.spawn)
-        objects = []
-        for cfile in self.cfilenames: 
-            cfile = py.path.local(cfile)
-            old = cfile.dirpath().chdir() 
-            try: 
-                res = compiler.compile([cfile.basename], 
-                                       include_dirs=self.include_dirs,
-                                       extra_preargs=self.compile_extra)
-                assert len(res) == 1
-                cobjfile = py.path.local(res[0]) 
-                assert cobjfile.check()
-                objects.append(str(cobjfile))
-            finally: 
-                old.chdir() 
-        compiler.link_executable(objects, str(self.outputfilename),
-                                 libraries=self.libraries,
-                                 extra_preargs=self.link_extra,
-                                 library_dirs=self.library_dirs)
-
-def build_executable(*args, **kwds):
-    noerr = kwds.pop('noerr', False)
-    compiler = CCompiler(*args, **kwds)
-    compiler.build(noerr=noerr)
-    return str(compiler.outputfilename)
diff --git a/ctypes_configure/configure.py b/ctypes_configure/configure.py
deleted file mode 100755
--- a/ctypes_configure/configure.py
+++ /dev/null
@@ -1,621 +0,0 @@
-#! /usr/bin/env python
-
-import os, py, sys
-import ctypes
-from ctypes_configure.cbuild import build_executable, configdir, try_compile
-from ctypes_configure.cbuild import ExternalCompilationInfo
-import distutils
-
-# ____________________________________________________________
-#
-# Helpers for simple cases
-
-def eci_from_header(c_header_source):
-    return ExternalCompilationInfo(
-        pre_include_lines=c_header_source.split("\n")
-    )
-
-
-def getstruct(name, c_header_source, interesting_fields):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        STRUCT = Struct(name, interesting_fields)
-    return configure(CConfig)['STRUCT']
-
-def getsimpletype(name, c_header_source, ctype_hint=ctypes.c_int):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        TYPE = SimpleType(name, ctype_hint)
-    return configure(CConfig)['TYPE']
-
-def getconstantinteger(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        CONST = ConstantInteger(name)
-    return configure(CConfig)['CONST']
-
-def getdefined(macro, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        DEFINED = Defined(macro)
-    return configure(CConfig)['DEFINED']
-
-def has(name, c_header_source):
-    class CConfig:
-        _compilation_info_ = eci_from_header(c_header_source)
-        HAS = Has(name)
-    return configure(CConfig)['HAS']
-
-def check_eci(eci):
-    """Check if a given ExternalCompilationInfo compiles and links."""
-    class CConfig:
-        _compilation_info_ = eci
-        WORKS = Works()
-    return configure(CConfig)['WORKS']
-
-def sizeof(name, eci, **kwds):
-    class CConfig:
-        _compilation_info_ = eci
-        SIZE = SizeOf(name)
-    for k, v in kwds.items():
-        setattr(CConfig, k, v)
-    return configure(CConfig)['SIZE']
-
-def memory_alignment():
-    """Return the alignment (in bytes) of memory allocations.
-    This is enough to make sure a structure with pointers and 'double'
-    fields is properly aligned."""
-    global _memory_alignment
-    if _memory_alignment is None:
-        S = getstruct('struct memory_alignment_test', """
-           struct memory_alignment_test {
-               double d;
-               void* p;
-           };
-        """, [])
-        result = ctypes.alignment(S)
-        assert result & (result-1) == 0, "not a power of two??"
-        _memory_alignment = result
-    return _memory_alignment
-_memory_alignment = None
-
-# ____________________________________________________________
-#
-# General interface
-
-class ConfigResult:
-    def __init__(self, CConfig, info, entries):
-        self.CConfig = CConfig
-        self.result = {}
-        self.info = info
-        self.entries = entries
-        
-    def get_entry_result(self, entry):
-        try:
-            return self.result[entry]
-        except KeyError:
-            pass
-        name = self.entries[entry]
-        info = self.info[name]
-        self.result[entry] = entry.build_result(info, self)
-
-    def get_result(self):
-        return dict([(name, self.result[entry])
-                     for entry, name in self.entries.iteritems()])
-
-
-class _CWriter(object):
-    """ A simple class which aggregates config parts
-    """
-    def __init__(self, CConfig):
-        self.path = uniquefilepath()
-        self.f = self.path.open("w")
-        self.config = CConfig
-
-    def write_header(self):
-        f = self.f
-        CConfig = self.config
-        CConfig._compilation_info_.write_c_header(f)
-        print >> f, C_HEADER
-        print >> f
-
-    def write_entry(self, key, entry):
-        f = self.f
-        print >> f, 'void dump_section_%s(void) {' % (key,)
-        for line in entry.prepare_code():
-            if line and line[0] != '#':
-                line = '\t' + line
-            print >> f, line
-        print >> f, '}'
-        print >> f
-
-    def write_entry_main(self, key):
-        print >> self.f, '\tprintf("-+- %s\\n");' % (key,)
-        print >> self.f, '\tdump_section_%s();' % (key,)
-        print >> self.f, '\tprintf("---\\n");'
-
-    def start_main(self):
-        print >> self.f, 'int main(int argc, char *argv[]) {'
-
-    def close(self):
-        f = self.f
-        print >> f, '\treturn 0;'
-        print >> f, '}'
-        f.close()
-
-    def ask_gcc(self, question):
-        self.start_main()
-        self.f.write(question + "\n")
-        self.close()
-        eci = self.config._compilation_info_
-        return try_compile([self.path], eci)
-
-        
-def configure(CConfig, noerr=False):
-    """Examine the local system by running the C compiler.
-    The CConfig class contains CConfigEntry attribues that describe
-    what should be inspected; configure() returns a dict mapping
-    names to the results.
-    """
-    for attr in ['_includes_', '_libraries_', '_sources_', '_library_dirs_',
-                 '_include_dirs_', '_header_']:
-        assert not hasattr(CConfig, attr), "Found legacy attribut %s on CConfig" % (attr,)
-    entries = []
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigEntry):
-            entries.append((key, value))
-
-    if entries:   # can be empty if there are only CConfigSingleEntries
-        writer = _CWriter(CConfig)
-        writer.write_header()
-        for key, entry in entries:
-            writer.write_entry(key, entry)
-
-        f = writer.f
-        writer.start_main()
-        for key, entry in entries:
-            writer.write_entry_main(key)
-        writer.close()
-
-        eci = CConfig._compilation_info_
-        infolist = list(run_example_code(writer.path, eci, noerr=noerr))
-        assert len(infolist) == len(entries)
-
-        resultinfo = {}
-        resultentries = {}
-        for info, (key, entry) in zip(infolist, entries):
-            resultinfo[key] = info
-            resultentries[entry] = key
-
-        result = ConfigResult(CConfig, resultinfo, resultentries)
-        for name, entry in entries:
-            result.get_entry_result(entry)
-        res = result.get_result()
-    else:
-        res = {}
-
-    for key in dir(CConfig):
-        value = getattr(CConfig, key)
-        if isinstance(value, CConfigSingleEntry):
-            writer = _CWriter(CConfig)
-            writer.write_header()
-            res[key] = value.question(writer.ask_gcc)
-    return res
-
-# ____________________________________________________________
-
-
-class CConfigEntry(object):
-    "Abstract base class."
-
-class Struct(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined structure.
-    """
-    def __init__(self, name, interesting_fields, ifdef=None):
-        self.name = name
-        self.interesting_fields = interesting_fields
-        self.ifdef = ifdef
-
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield 'typedef struct {'
-        yield '    char c;'
-        yield '    ctypesplatcheck_t s;'
-        yield '} ctypesplatcheck2_t;'
-        yield ''
-        yield 'ctypesplatcheck_t s;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("align", offsetof(ctypesplatcheck2_t, s));'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        for fieldname, fieldtype in self.interesting_fields:
-            yield 'dump("fldofs %s", offsetof(ctypesplatcheck_t, %s));'%(
-                fieldname, fieldname)
-            yield 'dump("fldsize %s",   sizeof(s.%s));' % (
-                fieldname, fieldname)
-            if fieldtype in integer_class:
-                yield 's.%s = 0; s.%s = ~s.%s;' % (fieldname,
-                                                   fieldname,
-                                                   fieldname)
-                yield 'dump("fldunsigned %s", s.%s > 0);' % (fieldname,
-                                                             fieldname)
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None:
-            if not info['defined']:
-                return None
-        alignment = 1
-        layout = [None] * info['size']
-        for fieldname, fieldtype in self.interesting_fields:
-            if isinstance(fieldtype, Struct):
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                c_fieldtype = config_result.get_entry_result(fieldtype)
-                layout_addfield(layout, offset, c_fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(c_fieldtype))
-            else:
-                offset = info['fldofs '  + fieldname]
-                size   = info['fldsize ' + fieldname]
-                sign   = info.get('fldunsigned ' + fieldname, False)
-                if (size, sign) != size_and_sign(fieldtype):
-                    fieldtype = fixup_ctype(fieldtype, fieldname, (size, sign))
-                layout_addfield(layout, offset, fieldtype, fieldname)
-                alignment = max(alignment, ctype_alignment(fieldtype))
-
-        # try to enforce the same alignment as the one of the original
-        # structure
-        if alignment < info['align']:
-            choices = [ctype for ctype in alignment_types
-                             if ctype_alignment(ctype) == info['align']]
-            assert choices, "unsupported alignment %d" % (info['align'],)
-            choices = [(ctypes.sizeof(ctype), i, ctype)
-                       for i, ctype in enumerate(choices)]
-            csize, _, ctype = min(choices)
-            for i in range(0, info['size'] - csize + 1, info['align']):
-                if layout[i:i+csize] == [None] * csize:
-                    layout_addfield(layout, i, ctype, '_alignment')
-                    break
-            else:
-                raise AssertionError("unenforceable alignment %d" % (
-                    info['align'],))
-
-        n = 0
-        for i, cell in enumerate(layout):
-            if cell is not None:
-                continue
-            layout_addfield(layout, i, ctypes.c_char, '_pad%d' % (n,))
-            n += 1
-
-        # build the ctypes Structure
-        seen = {}
-        fields = []
-        for cell in layout:
-            if cell in seen:
-                continue
-            fields.append((cell.name, cell.ctype))
-            seen[cell] = True
-
-        class S(ctypes.Structure):
-            _fields_ = fields
-        name = self.name
-        if name.startswith('struct '):
-            name = name[7:]
-        S.__name__ = name
-        return S
-
-class SimpleType(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined simple numeric type.
-    """
-    def __init__(self, name, ctype_hint=ctypes.c_int, ifdef=None):
-        self.name = name
-        self.ctype_hint = ctype_hint
-        self.ifdef = ifdef
-        
-    def prepare_code(self):
-        if self.ifdef is not None:
-            yield '#ifdef %s' % (self.ifdef,)
-        yield 'typedef %s ctypesplatcheck_t;' % (self.name,)
-        yield ''
-        yield 'ctypesplatcheck_t x;'
-        if self.ifdef is not None:
-            yield 'dump("defined", 1);'
-        yield 'dump("size",  sizeof(ctypesplatcheck_t));'
-        if self.ctype_hint in integer_class:
-            yield 'x = 0; x = ~x;'
-            yield 'dump("unsigned", x > 0);'
-        if self.ifdef is not None:
-            yield '#else'
-            yield 'dump("defined", 0);'
-            yield '#endif'
-
-    def build_result(self, info, config_result):
-        if self.ifdef is not None and not info['defined']:
-            return None
-        size = info['size']
-        sign = info.get('unsigned', False)
-        ctype = self.ctype_hint
-        if (size, sign) != size_and_sign(ctype):
-            ctype = fixup_ctype(ctype, self.name, (size, sign))
-        return ctype
-
-class ConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant.
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'if ((%s) < 0) {' % (self.name,)
-        yield '    long long x = (long long)(%s);' % (self.name,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.name,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-
-    def build_result(self, info, config_result):
-        return info['value']
-
-class DefinedConstantInteger(CConfigEntry):
-    """An entry in a CConfig class that stands for an externally
-    defined integer constant. If not #defined the value will be None.
-    """
-    def __init__(self, macro):
-        self.name = self.macro = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'if ((%s) < 0) {' % (self.macro,)
-        yield '    long long x = (long long)(%s);' % (self.macro,)
-        yield '    printf("value: %lld\\n", x);'
-        yield '} else {'
-        yield '    unsigned long long x = (unsigned long long)(%s);' % (
-                        self.macro,)
-        yield '    printf("value: %llu\\n", x);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            return info['value']
-        return None
-
-
-class DefinedConstantString(CConfigEntry):
-    """
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % self.macro
-        yield 'int i;'
-        yield 'char *p = %s;' % self.macro
-        yield 'dump("defined", 1);'
-        yield 'for (i = 0; p[i] != 0; i++ ) {'
-        yield '  printf("value_%d: %d\\n", i, (int)(unsigned char)p[i]);'
-        yield '}'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        if info["defined"]:
-            string = ''
-            d = 0
-            while info.has_key('value_%d' % d):
-                string += chr(info['value_%d' % d])
-                d += 1
-            return string
-        return None
-
-
-class Defined(CConfigEntry):
-    """A boolean, corresponding to an #ifdef.
-    """
-    def __init__(self, macro):
-        self.macro = macro
-        self.name = macro
-
-    def prepare_code(self):
-        yield '#ifdef %s' % (self.macro,)
-        yield 'dump("defined", 1);'
-        yield '#else'
-        yield 'dump("defined", 0);'
-        yield '#endif'
-
-    def build_result(self, info, config_result):
-        return bool(info['defined'])
-
-class CConfigSingleEntry(object):
-    """ An abstract class of type which requires
-    gcc succeeding/failing instead of only asking
-    """
-    pass
-
-class Has(CConfigSingleEntry):
-    def __init__(self, name):
-        self.name = name
-    
-    def question(self, ask_gcc):
-        return ask_gcc(self.name + ';')
-
-class Works(CConfigSingleEntry):
-    def question(self, ask_gcc):
-        return ask_gcc("")
-
-class SizeOf(CConfigEntry):
-    """An entry in a CConfig class that stands for
-    some external opaque type
-    """
-    def __init__(self, name):
-        self.name = name
-
-    def prepare_code(self):
-        yield 'dump("size",  sizeof(%s));' % self.name
-
-    def build_result(self, info, config_result):
-        return info['size']
-
-# ____________________________________________________________
-#
-# internal helpers
-
-def ctype_alignment(c_type):
-    if issubclass(c_type, ctypes.Structure):
-        return max([ctype_alignment(fld_type)
-                     for fld_name, fld_type in c_type._fields_])
-    
-    return ctypes.alignment(c_type)
-
-def uniquefilepath(LAST=[0]):
-    i = LAST[0]
-    LAST[0] += 1
-    return configdir.join('ctypesplatcheck_%d.c' % i)
-
-alignment_types = [
-    ctypes.c_short,
-    ctypes.c_int,
-    ctypes.c_long,
-    ctypes.c_float,
-    ctypes.c_double,
-    ctypes.c_char_p,
-    ctypes.c_void_p,
-    ctypes.c_longlong,
-    ctypes.c_wchar,
-    ctypes.c_wchar_p,
-    ]
-
-integer_class = [ctypes.c_byte,     ctypes.c_ubyte,
-                 ctypes.c_short,    ctypes.c_ushort,
-                 ctypes.c_int,      ctypes.c_uint,
-                 ctypes.c_long,     ctypes.c_ulong,
-                 ctypes.c_longlong, ctypes.c_ulonglong,
-                 ]
-float_class = [ctypes.c_float, ctypes.c_double]
-
-class Field(object):
-    def __init__(self, name, ctype):
-        self.name = name
-        self.ctype = ctype
-    def __repr__(self):
-        return '<field %s: %s>' % (self.name, self.ctype)
-
-def layout_addfield(layout, offset, ctype, prefix):
-    size = ctypes.sizeof(ctype)
-    name = prefix
-    i = 0
-    while name in layout:
-        i += 1
-        name = '%s_%d' % (prefix, i)
-    field = Field(name, ctype)
-    for i in range(offset, offset+size):
-        assert layout[i] is None, "%s overlaps %r" % (fieldname, layout[i])
-        layout[i] = field
-    return field
-
-def size_and_sign(ctype):
-    return (ctypes.sizeof(ctype),
-            ctype in integer_class and ctype(-1).value > 0)
-
-def fixup_ctype(fieldtype, fieldname, expected_size_and_sign):
-    for typeclass in [integer_class, float_class]:
-        if fieldtype in typeclass:
-            for ctype in typeclass:
-                if size_and_sign(ctype) == expected_size_and_sign:
-                    return ctype
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_char):
-        # for now, assume it is an array of chars; otherwise we'd also
-        # have to check the exact integer type of the elements of the array
-        size, sign = expected_size_and_sign
-        return ctypes.c_char * size
-    if (hasattr(fieldtype, '_length_')
-        and getattr(fieldtype, '_type_', None) == ctypes.c_ubyte):
-        # grumble, fields of type 'c_char array' have automatic cast-to-
-        # Python-string behavior in ctypes, which may not be what you
-        # want, so here is the same with c_ubytes instead...
-        size, sign = expected_size_and_sign
-        return ctypes.c_ubyte * size
-    raise TypeError("conflicting field type %r for %r" % (fieldtype,
-                                                          fieldname))
-
-
-C_HEADER = """
-#include <stdio.h>
-#include <stddef.h>   /* for offsetof() */
-#ifndef _WIN32
-#  include <stdint.h>   /* FreeBSD: for uint64_t */
-#endif
-
-void dump(char* key, int value) {
-    printf("%s: %d\\n", key, value);
-}
-"""
-
-def run_example_code(filepath, eci, noerr=False):
-    executable = build_executable([filepath], eci, noerr=noerr)
-    output = py.process.cmdexec(executable)
-    section = None
-    for line in output.splitlines():
-        line = line.strip()
-        if line.startswith('-+- '):      # start of a new section
-            section = {}
-        elif line == '---':              # section end
-            assert section is not None
-            yield section
-            section = None
-        elif line:
-            assert section is not None
-            key, value = line.split(': ')
-            section[key] = int(value)
-
-# ____________________________________________________________
-
-def get_python_include_dir():
-    from distutils import sysconfig
-    gcv = sysconfig.get_config_vars()
-    return gcv['INCLUDEPY']
-
-if __name__ == '__main__':
-    doc = """Example:
-    
-       ctypes_platform.py  -h sys/types.h  -h netinet/in.h
-                           'struct sockaddr_in'
-                           sin_port  c_int
-    """
-    import sys, getopt
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'h:')
-    if not args:
-        print >> sys.stderr, doc
-    else:
-        assert len(args) % 2 == 1
-        headers = []
-        for opt, value in opts:
-            if opt == '-h':
-                headers.append('#include <%s>' % (value,))
-        name = args[0]
-        fields = []
-        for i in range(1, len(args), 2):
-            ctype = getattr(ctypes, args[i+1])
-            fields.append((args[i], ctype))
-
-        S = getstruct(name, '\n'.join(headers), fields)
-
-        for key, value in S._fields_:
-            print key, value
diff --git a/ctypes_configure/doc/configure.html b/ctypes_configure/doc/configure.html
deleted file mode 100644
--- a/ctypes_configure/doc/configure.html
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="latin1" ?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=latin1" />
-<meta name="generator" content="Docutils 0.4.1: http://docutils.sourceforge.net/" />
-<title>ctypes configure</title>
-</head>
-<body>
-<div class="document" id="ctypes-configure">
-<h1 class="title">ctypes configure</h1>
-<div class="section">
-<h1><a id="idea" name="idea">idea</a></h1>
-<p>One of ctypes problems is that ctypes programs are usually not very
-platform-independent. We created ctypes_configure, which invokes gcc

From noreply at buildbot.pypy.org  Sat Apr 14 10:01:30 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 10:01:30 +0200 (CEST)
Subject: [pypy-commit] pypy vendor/stdlib: revert hgignore on vendor branch
Message-ID: <20120414080130.302B982F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: vendor/stdlib
Changeset: r54353:4aa4e1f8e19a
Date: 2012-04-14 09:58 +0200
http://bitbucket.org/pypy/pypy/changeset/4aa4e1f8e19a/

Log:	revert hgignore on vendor branch

diff --git a/.hgignore b/.hgignore
new file mode 100644
--- /dev/null
+++ b/.hgignore
@@ -0,0 +1,76 @@
+syntax: glob
+*.py[co]
+*~
+.*.swp
+.idea
+.project
+.pydevproject
+
+syntax: regexp
+^testresult$
+^site-packages$
+^site-packages/.*$
+^site-packages/.*$
+^bin$
+^pypy/bin/pypy-c
+^pypy/module/cpyext/src/.+\.o$
+^pypy/module/cpyext/src/.+\.obj$
+^pypy/module/cpyext/test/.+\.errors$
+^pypy/module/cpyext/test/.+\.o$
+^pypy/module/cpyext/test/.+\.obj$
+^pypy/module/cpyext/test/.+\.manifest$
+^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
+^pypy/doc/.+\.html$
+^pypy/doc/config/.+\.rst$
+^pypy/doc/basicblock\.asc$
+^pypy/doc/.+\.svninfo$
+^pypy/translator/c/src/libffi_msvc/.+\.obj$
+^pypy/translator/c/src/libffi_msvc/.+\.dll$
+^pypy/translator/c/src/libffi_msvc/.+\.lib$
+^pypy/translator/c/src/libffi_msvc/.+\.exp$
+^pypy/translator/c/src/cjkcodecs/.+\.o$
+^pypy/translator/c/src/cjkcodecs/.+\.obj$
+^pypy/translator/jvm/\.project$
+^pypy/translator/jvm/\.classpath$
+^pypy/translator/jvm/eclipse-bin$
+^pypy/translator/jvm/src/pypy/.+\.class$
+^pypy/translator/benchmark/docutils$
+^pypy/translator/benchmark/templess$
+^pypy/translator/benchmark/gadfly$
+^pypy/translator/benchmark/mako$
+^pypy/translator/benchmark/bench-custom\.benchmark_result$
+^pypy/translator/benchmark/shootout_benchmarks$
+^pypy/translator/goal/pypy-translation-snapshot$
+^pypy/translator/goal/pypy-c
+^pypy/translator/goal/pypy-jvm
+^pypy/translator/goal/pypy-jvm.jar
+^pypy/translator/goal/.+\.exe$
+^pypy/translator/goal/.+\.dll$
+^pypy/translator/goal/target.+-c$
+^pypy/_cache$
+^pypy/doc/statistic/.+\.html$
+^pypy/doc/statistic/.+\.eps$
+^pypy/doc/statistic/.+\.pdf$
+^pypy/translator/cli/src/pypylib\.dll$
+^pypy/translator/cli/src/query\.exe$
+^pypy/translator/cli/src/main\.exe$
+^lib_pypy/ctypes_config_cache/_.+_cache\.py$
+^lib_pypy/ctypes_config_cache/_.+_.+_\.py$
+^pypy/translator/cli/query-descriptions$
+^pypy/doc/discussion/.+\.html$
+^include/.+\.h$
+^include/.+\.inl$
+^pypy/doc/_build/.*$
+^pypy/doc/config/.+\.html$
+^pypy/doc/config/style\.css$
+^pypy/doc/jit/.+\.html$
+^pypy/doc/jit/style\.css$
+^pypy/doc/image/lattice1\.png$
+^pypy/doc/image/lattice2\.png$
+^pypy/doc/image/lattice3\.png$
+^pypy/doc/image/stackless_informal\.png$
+^pypy/doc/image/parsing_example.+\.png$
+^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
+^compiled
+^.git/
+^release/

From noreply at buildbot.pypy.org  Sat Apr 14 10:01:31 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 10:01:31 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: merge from vendor/stdlib
Message-ID: <20120414080131.6F36382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54354:169138063d63
Date: 2012-04-14 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/169138063d63/

Log:	merge from vendor/stdlib


From noreply at buildbot.pypy.org  Sat Apr 14 11:12:50 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 11:12:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: tool.clean_old_branches: dont
 update/purge for every head
Message-ID: <20120414091250.D1B6582F4F@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54356:ffcf6c713e0b
Date: 2012-04-14 11:12 +0200
http://bitbucket.org/pypy/pypy/changeset/ffcf6c713e0b/

Log:	tool.clean_old_branches: dont update/purge for every head

diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -54,11 +54,13 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
     do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 

From noreply at buildbot.pypy.org  Sat Apr 14 11:12:49 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 11:12:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: tool.clean_old_branches: use
 debugsetparents to avoid merge+rm
Message-ID: <20120414091249.8C55782F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54355:7d2a37b843fc
Date: 2012-04-14 11:12 +0200
http://bitbucket.org/pypy/pypy/changeset/7d2a37b843fc/

Log:	tool.clean_old_branches: use debugsetparents to avoid merge+rm

diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -61,11 +59,7 @@
     print '***** %s ***** %s *****' % (branch, head)
     do("hg up --clean closed-branches")
     do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print

From noreply at buildbot.pypy.org  Sat Apr 14 11:26:33 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 11:26:33 +0200 (CEST)
Subject: [pypy-commit] pypy vendor/stdlib: add the 3.2.3 version of the
 stdlib in lib-python/3.2
Message-ID: <20120414092633.B40BB82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: vendor/stdlib
Changeset: r54357:359343b9ac0e
Date: 2012-04-14 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/359343b9ac0e/

Log:	add the 3.2.3 version of the stdlib in lib-python/3.2

diff too long, truncating to 10000 out of 558051 lines

diff --git a/lib-python/3.2/__future__.py b/lib-python/3.2/__future__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/__future__.py
@@ -0,0 +1,134 @@
+"""Record of phased-in incompatible language changes.
+
+Each line is of the form:
+
+    FeatureName = "_Feature(" OptionalRelease "," MandatoryRelease ","
+                              CompilerFlag ")"
+
+where, normally, OptionalRelease < MandatoryRelease, and both are 5-tuples
+of the same form as sys.version_info:
+
+    (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int
+     PY_MINOR_VERSION, # the 1; an int
+     PY_MICRO_VERSION, # the 0; an int
+     PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string
+     PY_RELEASE_SERIAL # the 3; an int
+    )
+
+OptionalRelease records the first release in which
+
+    from __future__ import FeatureName
+
+was accepted.
+
+In the case of MandatoryReleases that have not yet occurred,
+MandatoryRelease predicts the release in which the feature will become part
+of the language.
+
+Else MandatoryRelease records when the feature became part of the language;
+in releases at or after that, modules no longer need
+
+    from __future__ import FeatureName
+
+to use the feature in question, but may continue to use such imports.
+
+MandatoryRelease may also be None, meaning that a planned feature got
+dropped.
+
+Instances of class _Feature have two corresponding methods,
+.getOptionalRelease() and .getMandatoryRelease().
+
+CompilerFlag is the (bitfield) flag that should be passed in the fourth
+argument to the builtin function compile() to enable the feature in
+dynamically compiled code.  This flag is stored in the .compiler_flag
+attribute on _Future instances.  These values must match the appropriate
+#defines of CO_xxx flags in Include/compile.h.
+
+No feature line is ever to be deleted from this file.
+"""
+
+all_feature_names = [
+    "nested_scopes",
+    "generators",
+    "division",
+    "absolute_import",
+    "with_statement",
+    "print_function",
+    "unicode_literals",
+    "barry_as_FLUFL",
+]
+
+__all__ = ["all_feature_names"] + all_feature_names
+
+# The CO_xxx symbols are defined here under the same names used by
+# compile.h, so that an editor search will find them here.  However,
+# they're not exported in __all__, because they don't really belong to
+# this module.
+CO_NESTED            = 0x0010   # nested_scopes
+CO_GENERATOR_ALLOWED = 0        # generators (obsolete, was 0x1000)
+CO_FUTURE_DIVISION   = 0x2000   # division
+CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
+CO_FUTURE_WITH_STATEMENT  = 0x8000   # with statement
+CO_FUTURE_PRINT_FUNCTION  = 0x10000   # print function
+CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
+CO_FUTURE_BARRY_AS_BDFL = 0x40000
+
+class _Feature:
+    def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
+        self.optional = optionalRelease
+        self.mandatory = mandatoryRelease
+        self.compiler_flag = compiler_flag
+
+    def getOptionalRelease(self):
+        """Return first release in which this feature was recognized.
+
+        This is a 5-tuple, of the same form as sys.version_info.
+        """
+
+        return self.optional
+
+    def getMandatoryRelease(self):
+        """Return release in which this feature will become mandatory.
+
+        This is a 5-tuple, of the same form as sys.version_info, or, if
+        the feature was dropped, is None.
+        """
+
+        return self.mandatory
+
+    def __repr__(self):
+        return "_Feature" + repr((self.optional,
+                                  self.mandatory,
+                                  self.compiler_flag))
+
+nested_scopes = _Feature((2, 1, 0, "beta",  1),
+                         (2, 2, 0, "alpha", 0),
+                         CO_NESTED)
+
+generators = _Feature((2, 2, 0, "alpha", 1),
+                      (2, 3, 0, "final", 0),
+                      CO_GENERATOR_ALLOWED)
+
+division = _Feature((2, 2, 0, "alpha", 2),
+                    (3, 0, 0, "alpha", 0),
+                    CO_FUTURE_DIVISION)
+
+absolute_import = _Feature((2, 5, 0, "alpha", 1),
+                           (2, 7, 0, "alpha", 0),
+                           CO_FUTURE_ABSOLUTE_IMPORT)
+
+with_statement = _Feature((2, 5, 0, "alpha", 1),
+                          (2, 6, 0, "alpha", 0),
+                          CO_FUTURE_WITH_STATEMENT)
+
+print_function = _Feature((2, 6, 0, "alpha", 2),
+                          (3, 0, 0, "alpha", 0),
+                          CO_FUTURE_PRINT_FUNCTION)
+
+unicode_literals = _Feature((2, 6, 0, "alpha", 2),
+                            (3, 0, 0, "alpha", 0),
+                            CO_FUTURE_UNICODE_LITERALS)
+
+barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
+                         (3, 9, 0, "alpha", 0),
+                         CO_FUTURE_BARRY_AS_BDFL)
diff --git a/lib-python/3.2/__phello__.foo.py b/lib-python/3.2/__phello__.foo.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/__phello__.foo.py
@@ -0,0 +1,1 @@
+# This file exists as a helper for the test.test_frozen module.
diff --git a/lib-python/3.2/_abcoll.py b/lib-python/3.2/_abcoll.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_abcoll.py
@@ -0,0 +1,623 @@
+# Copyright 2007 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Abstract Base Classes (ABCs) for collections, according to PEP 3119.
+
+DON'T USE THIS MODULE DIRECTLY!  The classes here should be imported
+via collections; they are defined here only to alleviate certain
+bootstrapping issues.  Unit tests are in test_collections.
+"""
+
+from abc import ABCMeta, abstractmethod
+import sys
+
+__all__ = ["Hashable", "Iterable", "Iterator",
+           "Sized", "Container", "Callable",
+           "Set", "MutableSet",
+           "Mapping", "MutableMapping",
+           "MappingView", "KeysView", "ItemsView", "ValuesView",
+           "Sequence", "MutableSequence",
+           "ByteString",
+           ]
+
+
+### collection related types which are not exposed through builtin ###
+## iterators ##
+bytes_iterator = type(iter(b''))
+bytearray_iterator = type(iter(bytearray()))
+#callable_iterator = ???
+dict_keyiterator = type(iter({}.keys()))
+dict_valueiterator = type(iter({}.values()))
+dict_itemiterator = type(iter({}.items()))
+list_iterator = type(iter([]))
+list_reverseiterator = type(iter(reversed([])))
+range_iterator = type(iter(range(0)))
+set_iterator = type(iter(set()))
+str_iterator = type(iter(""))
+tuple_iterator = type(iter(()))
+zip_iterator = type(iter(zip()))
+## views ##
+dict_keys = type({}.keys())
+dict_values = type({}.values())
+dict_items = type({}.items())
+## misc ##
+dict_proxy = type(type.__dict__)
+
+
+### ONE-TRICK PONIES ###
+
+class Hashable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __hash__(self):
+        return 0
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Hashable:
+            for B in C.__mro__:
+                if "__hash__" in B.__dict__:
+                    if B.__dict__["__hash__"]:
+                        return True
+                    break
+        return NotImplemented
+
+
+class Iterable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __iter__(self):
+        while False:
+            yield None
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Iterable:
+            if any("__iter__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Iterator(Iterable):
+
+    @abstractmethod
+    def __next__(self):
+        raise StopIteration
+
+    def __iter__(self):
+        return self
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Iterator:
+            if (any("__next__" in B.__dict__ for B in C.__mro__) and
+                any("__iter__" in B.__dict__ for B in C.__mro__)):
+                return True
+        return NotImplemented
+
+Iterator.register(bytes_iterator)
+Iterator.register(bytearray_iterator)
+#Iterator.register(callable_iterator)
+Iterator.register(dict_keyiterator)
+Iterator.register(dict_valueiterator)
+Iterator.register(dict_itemiterator)
+Iterator.register(list_iterator)
+Iterator.register(list_reverseiterator)
+Iterator.register(range_iterator)
+Iterator.register(set_iterator)
+Iterator.register(str_iterator)
+Iterator.register(tuple_iterator)
+Iterator.register(zip_iterator)
+
+class Sized(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __len__(self):
+        return 0
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Sized:
+            if any("__len__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Container(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __contains__(self, x):
+        return False
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Container:
+            if any("__contains__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Callable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __call__(self, *args, **kwds):
+        return False
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Callable:
+            if any("__call__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+### SETS ###
+
+
+class Set(Sized, Iterable, Container):
+
+    """A set is a finite, iterable container.
+
+    This class provides concrete generic implementations of all
+    methods except for __contains__, __iter__ and __len__.
+
+    To override the comparisons (presumably for speed, as the
+    semantics are fixed), all you have to do is redefine __le__ and
+    then the other operations will automatically follow suit.
+    """
+
+    def __le__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        if len(self) > len(other):
+            return False
+        for elem in self:
+            if elem not in other:
+                return False
+        return True
+
+    def __lt__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return len(self) < len(other) and self.__le__(other)
+
+    def __gt__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return other < self
+
+    def __ge__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return other <= self
+
+    def __eq__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return len(self) == len(other) and self.__le__(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    @classmethod
+    def _from_iterable(cls, it):
+        '''Construct an instance of the class from any iterable input.
+
+        Must override this method if the class constructor signature
+        does not accept an iterable for an input.
+        '''
+        return cls(it)
+
+    def __and__(self, other):
+        if not isinstance(other, Iterable):
+            return NotImplemented
+        return self._from_iterable(value for value in other if value in self)
+
+    def isdisjoint(self, other):
+        for value in other:
+            if value in self:
+                return False
+        return True
+
+    def __or__(self, other):
+        if not isinstance(other, Iterable):
+            return NotImplemented
+        chain = (e for s in (self, other) for e in s)
+        return self._from_iterable(chain)
+
+    def __sub__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return self._from_iterable(value for value in self
+                                   if value not in other)
+
+    def __xor__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return (self - other) | (other - self)
+
+    def _hash(self):
+        """Compute the hash value of a set.
+
+        Note that we don't define __hash__: not all sets are hashable.
+        But if you define a hashable set type, its __hash__ should
+        call this function.
+
+        This must be compatible __eq__.
+
+        All sets ought to compare equal if they contain the same
+        elements, regardless of how they are implemented, and
+        regardless of the order of the elements; so there's not much
+        freedom for __eq__ or __hash__.  We match the algorithm used
+        by the built-in frozenset type.
+        """
+        MAX = sys.maxsize
+        MASK = 2 * MAX + 1
+        n = len(self)
+        h = 1927868237 * (n + 1)
+        h &= MASK
+        for x in self:
+            hx = hash(x)
+            h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167
+            h &= MASK
+        h = h * 69069 + 907133923
+        h &= MASK
+        if h > MAX:
+            h -= MASK + 1
+        if h == -1:
+            h = 590923713
+        return h
+
+Set.register(frozenset)
+
+
+class MutableSet(Set):
+
+    @abstractmethod
+    def add(self, value):
+        """Add an element."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def discard(self, value):
+        """Remove an element.  Do not raise an exception if absent."""
+        raise NotImplementedError
+
+    def remove(self, value):
+        """Remove an element. If not a member, raise a KeyError."""
+        if value not in self:
+            raise KeyError(value)
+        self.discard(value)
+
+    def pop(self):
+        """Return the popped value.  Raise KeyError if empty."""
+        it = iter(self)
+        try:
+            value = next(it)
+        except StopIteration:
+            raise KeyError
+        self.discard(value)
+        return value
+
+    def clear(self):
+        """This is slow (creates N new iterators!) but effective."""
+        try:
+            while True:
+                self.pop()
+        except KeyError:
+            pass
+
+    def __ior__(self, it):
+        for value in it:
+            self.add(value)
+        return self
+
+    def __iand__(self, it):
+        for value in (self - it):
+            self.discard(value)
+        return self
+
+    def __ixor__(self, it):
+        if it is self:
+            self.clear()
+        else:
+            if not isinstance(it, Set):
+                it = self._from_iterable(it)
+            for value in it:
+                if value in self:
+                    self.discard(value)
+                else:
+                    self.add(value)
+        return self
+
+    def __isub__(self, it):
+        if it is self:
+            self.clear()
+        else:
+            for value in it:
+                self.discard(value)
+        return self
+
+MutableSet.register(set)
+
+
+### MAPPINGS ###
+
+
+class Mapping(Sized, Iterable, Container):
+
+    @abstractmethod
+    def __getitem__(self, key):
+        raise KeyError
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def __contains__(self, key):
+        try:
+            self[key]
+        except KeyError:
+            return False
+        else:
+            return True
+
+    def keys(self):
+        return KeysView(self)
+
+    def items(self):
+        return ItemsView(self)
+
+    def values(self):
+        return ValuesView(self)
+
+    def __eq__(self, other):
+        if not isinstance(other, Mapping):
+            return NotImplemented
+        return dict(self.items()) == dict(other.items())
+
+    def __ne__(self, other):
+        return not (self == other)
+
+
+class MappingView(Sized):
+
+    def __init__(self, mapping):
+        self._mapping = mapping
+
+    def __len__(self):
+        return len(self._mapping)
+
+    def __repr__(self):
+        return '{0.__class__.__name__}({0._mapping!r})'.format(self)
+
+
+class KeysView(MappingView, Set):
+
+    @classmethod
+    def _from_iterable(self, it):
+        return set(it)
+
+    def __contains__(self, key):
+        return key in self._mapping
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield key
+
+KeysView.register(dict_keys)
+
+
+class ItemsView(MappingView, Set):
+
+    @classmethod
+    def _from_iterable(self, it):
+        return set(it)
+
+    def __contains__(self, item):
+        key, value = item
+        try:
+            v = self._mapping[key]
+        except KeyError:
+            return False
+        else:
+            return v == value
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield (key, self._mapping[key])
+
+ItemsView.register(dict_items)
+
+
+class ValuesView(MappingView):
+
+    def __contains__(self, value):
+        for key in self._mapping:
+            if value == self._mapping[key]:
+                return True
+        return False
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield self._mapping[key]
+
+ValuesView.register(dict_values)
+
+
+class MutableMapping(Mapping):
+
+    @abstractmethod
+    def __setitem__(self, key, value):
+        raise KeyError
+
+    @abstractmethod
+    def __delitem__(self, key):
+        raise KeyError
+
+    __marker = object()
+
+    def pop(self, key, default=__marker):
+        try:
+            value = self[key]
+        except KeyError:
+            if default is self.__marker:
+                raise
+            return default
+        else:
+            del self[key]
+            return value
+
+    def popitem(self):
+        try:
+            key = next(iter(self))
+        except StopIteration:
+            raise KeyError
+        value = self[key]
+        del self[key]
+        return key, value
+
+    def clear(self):
+        try:
+            while True:
+                self.popitem()
+        except KeyError:
+            pass
+
+    def update(*args, **kwds):
+        if len(args) > 2:
+            raise TypeError("update() takes at most 2 positional "
+                            "arguments ({} given)".format(len(args)))
+        elif not args:
+            raise TypeError("update() takes at least 1 argument (0 given)")
+        self = args[0]
+        other = args[1] if len(args) >= 2 else ()
+
+        if isinstance(other, Mapping):
+            for key in other:
+                self[key] = other[key]
+        elif hasattr(other, "keys"):
+            for key in other.keys():
+                self[key] = other[key]
+        else:
+            for key, value in other:
+                self[key] = value
+        for key, value in kwds.items():
+            self[key] = value
+
+    def setdefault(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            self[key] = default
+        return default
+
+MutableMapping.register(dict)
+
+
+### SEQUENCES ###
+
+
+class Sequence(Sized, Iterable, Container):
+
+    """All the operations on a read-only sequence.
+
+    Concrete subclasses must override __new__ or __init__,
+    __getitem__, and __len__.
+    """
+
+    @abstractmethod
+    def __getitem__(self, index):
+        raise IndexError
+
+    def __iter__(self):
+        i = 0
+        try:
+            while True:
+                v = self[i]
+                yield v
+                i += 1
+        except IndexError:
+            return
+
+    def __contains__(self, value):
+        for v in self:
+            if v == value:
+                return True
+        return False
+
+    def __reversed__(self):
+        for i in reversed(range(len(self))):
+            yield self[i]
+
+    def index(self, value):
+        for i, v in enumerate(self):
+            if v == value:
+                return i
+        raise ValueError
+
+    def count(self, value):
+        return sum(1 for v in self if v == value)
+
+Sequence.register(tuple)
+Sequence.register(str)
+Sequence.register(range)
+
+
+class ByteString(Sequence):
+
+    """This unifies bytes and bytearray.
+
+    XXX Should add all their methods.
+    """
+
+ByteString.register(bytes)
+ByteString.register(bytearray)
+
+
+class MutableSequence(Sequence):
+
+    @abstractmethod
+    def __setitem__(self, index, value):
+        raise IndexError
+
+    @abstractmethod
+    def __delitem__(self, index):
+        raise IndexError
+
+    @abstractmethod
+    def insert(self, index, value):
+        raise IndexError
+
+    def append(self, value):
+        self.insert(len(self), value)
+
+    def reverse(self):
+        n = len(self)
+        for i in range(n//2):
+            self[i], self[n-i-1] = self[n-i-1], self[i]
+
+    def extend(self, values):
+        for v in values:
+            self.append(v)
+
+    def pop(self, index=-1):
+        v = self[index]
+        del self[index]
+        return v
+
+    def remove(self, value):
+        del self[self.index(value)]
+
+    def __iadd__(self, values):
+        self.extend(values)
+        return self
+
+MutableSequence.register(list)
+MutableSequence.register(bytearray)  # Multiply inheriting, see ByteString
diff --git a/lib-python/3.2/_compat_pickle.py b/lib-python/3.2/_compat_pickle.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_compat_pickle.py
@@ -0,0 +1,81 @@
+# This module is used to map the old Python 2 names to the new names used in
+# Python 3 for the pickle module.  This needed to make pickle streams
+# generated with Python 2 loadable by Python 3.
+
+# This is a copy of lib2to3.fixes.fix_imports.MAPPING.  We cannot import
+# lib2to3 and use the mapping defined there, because lib2to3 uses pickle.
+# Thus, this could cause the module to be imported recursively.
+IMPORT_MAPPING = {
+    'StringIO':  'io',
+    'cStringIO': 'io',
+    'cPickle': 'pickle',
+    '__builtin__' : 'builtins',
+    'copy_reg': 'copyreg',
+    'Queue': 'queue',
+    'SocketServer': 'socketserver',
+    'ConfigParser': 'configparser',
+    'repr': 'reprlib',
+    'FileDialog': 'tkinter.filedialog',
+    'tkFileDialog': 'tkinter.filedialog',
+    'SimpleDialog': 'tkinter.simpledialog',
+    'tkSimpleDialog': 'tkinter.simpledialog',
+    'tkColorChooser': 'tkinter.colorchooser',
+    'tkCommonDialog': 'tkinter.commondialog',
+    'Dialog': 'tkinter.dialog',
+    'Tkdnd': 'tkinter.dnd',
+    'tkFont': 'tkinter.font',
+    'tkMessageBox': 'tkinter.messagebox',
+    'ScrolledText': 'tkinter.scrolledtext',
+    'Tkconstants': 'tkinter.constants',
+    'Tix': 'tkinter.tix',
+    'ttk': 'tkinter.ttk',
+    'Tkinter': 'tkinter',
+    'markupbase': '_markupbase',
+    '_winreg': 'winreg',
+    'thread': '_thread',
+    'dummy_thread': '_dummy_thread',
+    'dbhash': 'dbm.bsd',
+    'dumbdbm': 'dbm.dumb',
+    'dbm': 'dbm.ndbm',
+    'gdbm': 'dbm.gnu',
+    'xmlrpclib': 'xmlrpc.client',
+    'DocXMLRPCServer': 'xmlrpc.server',
+    'SimpleXMLRPCServer': 'xmlrpc.server',
+    'httplib': 'http.client',
+    'htmlentitydefs' : 'html.entities',
+    'HTMLParser' : 'html.parser',
+    'Cookie': 'http.cookies',
+    'cookielib': 'http.cookiejar',
+    'BaseHTTPServer': 'http.server',
+    'SimpleHTTPServer': 'http.server',
+    'CGIHTTPServer': 'http.server',
+    'test.test_support': 'test.support',
+    'commands': 'subprocess',
+    'UserString' : 'collections',
+    'UserList' : 'collections',
+    'urlparse' : 'urllib.parse',
+    'robotparser' : 'urllib.robotparser',
+    'whichdb': 'dbm',
+    'anydbm': 'dbm'
+}
+
+
+# This contains rename rules that are easy to handle.  We ignore the more
+# complex stuff (e.g. mapping the names in the urllib and types modules).
+# These rules should be run before import names are fixed.
+NAME_MAPPING = {
+    ('__builtin__', 'xrange'):     ('builtins', 'range'),
+    ('__builtin__', 'reduce'):     ('functools', 'reduce'),
+    ('__builtin__', 'intern'):     ('sys', 'intern'),
+    ('__builtin__', 'unichr'):     ('builtins', 'chr'),
+    ('__builtin__', 'basestring'): ('builtins', 'str'),
+    ('__builtin__', 'long'):       ('builtins', 'int'),
+    ('itertools', 'izip'):         ('builtins', 'zip'),
+    ('itertools', 'imap'):         ('builtins', 'map'),
+    ('itertools', 'ifilter'):      ('builtins', 'filter'),
+    ('itertools', 'ifilterfalse'): ('itertools', 'filterfalse'),
+}
+
+# Same, but for 3.x to 2.x
+REVERSE_IMPORT_MAPPING = dict((v, k) for (k, v) in IMPORT_MAPPING.items())
+REVERSE_NAME_MAPPING = dict((v, k) for (k, v) in NAME_MAPPING.items())
diff --git a/lib-python/3.2/_dummy_thread.py b/lib-python/3.2/_dummy_thread.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_dummy_thread.py
@@ -0,0 +1,155 @@
+"""Drop-in replacement for the thread module.
+
+Meant to be used as a brain-dead substitute so that threaded code does
+not need to be rewritten for when the thread module is not present.
+
+Suggested usage is::
+
+    try:
+        import _thread
+    except ImportError:
+        import _dummy_thread as _thread
+
+"""
+# Exports only things specified by thread documentation;
+# skipping obsolete synonyms allocate(), start_new(), exit_thread().
+__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
+           'interrupt_main', 'LockType']
+
+# A dummy value
+TIMEOUT_MAX = 2**31
+
+# NOTE: this module can be imported early in the extension building process,
+# and so top level imports of other modules should be avoided.  Instead, all
+# imports are done when needed on a function-by-function basis.  Since threads
+# are disabled, the import lock should not be an issue anyway (??).
+
+class error(Exception):
+    """Dummy implementation of _thread.error."""
+
+    def __init__(self, *args):
+        self.args = args
+
+def start_new_thread(function, args, kwargs={}):
+    """Dummy implementation of _thread.start_new_thread().
+
+    Compatibility is maintained by making sure that ``args`` is a
+    tuple and ``kwargs`` is a dictionary.  If an exception is raised
+    and it is SystemExit (which can be done by _thread.exit()) it is
+    caught and nothing is done; all other exceptions are printed out
+    by using traceback.print_exc().
+
+    If the executed function calls interrupt_main the KeyboardInterrupt will be
+    raised when the function returns.
+
+    """
+    if type(args) != type(tuple()):
+        raise TypeError("2nd arg must be a tuple")
+    if type(kwargs) != type(dict()):
+        raise TypeError("3rd arg must be a dict")
+    global _main
+    _main = False
+    try:
+        function(*args, **kwargs)
+    except SystemExit:
+        pass
+    except:
+        import traceback
+        traceback.print_exc()
+    _main = True
+    global _interrupt
+    if _interrupt:
+        _interrupt = False
+        raise KeyboardInterrupt
+
+def exit():
+    """Dummy implementation of _thread.exit()."""
+    raise SystemExit
+
+def get_ident():
+    """Dummy implementation of _thread.get_ident().
+
+    Since this module should only be used when _threadmodule is not
+    available, it is safe to assume that the current process is the
+    only thread.  Thus a constant can be safely returned.
+    """
+    return -1
+
+def allocate_lock():
+    """Dummy implementation of _thread.allocate_lock()."""
+    return LockType()
+
+def stack_size(size=None):
+    """Dummy implementation of _thread.stack_size()."""
+    if size is not None:
+        raise error("setting thread stack size not supported")
+    return 0
+
+class LockType(object):
+    """Class implementing dummy implementation of _thread.LockType.
+
+    Compatibility is maintained by maintaining self.locked_status
+    which is a boolean that stores the state of the lock.  Pickling of
+    the lock, though, should not be done since if the _thread module is
+    then used with an unpickled ``lock()`` from here problems could
+    occur from this class not having atomic methods.
+
+    """
+
+    def __init__(self):
+        self.locked_status = False
+
+    def acquire(self, waitflag=None, timeout=-1):
+        """Dummy implementation of acquire().
+
+        For blocking calls, self.locked_status is automatically set to
+        True and returned appropriately based on value of
+        ``waitflag``.  If it is non-blocking, then the value is
+        actually checked and not set if it is already acquired.  This
+        is all done so that threading.Condition's assert statements
+        aren't triggered and throw a little fit.
+
+        """
+        if waitflag is None or waitflag:
+            self.locked_status = True
+            return True
+        else:
+            if not self.locked_status:
+                self.locked_status = True
+                return True
+            else:
+                if timeout > 0:
+                    import time
+                    time.sleep(timeout)
+                return False
+
+    __enter__ = acquire
+
+    def __exit__(self, typ, val, tb):
+        self.release()
+
+    def release(self):
+        """Release the dummy lock."""
+        # XXX Perhaps shouldn't actually bother to test?  Could lead
+        #     to problems for complex, threaded code.
+        if not self.locked_status:
+            raise error
+        self.locked_status = False
+        return True
+
+    def locked(self):
+        return self.locked_status
+
+# Used to signal that interrupt_main was called in a "thread"
+_interrupt = False
+# True when not executing in a "thread"
+_main = True
+
+def interrupt_main():
+    """Set _interrupt flag to True to have start_new_thread raise
+    KeyboardInterrupt upon exiting."""
+    if _main:
+        raise KeyboardInterrupt
+    else:
+        global _interrupt
+        _interrupt = True
diff --git a/lib-python/3.2/_markupbase.py b/lib-python/3.2/_markupbase.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_markupbase.py
@@ -0,0 +1,395 @@
+"""Shared support for scanning document type declarations in HTML and XHTML.
+
+This module is used as a foundation for the html.parser module.  It has no
+documented public API and should not be used directly.
+
+"""
+
+import re
+
+_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
+_declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
+_commentclose = re.compile(r'--\s*>')
+_markedsectionclose = re.compile(r']\s*]\s*>')
+
+# An analysis of the MS-Word extensions is available at
+# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
+
+_msmarkedsectionclose = re.compile(r']\s*>')
+
+del re
+
+
+class ParserBase:
+    """Parser base class which provides some common support methods used
+    by the SGML/HTML and XHTML parsers."""
+
+    def __init__(self):
+        if self.__class__ is ParserBase:
+            raise RuntimeError(
+                "_markupbase.ParserBase must be subclassed")
+
+    def error(self, message):
+        raise NotImplementedError(
+            "subclasses of ParserBase must override error()")
+
+    def reset(self):
+        self.lineno = 1
+        self.offset = 0
+
+    def getpos(self):
+        """Return current line number and offset."""
+        return self.lineno, self.offset
+
+    # Internal -- update line number and offset.  This should be
+    # called for each piece of data exactly once, in order -- in other
+    # words the concatenation of all the input strings to this
+    # function should be exactly the entire input.
+    def updatepos(self, i, j):
+        if i >= j:
+            return j
+        rawdata = self.rawdata
+        nlines = rawdata.count("\n", i, j)
+        if nlines:
+            self.lineno = self.lineno + nlines
+            pos = rawdata.rindex("\n", i, j) # Should not fail
+            self.offset = j-(pos+1)
+        else:
+            self.offset = self.offset + j-i
+        return j
+
+    _decl_otherchars = ''
+
+    # Internal -- parse declaration (for use by subclasses).
+    def parse_declaration(self, i):
+        # This is some sort of declaration; in "HTML as
+        # deployed," this should only be the document type
+        # declaration ("<!DOCTYPE html...>").
+        # ISO 8879:1986, however, has more complex
+        # declaration syntax for elements in <!...>, including:
+        # --comment--
+        # [marked section]
+        # name in the following list: ENTITY, DOCTYPE, ELEMENT,
+        # ATTLIST, NOTATION, SHORTREF, USEMAP,
+        # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
+        rawdata = self.rawdata
+        j = i + 2
+        assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
+        if rawdata[j:j+1] == ">":
+            # the empty comment <!>
+            return j + 1
+        if rawdata[j:j+1] in ("-", ""):
+            # Start of comment followed by buffer boundary,
+            # or just a buffer boundary.
+            return -1
+        # A simple, practical version could look like: ((name|stringlit) S*) + '>'
+        n = len(rawdata)
+        if rawdata[j:j+2] == '--': #comment
+            # Locate --.*-- as the body of the comment
+            return self.parse_comment(i)
+        elif rawdata[j] == '[': #marked section
+            # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
+            # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
+            # Note that this is extended by Microsoft Office "Save as Web" function
+            # to include [if...] and [endif].
+            return self.parse_marked_section(i)
+        else: #all other declaration elements
+            decltype, j = self._scan_name(j, i)
+        if j < 0:
+            return j
+        if decltype == "doctype":
+            self._decl_otherchars = ''
+        while j < n:
+            c = rawdata[j]
+            if c == ">":
+                # end of declaration syntax
+                data = rawdata[i+2:j]
+                if decltype == "doctype":
+                    self.handle_decl(data)
+                else:
+                    # According to the HTML5 specs sections "8.2.4.44 Bogus
+                    # comment state" and "8.2.4.45 Markup declaration open
+                    # state", a comment token should be emitted.
+                    # Calling unknown_decl provides more flexibility though.
+                    self.unknown_decl(data)
+                return j + 1
+            if c in "\"'":
+                m = _declstringlit_match(rawdata, j)
+                if not m:
+                    return -1 # incomplete
+                j = m.end()
+            elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
+                name, j = self._scan_name(j, i)
+            elif c in self._decl_otherchars:
+                j = j + 1
+            elif c == "[":
+                # this could be handled in a separate doctype parser
+                if decltype == "doctype":
+                    j = self._parse_doctype_subset(j + 1, i)
+                elif decltype in {"attlist", "linktype", "link", "element"}:
+                    # must tolerate []'d groups in a content model in an element declaration
+                    # also in data attribute specifications of attlist declaration
+                    # also link type declaration subsets in linktype declarations
+                    # also link attribute specification lists in link declarations
+                    self.error("unsupported '[' char in %s declaration" % decltype)
+                else:
+                    self.error("unexpected '[' char in declaration")
+            else:
+                self.error(
+                    "unexpected %r char in declaration" % rawdata[j])
+            if j < 0:
+                return j
+        return -1 # incomplete
+
+    # Internal -- parse a marked section
+    # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
+    def parse_marked_section(self, i, report=1):
+        rawdata= self.rawdata
+        assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()"
+        sectName, j = self._scan_name( i+3, i )
+        if j < 0:
+            return j
+        if sectName in {"temp", "cdata", "ignore", "include", "rcdata"}:
+            # look for standard ]]> ending
+            match= _markedsectionclose.search(rawdata, i+3)
+        elif sectName in {"if", "else", "endif"}:
+            # look for MS Office ]> ending
+            match= _msmarkedsectionclose.search(rawdata, i+3)
+        else:
+            self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
+        if not match:
+            return -1
+        if report:
+            j = match.start(0)
+            self.unknown_decl(rawdata[i+3: j])
+        return match.end(0)
+
+    # Internal -- parse comment, return length or -1 if not terminated
+    def parse_comment(self, i, report=1):
+        rawdata = self.rawdata
+        if rawdata[i:i+4] != '<!--':
+            self.error('unexpected call to parse_comment()')
+        match = _commentclose.search(rawdata, i+4)
+        if not match:
+            return -1
+        if report:
+            j = match.start(0)
+            self.handle_comment(rawdata[i+4: j])
+        return match.end(0)
+
+    # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
+    # returning the index just past any whitespace following the trailing ']'.
+    def _parse_doctype_subset(self, i, declstartpos):
+        rawdata = self.rawdata
+        n = len(rawdata)
+        j = i
+        while j < n:
+            c = rawdata[j]
+            if c == "<":
+                s = rawdata[j:j+2]
+                if s == "<":
+                    # end of buffer; incomplete
+                    return -1
+                if s != "<!":
+                    self.updatepos(declstartpos, j + 1)
+                    self.error("unexpected char in internal subset (in %r)" % s)
+                if (j + 2) == n:
+                    # end of buffer; incomplete
+                    return -1
+                if (j + 4) > n:
+                    # end of buffer; incomplete
+                    return -1
+                if rawdata[j:j+4] == "<!--":
+                    j = self.parse_comment(j, report=0)
+                    if j < 0:
+                        return j
+                    continue
+                name, j = self._scan_name(j + 2, declstartpos)
+                if j == -1:
+                    return -1
+                if name not in {"attlist", "element", "entity", "notation"}:
+                    self.updatepos(declstartpos, j + 2)
+                    self.error(
+                        "unknown declaration %r in internal subset" % name)
+                # handle the individual names
+                meth = getattr(self, "_parse_doctype_" + name)
+                j = meth(j, declstartpos)
+                if j < 0:
+                    return j
+            elif c == "%":
+                # parameter entity reference
+                if (j + 1) == n:
+                    # end of buffer; incomplete
+                    return -1
+                s, j = self._scan_name(j + 1, declstartpos)
+                if j < 0:
+                    return j
+                if rawdata[j] == ";":
+                    j = j + 1
+            elif c == "]":
+                j = j + 1
+                while j < n and rawdata[j].isspace():
+                    j = j + 1
+                if j < n:
+                    if rawdata[j] == ">":
+                        return j
+                    self.updatepos(declstartpos, j)
+                    self.error("unexpected char after internal subset")
+                else:
+                    return -1
+            elif c.isspace():
+                j = j + 1
+            else:
+                self.updatepos(declstartpos, j)
+                self.error("unexpected char %r in internal subset" % c)
+        # end of buffer reached
+        return -1
+
+    # Internal -- scan past <!ELEMENT declarations
+    def _parse_doctype_element(self, i, declstartpos):
+        name, j = self._scan_name(i, declstartpos)
+        if j == -1:
+            return -1
+        # style content model; just skip until '>'
+        rawdata = self.rawdata
+        if '>' in rawdata[j:]:
+            return rawdata.find(">", j) + 1
+        return -1
+
+    # Internal -- scan past <!ATTLIST declarations
+    def _parse_doctype_attlist(self, i, declstartpos):
+        rawdata = self.rawdata
+        name, j = self._scan_name(i, declstartpos)
+        c = rawdata[j:j+1]
+        if c == "":
+            return -1
+        if c == ">":
+            return j + 1
+        while 1:
+            # scan a series of attribute descriptions; simplified:
+            #   name type [value] [#constraint]
+            name, j = self._scan_name(j, declstartpos)
+            if j < 0:
+                return j
+            c = rawdata[j:j+1]
+            if c == "":
+                return -1
+            if c == "(":
+                # an enumerated type; look for ')'
+                if ")" in rawdata[j:]:
+                    j = rawdata.find(")", j) + 1
+                else:
+                    return -1
+                while rawdata[j:j+1].isspace():
+                    j = j + 1
+                if not rawdata[j:]:
+                    # end of buffer, incomplete
+                    return -1
+            else:
+                name, j = self._scan_name(j, declstartpos)
+            c = rawdata[j:j+1]
+            if not c:
+                return -1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if m:
+                    j = m.end()
+                else:
+                    return -1
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+            if c == "#":
+                if rawdata[j:] == "#":
+                    # end of buffer
+                    return -1
+                name, j = self._scan_name(j + 1, declstartpos)
+                if j < 0:
+                    return j
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+            if c == '>':
+                # all done
+                return j + 1
+
+    # Internal -- scan past <!NOTATION declarations
+    def _parse_doctype_notation(self, i, declstartpos):
+        name, j = self._scan_name(i, declstartpos)
+        if j < 0:
+            return j
+        rawdata = self.rawdata
+        while 1:
+            c = rawdata[j:j+1]
+            if not c:
+                # end of buffer; incomplete
+                return -1
+            if c == '>':
+                return j + 1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if not m:
+                    return -1
+                j = m.end()
+            else:
+                name, j = self._scan_name(j, declstartpos)
+                if j < 0:
+                    return j
+
+    # Internal -- scan past <!ENTITY declarations
+    def _parse_doctype_entity(self, i, declstartpos):
+        rawdata = self.rawdata
+        if rawdata[i:i+1] == "%":
+            j = i + 1
+            while 1:
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+                if c.isspace():
+                    j = j + 1
+                else:
+                    break
+        else:
+            j = i
+        name, j = self._scan_name(j, declstartpos)
+        if j < 0:
+            return j
+        while 1:
+            c = self.rawdata[j:j+1]
+            if not c:
+                return -1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if m:
+                    j = m.end()
+                else:
+                    return -1    # incomplete
+            elif c == ">":
+                return j + 1
+            else:
+                name, j = self._scan_name(j, declstartpos)
+                if j < 0:
+                    return j
+
+    # Internal -- scan a name token and the new position and the token, or
+    # return -1 if we've reached the end of the buffer.
+    def _scan_name(self, i, declstartpos):
+        rawdata = self.rawdata
+        n = len(rawdata)
+        if i == n:
+            return None, -1
+        m = _declname_match(rawdata, i)
+        if m:
+            s = m.group()
+            name = s.strip()
+            if (i + len(s)) == n:
+                return None, -1  # end of buffer
+            return name.lower(), m.end()
+        else:
+            self.updatepos(declstartpos, i)
+            self.error("expected name token at %r"
+                       % rawdata[declstartpos:declstartpos+20])
+
+    # To be overridden -- handlers for unknown objects
+    def unknown_decl(self, data):
+        pass
diff --git a/lib-python/3.2/_pyio.py b/lib-python/3.2/_pyio.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_pyio.py
@@ -0,0 +1,2033 @@
+"""
+Python implementation of the io module.
+"""
+
+import os
+import abc
+import codecs
+import warnings
+import errno
+# Import _thread instead of threading to reduce startup cost
+try:
+    from _thread import allocate_lock as Lock
+except ImportError:
+    from _dummy_thread import allocate_lock as Lock
+
+import io
+from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
+from errno import EINTR
+
+# open() uses st_blksize whenever we can
+DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
+
+# NOTE: Base classes defined here are registered with the "official" ABCs
+# defined in io.py. We don't use real inheritance though, because we don't
+# want to inherit the C implementations.
+
+
+class BlockingIOError(IOError):
+
+    """Exception raised when I/O would block on a non-blocking I/O stream."""
+
+    def __init__(self, errno, strerror, characters_written=0):
+        super().__init__(errno, strerror)
+        if not isinstance(characters_written, int):
+            raise TypeError("characters_written must be a integer")
+        self.characters_written = characters_written
+
+
+def open(file, mode="r", buffering=-1, encoding=None, errors=None,
+         newline=None, closefd=True):
+
+    r"""Open file and return a stream.  Raise IOError upon failure.
+
+    file is either a text or byte string giving the name (and the path
+    if the file isn't in the current working directory) of the file to
+    be opened or an integer file descriptor of the file to be
+    wrapped. (If a file descriptor is given, it is closed when the
+    returned I/O object is closed, unless closefd is set to False.)
+
+    mode is an optional string that specifies the mode in which the file
+    is opened. It defaults to 'r' which means open for reading in text
+    mode.  Other common values are 'w' for writing (truncating the file if
+    it already exists), and 'a' for appending (which on some Unix systems,
+    means that all writes append to the end of the file regardless of the
+    current seek position). In text mode, if encoding is not specified the
+    encoding used is platform dependent. (For reading and writing raw
+    bytes use binary mode and leave encoding unspecified.) The available
+    modes are:
+
+    ========= ===============================================================
+    Character Meaning
+    --------- ---------------------------------------------------------------
+    'r'       open for reading (default)
+    'w'       open for writing, truncating the file first
+    'a'       open for writing, appending to the end of the file if it exists
+    'b'       binary mode
+    't'       text mode (default)
+    '+'       open a disk file for updating (reading and writing)
+    'U'       universal newline mode (for backwards compatibility; unneeded
+              for new code)
+    ========= ===============================================================
+
+    The default mode is 'rt' (open for reading text). For binary random
+    access, the mode 'w+b' opens and truncates the file to 0 bytes, while
+    'r+b' opens the file without truncation.
+
+    Python distinguishes between files opened in binary and text modes,
+    even when the underlying operating system doesn't. Files opened in
+    binary mode (appending 'b' to the mode argument) return contents as
+    bytes objects without any decoding. In text mode (the default, or when
+    't' is appended to the mode argument), the contents of the file are
+    returned as strings, the bytes having been first decoded using a
+    platform-dependent encoding or using the specified encoding if given.
+
+    buffering is an optional integer used to set the buffering policy.
+    Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
+    line buffering (only usable in text mode), and an integer > 1 to indicate
+    the size of a fixed-size chunk buffer.  When no buffering argument is
+    given, the default buffering policy works as follows:
+
+    * Binary files are buffered in fixed-size chunks; the size of the buffer
+      is chosen using a heuristic trying to determine the underlying device's
+      "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
+      On many systems, the buffer will typically be 4096 or 8192 bytes long.
+
+    * "Interactive" text files (files for which isatty() returns True)
+      use line buffering.  Other text files use the policy described above
+      for binary files.
+
+    encoding is the str name of the encoding used to decode or encode the
+    file. This should only be used in text mode. The default encoding is
+    platform dependent, but any encoding supported by Python can be
+    passed.  See the codecs module for the list of supported encodings.
+
+    errors is an optional string that specifies how encoding errors are to
+    be handled---this argument should not be used in binary mode. Pass
+    'strict' to raise a ValueError exception if there is an encoding error
+    (the default of None has the same effect), or pass 'ignore' to ignore
+    errors. (Note that ignoring encoding errors can lead to data loss.)
+    See the documentation for codecs.register for a list of the permitted
+    encoding error strings.
+
+    newline is a string controlling how universal newlines works (it only
+    applies to text mode). It can be None, '', '\n', '\r', and '\r\n'.  It works
+    as follows:
+
+    * On input, if newline is None, universal newlines mode is
+      enabled. Lines in the input can end in '\n', '\r', or '\r\n', and
+      these are translated into '\n' before being returned to the
+      caller. If it is '', universal newline mode is enabled, but line
+      endings are returned to the caller untranslated. If it has any of
+      the other legal values, input lines are only terminated by the given
+      string, and the line ending is returned to the caller untranslated.
+
+    * On output, if newline is None, any '\n' characters written are
+      translated to the system default line separator, os.linesep. If
+      newline is '', no translation takes place. If newline is any of the
+      other legal values, any '\n' characters written are translated to
+      the given string.
+
+    closedfd is a bool. If closefd is False, the underlying file descriptor will
+    be kept open when the file is closed. This does not work when a file name is
+    given and must be True in that case.
+
+    open() returns a file object whose type depends on the mode, and
+    through which the standard file operations such as reading and writing
+    are performed. When open() is used to open a file in a text mode ('w',
+    'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open
+    a file in a binary mode, the returned class varies: in read binary
+    mode, it returns a BufferedReader; in write binary and append binary
+    modes, it returns a BufferedWriter, and in read/write mode, it returns
+    a BufferedRandom.
+
+    It is also possible to use a string or bytearray as a file for both
+    reading and writing. For strings StringIO can be used like a file
+    opened in a text mode, and for bytes a BytesIO can be used like a file
+    opened in a binary mode.
+    """
+    if not isinstance(file, (str, bytes, int)):
+        raise TypeError("invalid file: %r" % file)
+    if not isinstance(mode, str):
+        raise TypeError("invalid mode: %r" % mode)
+    if not isinstance(buffering, int):
+        raise TypeError("invalid buffering: %r" % buffering)
+    if encoding is not None and not isinstance(encoding, str):
+        raise TypeError("invalid encoding: %r" % encoding)
+    if errors is not None and not isinstance(errors, str):
+        raise TypeError("invalid errors: %r" % errors)
+    modes = set(mode)
+    if modes - set("arwb+tU") or len(mode) > len(modes):
+        raise ValueError("invalid mode: %r" % mode)
+    reading = "r" in modes
+    writing = "w" in modes
+    appending = "a" in modes
+    updating = "+" in modes
+    text = "t" in modes
+    binary = "b" in modes
+    if "U" in modes:
+        if writing or appending:
+            raise ValueError("can't use U and writing mode at once")
+        reading = True
+    if text and binary:
+        raise ValueError("can't have text and binary mode at once")
+    if reading + writing + appending > 1:
+        raise ValueError("can't have read/write/append mode at once")
+    if not (reading or writing or appending):
+        raise ValueError("must have exactly one of read/write/append mode")
+    if binary and encoding is not None:
+        raise ValueError("binary mode doesn't take an encoding argument")
+    if binary and errors is not None:
+        raise ValueError("binary mode doesn't take an errors argument")
+    if binary and newline is not None:
+        raise ValueError("binary mode doesn't take a newline argument")
+    raw = FileIO(file,
+                 (reading and "r" or "") +
+                 (writing and "w" or "") +
+                 (appending and "a" or "") +
+                 (updating and "+" or ""),
+                 closefd)
+    line_buffering = False
+    if buffering == 1 or buffering < 0 and raw.isatty():
+        buffering = -1
+        line_buffering = True
+    if buffering < 0:
+        buffering = DEFAULT_BUFFER_SIZE
+        try:
+            bs = os.fstat(raw.fileno()).st_blksize
+        except (os.error, AttributeError):
+            pass
+        else:
+            if bs > 1:
+                buffering = bs
+    if buffering < 0:
+        raise ValueError("invalid buffering size")
+    if buffering == 0:
+        if binary:
+            return raw
+        raise ValueError("can't have unbuffered text I/O")
+    if updating:
+        buffer = BufferedRandom(raw, buffering)
+    elif writing or appending:
+        buffer = BufferedWriter(raw, buffering)
+    elif reading:
+        buffer = BufferedReader(raw, buffering)
+    else:
+        raise ValueError("unknown mode: %r" % mode)
+    if binary:
+        return buffer
+    text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
+    text.mode = mode
+    return text
+
+
+class DocDescriptor:
+    """Helper for builtins.open.__doc__
+    """
+    def __get__(self, obj, typ):
+        return (
+            "open(file, mode='r', buffering=-1, encoding=None, "
+                 "errors=None, newline=None, closefd=True)\n\n" +
+            open.__doc__)
+
+class OpenWrapper:
+    """Wrapper for builtins.open
+
+    Trick so that open won't become a bound method when stored
+    as a class variable (as dbm.dumb does).
+
+    See initstdio() in Python/pythonrun.c.
+    """
+    __doc__ = DocDescriptor()
+
+    def __new__(cls, *args, **kwargs):
+        return open(*args, **kwargs)
+
+
+# In normal operation, both `UnsupportedOperation`s should be bound to the
+# same object.
+try:
+    UnsupportedOperation = io.UnsupportedOperation
+except AttributeError:
+    class UnsupportedOperation(ValueError, IOError):
+        pass
+
+
+class IOBase(metaclass=abc.ABCMeta):
+
+    """The abstract base class for all I/O classes, acting on streams of
+    bytes. There is no public constructor.
+
+    This class provides dummy implementations for many methods that
+    derived classes can override selectively; the default implementations
+    represent a file that cannot be read, written or seeked.
+
+    Even though IOBase does not declare read, readinto, or write because
+    their signatures will vary, implementations and clients should
+    consider those methods part of the interface. Also, implementations
+    may raise UnsupportedOperation when operations they do not support are
+    called.
+
+    The basic type used for binary data read from or written to a file is
+    bytes. bytearrays are accepted too, and in some cases (such as
+    readinto) needed. Text I/O classes work with str data.
+
+    Note that calling any method (even inquiries) on a closed stream is
+    undefined. Implementations may raise IOError in this case.
+
+    IOBase (and its subclasses) support the iterator protocol, meaning
+    that an IOBase object can be iterated over yielding the lines in a
+    stream.
+
+    IOBase also supports the :keyword:`with` statement. In this example,
+    fp is closed after the suite of the with statement is complete:
+
+    with open('spam.txt', 'r') as fp:
+        fp.write('Spam and eggs!')
+    """
+
+    ### Internal ###
+
+    def _unsupported(self, name):
+        """Internal: raise an IOError exception for unsupported operations."""
+        raise UnsupportedOperation("%s.%s() not supported" %
+                                   (self.__class__.__name__, name))
+
+    ### Positioning ###
+
+    def seek(self, pos, whence=0):
+        """Change stream position.
+
+        Change the stream position to byte offset offset. offset is
+        interpreted relative to the position indicated by whence.  Values
+        for whence are ints:
+
+        * 0 -- start of stream (the default); offset should be zero or positive
+        * 1 -- current stream position; offset may be negative
+        * 2 -- end of stream; offset is usually negative
+
+        Return an int indicating the new absolute position.
+        """
+        self._unsupported("seek")
+
+    def tell(self):
+        """Return an int indicating the current stream position."""
+        return self.seek(0, 1)
+
+    def truncate(self, pos=None):
+        """Truncate file to size bytes.
+
+        Size defaults to the current IO position as reported by tell().  Return
+        the new size.
+        """
+        self._unsupported("truncate")
+
+    ### Flush and close ###
+
+    def flush(self):
+        """Flush write buffers, if applicable.
+
+        This is not implemented for read-only and non-blocking streams.
+        """
+        self._checkClosed()
+        # XXX Should this return the number of bytes written???
+
+    __closed = False
+
+    def close(self):
+        """Flush and close the IO object.
+
+        This method has no effect if the file is already closed.
+        """
+        if not self.__closed:
+            self.flush()
+            self.__closed = True
+
+    def __del__(self):
+        """Destructor.  Calls close()."""
+        # The try/except block is in case this is called at program
+        # exit time, when it's possible that globals have already been
+        # deleted, and then the close() call might fail.  Since
+        # there's nothing we can do about such failures and they annoy
+        # the end users, we suppress the traceback.
+        try:
+            self.close()
+        except:
+            pass
+
+    ### Inquiries ###
+
+    def seekable(self):
+        """Return a bool indicating whether object supports random access.
+
+        If False, seek(), tell() and truncate() will raise UnsupportedOperation.
+        This method may need to do a test seek().
+        """
+        return False
+
+    def _checkSeekable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not seekable
+        """
+        if not self.seekable():
+            raise UnsupportedOperation("File or stream is not seekable."
+                                       if msg is None else msg)
+
+    def readable(self):
+        """Return a bool indicating whether object was opened for reading.
+
+        If False, read() will raise UnsupportedOperation.
+        """
+        return False
+
+    def _checkReadable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not readable
+        """
+        if not self.readable():
+            raise UnsupportedOperation("File or stream is not readable."
+                                       if msg is None else msg)
+
+    def writable(self):
+        """Return a bool indicating whether object was opened for writing.
+
+        If False, write() and truncate() will raise UnsupportedOperation.
+        """
+        return False
+
+    def _checkWritable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not writable
+        """
+        if not self.writable():
+            raise UnsupportedOperation("File or stream is not writable."
+                                       if msg is None else msg)
+
+    @property
+    def closed(self):
+        """closed: bool.  True iff the file has been closed.
+
+        For backwards compatibility, this is a property, not a predicate.
+        """
+        return self.__closed
+
+    def _checkClosed(self, msg=None):
+        """Internal: raise an ValueError if file is closed
+        """
+        if self.closed:
+            raise ValueError("I/O operation on closed file."
+                             if msg is None else msg)
+
+    ### Context manager ###
+
+    def __enter__(self):  # That's a forward reference
+        """Context management protocol.  Returns self (an instance of IOBase)."""
+        self._checkClosed()
+        return self
+
+    def __exit__(self, *args):
+        """Context management protocol.  Calls close()"""
+        self.close()
+
+    ### Lower-level APIs ###
+
+    # XXX Should these be present even if unimplemented?
+
+    def fileno(self):
+        """Returns underlying file descriptor (an int) if one exists.
+
+        An IOError is raised if the IO object does not use a file descriptor.
+        """
+        self._unsupported("fileno")
+
+    def isatty(self):
+        """Return a bool indicating whether this is an 'interactive' stream.
+
+        Return False if it can't be determined.
+        """
+        self._checkClosed()
+        return False
+
+    ### Readline[s] and writelines ###
+
+    def readline(self, limit=-1):
+        r"""Read and return a line of bytes from the stream.
+
+        If limit is specified, at most limit bytes will be read.
+        Limit should be an int.
+
+        The line terminator is always b'\n' for binary files; for text
+        files, the newlines argument to open can be used to select the line
+        terminator(s) recognized.
+        """
+        # For backwards compatibility, a (slowish) readline().
+        if hasattr(self, "peek"):
+            def nreadahead():
+                readahead = self.peek(1)
+                if not readahead:
+                    return 1
+                n = (readahead.find(b"\n") + 1) or len(readahead)
+                if limit >= 0:
+                    n = min(n, limit)
+                return n
+        else:
+            def nreadahead():
+                return 1
+        if limit is None:
+            limit = -1
+        elif not isinstance(limit, int):
+            raise TypeError("limit must be an integer")
+        res = bytearray()
+        while limit < 0 or len(res) < limit:
+            b = self.read(nreadahead())
+            if not b:
+                break
+            res += b
+            if res.endswith(b"\n"):
+                break
+        return bytes(res)
+
+    def __iter__(self):
+        self._checkClosed()
+        return self
+
+    def __next__(self):
+        line = self.readline()
+        if not line:
+            raise StopIteration
+        return line
+
+    def readlines(self, hint=None):
+        """Return a list of lines from the stream.
+
+        hint can be specified to control the number of lines read: no more
+        lines will be read if the total size (in bytes/characters) of all
+        lines so far exceeds hint.
+        """
+        if hint is None or hint <= 0:
+            return list(self)
+        n = 0
+        lines = []
+        for line in self:
+            lines.append(line)
+            n += len(line)
+            if n >= hint:
+                break
+        return lines
+
+    def writelines(self, lines):
+        self._checkClosed()
+        for line in lines:
+            self.write(line)
+
+io.IOBase.register(IOBase)
+
+
+class RawIOBase(IOBase):
+
+    """Base class for raw binary I/O."""
+
+    # The read() method is implemented by calling readinto(); derived
+    # classes that want to support read() only need to implement
+    # readinto() as a primitive operation.  In general, readinto() can be
+    # more efficient than read().
+
+    # (It would be tempting to also provide an implementation of
+    # readinto() in terms of read(), in case the latter is a more suitable
+    # primitive operation, but that would lead to nasty recursion in case
+    # a subclass doesn't implement either.)
+
+    def read(self, n=-1):
+        """Read and return up to n bytes, where n is an int.
+
+        Returns an empty bytes object on EOF, or None if the object is
+        set not to block and has no data to read.
+        """
+        if n is None:
+            n = -1
+        if n < 0:
+            return self.readall()
+        b = bytearray(n.__index__())
+        n = self.readinto(b)
+        if n is None:
+            return None
+        del b[n:]
+        return bytes(b)
+
+    def readall(self):
+        """Read until EOF, using multiple read() call."""
+        res = bytearray()
+        while True:
+            data = self.read(DEFAULT_BUFFER_SIZE)
+            if not data:
+                break
+            res += data
+        if res:
+            return bytes(res)
+        else:
+            # b'' or None
+            return data
+
+    def readinto(self, b):
+        """Read up to len(b) bytes into bytearray b.
+
+        Returns an int representing the number of bytes read (0 for EOF), or
+        None if the object is set not to block and has no data to read.
+        """
+        self._unsupported("readinto")
+
+    def write(self, b):
+        """Write the given buffer to the IO stream.
+
+        Returns the number of bytes written, which may be less than len(b).
+        """
+        self._unsupported("write")
+
+io.RawIOBase.register(RawIOBase)
+from _io import FileIO
+RawIOBase.register(FileIO)
+
+
+class BufferedIOBase(IOBase):
+
+    """Base class for buffered IO objects.
+
+    The main difference with RawIOBase is that the read() method
+    supports omitting the size argument, and does not have a default
+    implementation that defers to readinto().
+
+    In addition, read(), readinto() and write() may raise
+    BlockingIOError if the underlying raw stream is in non-blocking
+    mode and not ready; unlike their raw counterparts, they will never
+    return None.
+
+    A typical implementation should not inherit from a RawIOBase
+    implementation, but wrap one.
+    """
+
+    def read(self, n=None):
+        """Read and return up to n bytes, where n is an int.
+
+        If the argument is omitted, None, or negative, reads and
+        returns all data until EOF.
+
+        If the argument is positive, and the underlying raw stream is
+        not 'interactive', multiple raw reads may be issued to satisfy
+        the byte count (unless EOF is reached first).  But for
+        interactive raw streams (XXX and for pipes?), at most one raw
+        read will be issued, and a short result does not imply that
+        EOF is imminent.
+
+        Returns an empty bytes array on EOF.
+
+        Raises BlockingIOError if the underlying raw stream has no
+        data at the moment.
+        """
+        self._unsupported("read")
+
+    def read1(self, n=None):
+        """Read up to n bytes with at most one read() system call,
+        where n is an int.
+        """
+        self._unsupported("read1")
+
+    def readinto(self, b):
+        """Read up to len(b) bytes into bytearray b.
+
+        Like read(), this may issue multiple reads to the underlying raw
+        stream, unless the latter is 'interactive'.
+
+        Returns an int representing the number of bytes read (0 for EOF).
+
+        Raises BlockingIOError if the underlying raw stream has no
+        data at the moment.
+        """
+        # XXX This ought to work with anything that supports the buffer API
+        data = self.read(len(b))
+        n = len(data)
+        try:
+            b[:n] = data
+        except TypeError as err:
+            import array
+            if not isinstance(b, array.array):
+                raise err
+            b[:n] = array.array('b', data)
+        return n
+
+    def write(self, b):
+        """Write the given bytes buffer to the IO stream.
+
+        Return the number of bytes written, which is never less than
+        len(b).
+
+        Raises BlockingIOError if the buffer is full and the
+        underlying raw stream cannot accept more data at the moment.
+        """
+        self._unsupported("write")
+
+    def detach(self):
+        """
+        Separate the underlying raw stream from the buffer and return it.
+
+        After the raw stream has been detached, the buffer is in an unusable
+        state.
+        """
+        self._unsupported("detach")
+
+io.BufferedIOBase.register(BufferedIOBase)
+
+
+class _BufferedIOMixin(BufferedIOBase):
+
+    """A mixin implementation of BufferedIOBase with an underlying raw stream.
+
+    This passes most requests on to the underlying raw stream.  It
+    does *not* provide implementations of read(), readinto() or
+    write().
+    """
+
+    def __init__(self, raw):
+        self._raw = raw
+
+    ### Positioning ###
+
+    def seek(self, pos, whence=0):
+        new_position = self.raw.seek(pos, whence)
+        if new_position < 0:
+            raise IOError("seek() returned an invalid position")
+        return new_position
+
+    def tell(self):
+        pos = self.raw.tell()
+        if pos < 0:
+            raise IOError("tell() returned an invalid position")
+        return pos
+
+    def truncate(self, pos=None):
+        # Flush the stream.  We're mixing buffered I/O with lower-level I/O,
+        # and a flush may be necessary to synch both views of the current
+        # file state.
+        self.flush()
+
+        if pos is None:
+            pos = self.tell()
+        # XXX: Should seek() be used, instead of passing the position
+        # XXX  directly to truncate?
+        return self.raw.truncate(pos)
+
+    ### Flush and close ###
+
+    def flush(self):
+        if self.closed:
+            raise ValueError("flush of closed file")
+        self.raw.flush()
+
+    def close(self):
+        if self.raw is not None and not self.closed:
+            try:
+                # may raise BlockingIOError or BrokenPipeError etc
+                self.flush()
+            finally:
+                self.raw.close()
+
+    def detach(self):
+        if self.raw is None:
+            raise ValueError("raw stream already detached")
+        self.flush()
+        raw = self._raw
+        self._raw = None
+        return raw
+
+    ### Inquiries ###
+
+    def seekable(self):
+        return self.raw.seekable()
+
+    def readable(self):
+        return self.raw.readable()
+
+    def writable(self):
+        return self.raw.writable()
+
+    @property
+    def raw(self):
+        return self._raw
+
+    @property
+    def closed(self):
+        return self.raw.closed
+
+    @property
+    def name(self):
+        return self.raw.name
+
+    @property
+    def mode(self):
+        return self.raw.mode
+
+    def __getstate__(self):
+        raise TypeError("can not serialize a '{0}' object"
+                        .format(self.__class__.__name__))
+
+    def __repr__(self):
+        clsname = self.__class__.__name__
+        try:
+            name = self.name
+        except AttributeError:
+            return "<_pyio.{0}>".format(clsname)
+        else:
+            return "<_pyio.{0} name={1!r}>".format(clsname, name)
+
+    ### Lower-level APIs ###
+
+    def fileno(self):
+        return self.raw.fileno()
+
+    def isatty(self):
+        return self.raw.isatty()
+
+
+class BytesIO(BufferedIOBase):
+
+    """Buffered I/O implementation using an in-memory bytes buffer."""
+
+    def __init__(self, initial_bytes=None):
+        buf = bytearray()
+        if initial_bytes is not None:
+            buf += initial_bytes
+        self._buffer = buf
+        self._pos = 0
+
+    def __getstate__(self):
+        if self.closed:
+            raise ValueError("__getstate__ on closed file")
+        return self.__dict__.copy()
+
+    def getvalue(self):
+        """Return the bytes value (contents) of the buffer
+        """
+        if self.closed:
+            raise ValueError("getvalue on closed file")
+        return bytes(self._buffer)
+
+    def getbuffer(self):
+        """Return a readable and writable view of the buffer.
+        """
+        return memoryview(self._buffer)
+
+    def read(self, n=None):
+        if self.closed:
+            raise ValueError("read from closed file")
+        if n is None:
+            n = -1
+        if n < 0:
+            n = len(self._buffer)
+        if len(self._buffer) <= self._pos:
+            return b""
+        newpos = min(len(self._buffer), self._pos + n)
+        b = self._buffer[self._pos : newpos]
+        self._pos = newpos
+        return bytes(b)
+
+    def read1(self, n):
+        """This is the same as read.
+        """
+        return self.read(n)
+
+    def write(self, b):
+        if self.closed:
+            raise ValueError("write to closed file")
+        if isinstance(b, str):
+            raise TypeError("can't write str to binary stream")
+        n = len(b)
+        if n == 0:
+            return 0
+        pos = self._pos
+        if pos > len(self._buffer):
+            # Inserts null bytes between the current end of the file
+            # and the new write position.
+            padding = b'\x00' * (pos - len(self._buffer))
+            self._buffer += padding
+        self._buffer[pos:pos + n] = b
+        self._pos += n
+        return n
+
+    def seek(self, pos, whence=0):
+        if self.closed:
+            raise ValueError("seek on closed file")
+        try:
+            pos.__index__
+        except AttributeError as err:
+            raise TypeError("an integer is required") from err
+        if whence == 0:
+            if pos < 0:
+                raise ValueError("negative seek position %r" % (pos,))
+            self._pos = pos
+        elif whence == 1:
+            self._pos = max(0, self._pos + pos)
+        elif whence == 2:
+            self._pos = max(0, len(self._buffer) + pos)
+        else:
+            raise ValueError("invalid whence value")
+        return self._pos
+
+    def tell(self):
+        if self.closed:
+            raise ValueError("tell on closed file")
+        return self._pos
+
+    def truncate(self, pos=None):
+        if self.closed:
+            raise ValueError("truncate on closed file")
+        if pos is None:
+            pos = self._pos
+        else:
+            try:
+                pos.__index__
+            except AttributeError as err:
+                raise TypeError("an integer is required") from err
+            if pos < 0:
+                raise ValueError("negative truncate position %r" % (pos,))
+        del self._buffer[pos:]
+        return pos
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return True
+
+    def seekable(self):
+        return True
+
+
+class BufferedReader(_BufferedIOMixin):
+
+    """BufferedReader(raw[, buffer_size])
+
+    A buffer for a readable, sequential BaseRawIO object.
+
+    The constructor creates a BufferedReader for the given readable raw
+    stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE
+    is used.
+    """
+
+    def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
+        """Create a new buffered reader using the given readable raw IO object.
+        """
+        if not raw.readable():
+            raise IOError('"raw" argument must be readable.')
+
+        _BufferedIOMixin.__init__(self, raw)
+        if buffer_size <= 0:
+            raise ValueError("invalid buffer size")
+        self.buffer_size = buffer_size
+        self._reset_read_buf()
+        self._read_lock = Lock()
+
+    def _reset_read_buf(self):
+        self._read_buf = b""
+        self._read_pos = 0
+
+    def read(self, n=None):
+        """Read n bytes.
+
+        Returns exactly n bytes of data unless the underlying raw IO
+        stream reaches EOF or if the call would block in non-blocking
+        mode. If n is negative, read until EOF or until read() would
+        block.
+        """
+        if n is not None and n < -1:
+            raise ValueError("invalid number of bytes to read")
+        with self._read_lock:
+            return self._read_unlocked(n)
+
+    def _read_unlocked(self, n=None):
+        nodata_val = b""
+        empty_values = (b"", None)
+        buf = self._read_buf
+        pos = self._read_pos
+
+        # Special case for when the number of bytes to read is unspecified.
+        if n is None or n == -1:
+            self._reset_read_buf()
+            chunks = [buf[pos:]]  # Strip the consumed bytes.
+            current_size = 0
+            while True:
+                # Read until EOF or until read() would block.
+                try:
+                    chunk = self.raw.read()
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
+                if chunk in empty_values:
+                    nodata_val = chunk
+                    break
+                current_size += len(chunk)
+                chunks.append(chunk)
+            return b"".join(chunks) or nodata_val
+
+        # The number of bytes to read is specified, return at most n bytes.
+        avail = len(buf) - pos  # Length of the available buffered data.
+        if n <= avail:
+            # Fast path: the data to read is fully buffered.
+            self._read_pos += n
+            return buf[pos:pos+n]
+        # Slow path: read from the stream until enough bytes are read,
+        # or until an EOF occurs or until read() would block.
+        chunks = [buf[pos:]]
+        wanted = max(self.buffer_size, n)
+        while avail < n:
+            try:
+                chunk = self.raw.read(wanted)
+            except IOError as e:
+                if e.errno != EINTR:
+                    raise
+                continue
+            if chunk in empty_values:
+                nodata_val = chunk
+                break
+            avail += len(chunk)
+            chunks.append(chunk)
+        # n is more then avail only when an EOF occurred or when
+        # read() would have blocked.
+        n = min(n, avail)
+        out = b"".join(chunks)
+        self._read_buf = out[n:]  # Save the extra data in the buffer.
+        self._read_pos = 0
+        return out[:n] if out else nodata_val
+
+    def peek(self, n=0):
+        """Returns buffered bytes without advancing the position.
+
+        The argument indicates a desired minimal number of bytes; we
+        do at most one raw read to satisfy it.  We never return more
+        than self.buffer_size.
+        """
+        with self._read_lock:
+            return self._peek_unlocked(n)
+
+    def _peek_unlocked(self, n=0):
+        want = min(n, self.buffer_size)
+        have = len(self._read_buf) - self._read_pos
+        if have < want or have <= 0:
+            to_read = self.buffer_size - have
+            while True:
+                try:
+                    current = self.raw.read(to_read)
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
+                break
+            if current:
+                self._read_buf = self._read_buf[self._read_pos:] + current
+                self._read_pos = 0
+        return self._read_buf[self._read_pos:]
+
+    def read1(self, n):
+        """Reads up to n bytes, with at most one read() system call."""
+        # Returns up to n bytes.  If at least one byte is buffered, we
+        # only return buffered bytes.  Otherwise, we do one raw read.
+        if n < 0:
+            raise ValueError("number of bytes to read must be positive")
+        if n == 0:
+            return b""
+        with self._read_lock:
+            self._peek_unlocked(1)
+            return self._read_unlocked(
+                min(n, len(self._read_buf) - self._read_pos))
+
+    def tell(self):
+        return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence value")
+        with self._read_lock:
+            if whence == 1:
+                pos -= len(self._read_buf) - self._read_pos
+            pos = _BufferedIOMixin.seek(self, pos, whence)
+            self._reset_read_buf()
+            return pos
+
+class BufferedWriter(_BufferedIOMixin):
+
+    """A buffer for a writeable sequential RawIO object.
+
+    The constructor creates a BufferedWriter for the given writeable raw
+    stream. If the buffer_size is not given, it defaults to
+    DEFAULT_BUFFER_SIZE.
+    """
+
+    _warning_stack_offset = 2
+
+    def __init__(self, raw,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        if not raw.writable():
+            raise IOError('"raw" argument must be writable.')
+
+        _BufferedIOMixin.__init__(self, raw)
+        if buffer_size <= 0:
+            raise ValueError("invalid buffer size")
+        if max_buffer_size is not None:
+            warnings.warn("max_buffer_size is deprecated", DeprecationWarning,
+                          self._warning_stack_offset)
+        self.buffer_size = buffer_size
+        self._write_buf = bytearray()
+        self._write_lock = Lock()
+
+    def write(self, b):
+        if self.closed:
+            raise ValueError("write to closed file")
+        if isinstance(b, str):
+            raise TypeError("can't write str to binary stream")
+        with self._write_lock:
+            # XXX we can implement some more tricks to try and avoid
+            # partial writes
+            if len(self._write_buf) > self.buffer_size:
+                # We're full, so let's pre-flush the buffer.  (This may
+                # raise BlockingIOError with characters_written == 0.)
+                self._flush_unlocked()
+            before = len(self._write_buf)
+            self._write_buf.extend(b)
+            written = len(self._write_buf) - before
+            if len(self._write_buf) > self.buffer_size:
+                try:
+                    self._flush_unlocked()
+                except BlockingIOError as e:
+                    if len(self._write_buf) > self.buffer_size:
+                        # We've hit the buffer_size. We have to accept a partial
+                        # write and cut back our buffer.
+                        overage = len(self._write_buf) - self.buffer_size
+                        written -= overage
+                        self._write_buf = self._write_buf[:self.buffer_size]
+                        raise BlockingIOError(e.errno, e.strerror, written)
+            return written
+
+    def truncate(self, pos=None):
+        with self._write_lock:
+            self._flush_unlocked()
+            if pos is None:
+                pos = self.raw.tell()
+            return self.raw.truncate(pos)
+
+    def flush(self):
+        with self._write_lock:
+            self._flush_unlocked()
+
+    def _flush_unlocked(self):
+        if self.closed:
+            raise ValueError("flush of closed file")
+        while self._write_buf:
+            try:
+                n = self.raw.write(self._write_buf)
+            except BlockingIOError:
+                raise RuntimeError("self.raw should implement RawIOBase: it "
+                                   "should not raise BlockingIOError")
+            except IOError as e:
+                if e.errno != EINTR:
+                    raise
+                continue
+            if n is None:
+                raise BlockingIOError(
+                    errno.EAGAIN,
+                    "write could not complete without blocking", 0)
+            if n > len(self._write_buf) or n < 0:
+                raise IOError("write() returned incorrect number of bytes")
+            del self._write_buf[:n]
+
+    def tell(self):
+        return _BufferedIOMixin.tell(self) + len(self._write_buf)
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence")
+        with self._write_lock:
+            self._flush_unlocked()
+            return _BufferedIOMixin.seek(self, pos, whence)
+
+
+class BufferedRWPair(BufferedIOBase):
+
+    """A buffered reader and writer object together.
+
+    A buffered reader object and buffered writer object put together to
+    form a sequential IO object that can read and write. This is typically
+    used with a socket or two-way pipe.
+
+    reader and writer are RawIOBase objects that are readable and
+    writeable respectively. If the buffer_size is omitted it defaults to
+    DEFAULT_BUFFER_SIZE.
+    """
+
+    # XXX The usefulness of this (compared to having two separate IO
+    # objects) is questionable.
+
+    def __init__(self, reader, writer,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        """Constructor.
+
+        The arguments are two RawIO instances.
+        """
+        if max_buffer_size is not None:
+            warnings.warn("max_buffer_size is deprecated", DeprecationWarning, 2)
+
+        if not reader.readable():
+            raise IOError('"reader" argument must be readable.')
+
+        if not writer.writable():
+            raise IOError('"writer" argument must be writable.')
+
+        self.reader = BufferedReader(reader, buffer_size)
+        self.writer = BufferedWriter(writer, buffer_size)
+
+    def read(self, n=None):
+        if n is None:
+            n = -1
+        return self.reader.read(n)
+
+    def readinto(self, b):
+        return self.reader.readinto(b)
+
+    def write(self, b):
+        return self.writer.write(b)
+
+    def peek(self, n=0):
+        return self.reader.peek(n)
+
+    def read1(self, n):
+        return self.reader.read1(n)
+
+    def readable(self):
+        return self.reader.readable()
+
+    def writable(self):
+        return self.writer.writable()
+
+    def flush(self):
+        return self.writer.flush()
+
+    def close(self):
+        self.writer.close()
+        self.reader.close()
+
+    def isatty(self):
+        return self.reader.isatty() or self.writer.isatty()
+
+    @property
+    def closed(self):
+        return self.writer.closed
+
+
+class BufferedRandom(BufferedWriter, BufferedReader):
+
+    """A buffered interface to random access streams.
+
+    The constructor creates a reader and writer for a seekable stream,
+    raw, given in the first argument. If the buffer_size is omitted it
+    defaults to DEFAULT_BUFFER_SIZE.
+    """
+
+    _warning_stack_offset = 3
+
+    def __init__(self, raw,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        raw._checkSeekable()
+        BufferedReader.__init__(self, raw, buffer_size)
+        BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size)
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence")
+        self.flush()
+        if self._read_buf:
+            # Undo read ahead.
+            with self._read_lock:
+                self.raw.seek(self._read_pos - len(self._read_buf), 1)
+        # First do the raw seek, then empty the read buffer, so that
+        # if the raw seek fails, we don't lose buffered data forever.
+        pos = self.raw.seek(pos, whence)
+        with self._read_lock:
+            self._reset_read_buf()
+        if pos < 0:
+            raise IOError("seek() returned invalid position")
+        return pos
+
+    def tell(self):
+        if self._write_buf:
+            return BufferedWriter.tell(self)
+        else:
+            return BufferedReader.tell(self)
+
+    def truncate(self, pos=None):
+        if pos is None:
+            pos = self.tell()
+        # Use seek to flush the read buffer.
+        return BufferedWriter.truncate(self, pos)
+
+    def read(self, n=None):
+        if n is None:
+            n = -1
+        self.flush()
+        return BufferedReader.read(self, n)
+
+    def readinto(self, b):
+        self.flush()
+        return BufferedReader.readinto(self, b)
+
+    def peek(self, n=0):
+        self.flush()
+        return BufferedReader.peek(self, n)
+
+    def read1(self, n):
+        self.flush()
+        return BufferedReader.read1(self, n)
+
+    def write(self, b):
+        if self._read_buf:
+            # Undo readahead
+            with self._read_lock:
+                self.raw.seek(self._read_pos - len(self._read_buf), 1)
+                self._reset_read_buf()
+        return BufferedWriter.write(self, b)
+
+
+class TextIOBase(IOBase):
+
+    """Base class for text I/O.
+
+    This class provides a character and line based interface to stream
+    I/O. There is no readinto method because Python's character strings
+    are immutable. There is no public constructor.
+    """
+
+    def read(self, n=-1):
+        """Read at most n characters from stream, where n is an int.
+
+        Read from underlying buffer until we have n characters or we hit EOF.
+        If n is negative or omitted, read until EOF.
+
+        Returns a string.
+        """
+        self._unsupported("read")
+
+    def write(self, s):
+        """Write string s to stream and returning an int."""
+        self._unsupported("write")
+
+    def truncate(self, pos=None):
+        """Truncate size to pos, where pos is an int."""
+        self._unsupported("truncate")
+
+    def readline(self):
+        """Read until newline or EOF.
+
+        Returns an empty string if EOF is hit immediately.
+        """
+        self._unsupported("readline")
+
+    def detach(self):
+        """
+        Separate the underlying buffer from the TextIOBase and return it.
+
+        After the underlying buffer has been detached, the TextIO is in an
+        unusable state.
+        """
+        self._unsupported("detach")
+
+    @property
+    def encoding(self):
+        """Subclasses should override."""
+        return None
+
+    @property
+    def newlines(self):
+        """Line endings translated so far.
+
+        Only line endings translated during reading are considered.
+
+        Subclasses should override.
+        """
+        return None
+
+    @property
+    def errors(self):
+        """Error setting of the decoder or encoder.
+
+        Subclasses should override."""
+        return None
+
+io.TextIOBase.register(TextIOBase)
+
+
+class IncrementalNewlineDecoder(codecs.IncrementalDecoder):
+    r"""Codec used when reading a file in universal newlines mode.  It wraps
+    another incremental decoder, translating \r\n and \r into \n.  It also
+    records the types of newlines encountered.  When used with
+    translate=False, it ensures that the newline sequence is returned in
+    one piece.
+    """
+    def __init__(self, decoder, translate, errors='strict'):
+        codecs.IncrementalDecoder.__init__(self, errors=errors)
+        self.translate = translate
+        self.decoder = decoder
+        self.seennl = 0
+        self.pendingcr = False
+
+    def decode(self, input, final=False):
+        # decode input (with the eventual \r from a previous pass)
+        if self.decoder is None:
+            output = input
+        else:
+            output = self.decoder.decode(input, final=final)
+        if self.pendingcr and (output or final):
+            output = "\r" + output
+            self.pendingcr = False
+
+        # retain last \r even when not translating data:
+        # then readline() is sure to get \r\n in one pass
+        if output.endswith("\r") and not final:
+            output = output[:-1]
+            self.pendingcr = True
+
+        # Record which newlines are read
+        crlf = output.count('\r\n')
+        cr = output.count('\r') - crlf
+        lf = output.count('\n') - crlf
+        self.seennl |= (lf and self._LF) | (cr and self._CR) \
+                    | (crlf and self._CRLF)
+
+        if self.translate:
+            if crlf:
+                output = output.replace("\r\n", "\n")
+            if cr:
+                output = output.replace("\r", "\n")
+
+        return output
+
+    def getstate(self):
+        if self.decoder is None:
+            buf = b""
+            flag = 0
+        else:
+            buf, flag = self.decoder.getstate()
+        flag <<= 1
+        if self.pendingcr:
+            flag |= 1
+        return buf, flag
+
+    def setstate(self, state):
+        buf, flag = state
+        self.pendingcr = bool(flag & 1)
+        if self.decoder is not None:
+            self.decoder.setstate((buf, flag >> 1))
+
+    def reset(self):
+        self.seennl = 0
+        self.pendingcr = False
+        if self.decoder is not None:
+            self.decoder.reset()
+
+    _LF = 1
+    _CR = 2
+    _CRLF = 4
+
+    @property
+    def newlines(self):
+        return (None,
+                "\n",
+                "\r",
+                ("\r", "\n"),
+                "\r\n",
+                ("\n", "\r\n"),
+                ("\r", "\r\n"),
+                ("\r", "\n", "\r\n")
+               )[self.seennl]
+
+
+class TextIOWrapper(TextIOBase):
+
+    r"""Character and line based layer over a BufferedIOBase object, buffer.
+
+    encoding gives the name of the encoding that the stream will be
+    decoded or encoded with. It defaults to locale.getpreferredencoding.
+
+    errors determines the strictness of encoding and decoding (see the
+    codecs.register) and defaults to "strict".
+
+    newline can be None, '', '\n', '\r', or '\r\n'.  It controls the
+    handling of line endings. If it is None, universal newlines is
+    enabled.  With this enabled, on input, the lines endings '\n', '\r',
+    or '\r\n' are translated to '\n' before being returned to the
+    caller. Conversely, on output, '\n' is translated to the system
+    default line separator, os.linesep. If newline is any other of its
+    legal values, that newline becomes the newline when the file is read
+    and it is returned untranslated. On output, '\n' is converted to the
+    newline.
+
+    If line_buffering is True, a call to flush is implied when a call to
+    write contains a newline character.
+    """
+
+    _CHUNK_SIZE = 2048
+
+    def __init__(self, buffer, encoding=None, errors=None, newline=None,
+                 line_buffering=False, write_through=False):
+        if newline is not None and not isinstance(newline, str):
+            raise TypeError("illegal newline type: %r" % (type(newline),))
+        if newline not in (None, "", "\n", "\r", "\r\n"):
+            raise ValueError("illegal newline value: %r" % (newline,))
+        if encoding is None:
+            try:
+                encoding = os.device_encoding(buffer.fileno())
+            except (AttributeError, UnsupportedOperation):
+                pass
+            if encoding is None:
+                try:
+                    import locale
+                except ImportError:
+                    # Importing locale may fail if Python is being built
+                    encoding = "ascii"
+                else:
+                    encoding = locale.getpreferredencoding()
+
+        if not isinstance(encoding, str):
+            raise ValueError("invalid encoding: %r" % encoding)
+
+        if errors is None:
+            errors = "strict"
+        else:
+            if not isinstance(errors, str):
+                raise ValueError("invalid errors: %r" % errors)
+
+        self._buffer = buffer
+        self._line_buffering = line_buffering
+        self._encoding = encoding
+        self._errors = errors
+        self._readuniversal = not newline
+        self._readtranslate = newline is None
+        self._readnl = newline
+        self._writetranslate = newline != ''
+        self._writenl = newline or os.linesep
+        self._encoder = None
+        self._decoder = None
+        self._decoded_chars = ''  # buffer for text returned from decoder
+        self._decoded_chars_used = 0  # offset into _decoded_chars for read()
+        self._snapshot = None  # info for reconstructing decoder state
+        self._seekable = self._telling = self.buffer.seekable()
+        self._has_read1 = hasattr(self.buffer, 'read1')
+
+        if self._seekable and self.writable():
+            position = self.buffer.tell()
+            if position != 0:
+                try:
+                    self._get_encoder().setstate(0)
+                except LookupError:
+                    # Sometimes the encoder doesn't exist
+                    pass
+
+    # self._snapshot is either None, or a tuple (dec_flags, next_input)
+    # where dec_flags is the second (integer) item of the decoder state
+    # and next_input is the chunk of input bytes that comes next after the
+    # snapshot point.  We use this to reconstruct decoder states in tell().
+
+    # Naming convention:
+    #   - "bytes_..." for integer variables that count input bytes
+    #   - "chars_..." for integer variables that count decoded characters
+
+    def __repr__(self):
+        result = "<_pyio.TextIOWrapper"
+        try:
+            name = self.name
+        except AttributeError:
+            pass
+        else:
+            result += " name={0!r}".format(name)
+        try:
+            mode = self.mode
+        except AttributeError:
+            pass
+        else:
+            result += " mode={0!r}".format(mode)
+        return result + " encoding={0!r}>".format(self.encoding)
+
+    @property
+    def encoding(self):
+        return self._encoding
+
+    @property
+    def errors(self):
+        return self._errors
+
+    @property
+    def line_buffering(self):
+        return self._line_buffering
+
+    @property
+    def buffer(self):
+        return self._buffer
+
+    def seekable(self):
+        return self._seekable
+
+    def readable(self):
+        return self.buffer.readable()
+
+    def writable(self):
+        return self.buffer.writable()
+
+    def flush(self):
+        self.buffer.flush()
+        self._telling = self._seekable
+
+    def close(self):
+        if self.buffer is not None and not self.closed:
+            self.flush()
+            self.buffer.close()
+
+    @property
+    def closed(self):
+        return self.buffer.closed
+
+    @property
+    def name(self):
+        return self.buffer.name
+
+    def fileno(self):
+        return self.buffer.fileno()
+
+    def isatty(self):
+        return self.buffer.isatty()
+
+    def write(self, s):
+        'Write data, where s is a str'
+        if self.closed:
+            raise ValueError("write to closed file")
+        if not isinstance(s, str):
+            raise TypeError("can't write %s to text stream" %
+                            s.__class__.__name__)
+        length = len(s)
+        haslf = (self._writetranslate or self._line_buffering) and "\n" in s
+        if haslf and self._writetranslate and self._writenl != "\n":
+            s = s.replace("\n", self._writenl)
+        encoder = self._encoder or self._get_encoder()
+        # XXX What if we were just reading?
+        b = encoder.encode(s)
+        self.buffer.write(b)
+        if self._line_buffering and (haslf or "\r" in s):
+            self.flush()
+        self._snapshot = None
+        if self._decoder:
+            self._decoder.reset()
+        return length
+
+    def _get_encoder(self):
+        make_encoder = codecs.getincrementalencoder(self._encoding)
+        self._encoder = make_encoder(self._errors)
+        return self._encoder
+
+    def _get_decoder(self):
+        make_decoder = codecs.getincrementaldecoder(self._encoding)
+        decoder = make_decoder(self._errors)
+        if self._readuniversal:
+            decoder = IncrementalNewlineDecoder(decoder, self._readtranslate)
+        self._decoder = decoder
+        return decoder
+
+    # The following three methods implement an ADT for _decoded_chars.
+    # Text returned from the decoder is buffered here until the client
+    # requests it by calling our read() or readline() method.
+    def _set_decoded_chars(self, chars):
+        """Set the _decoded_chars buffer."""
+        self._decoded_chars = chars
+        self._decoded_chars_used = 0
+
+    def _get_decoded_chars(self, n=None):
+        """Advance into the _decoded_chars buffer."""
+        offset = self._decoded_chars_used
+        if n is None:
+            chars = self._decoded_chars[offset:]
+        else:
+            chars = self._decoded_chars[offset:offset + n]
+        self._decoded_chars_used += len(chars)
+        return chars
+
+    def _rewind_decoded_chars(self, n):
+        """Rewind the _decoded_chars buffer."""
+        if self._decoded_chars_used < n:
+            raise AssertionError("rewind decoded_chars out of bounds")
+        self._decoded_chars_used -= n
+
+    def _read_chunk(self):
+        """
+        Read and decode the next chunk of data from the BufferedReader.
+        """
+
+        # The return value is True unless EOF was reached.  The decoded
+        # string is placed in self._decoded_chars (replacing its previous
+        # value).  The entire input chunk is sent to the decoder, though
+        # some of it may remain buffered in the decoder, yet to be
+        # converted.
+
+        if self._decoder is None:
+            raise ValueError("no decoder")
+
+        if self._telling:
+            # To prepare for tell(), we need to snapshot a point in the
+            # file where the decoder's input buffer is empty.
+
+            dec_buffer, dec_flags = self._decoder.getstate()
+            # Given this, we know there was a valid snapshot point
+            # len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
+
+        # Read a chunk, decode it, and put the result in self._decoded_chars.
+        if self._has_read1:
+            input_chunk = self.buffer.read1(self._CHUNK_SIZE)
+        else:
+            input_chunk = self.buffer.read(self._CHUNK_SIZE)
+        eof = not input_chunk
+        self._set_decoded_chars(self._decoder.decode(input_chunk, eof))
+
+        if self._telling:
+            # At the snapshot point, len(dec_buffer) bytes before the read,
+            # the next input to be decoded is dec_buffer + input_chunk.
+            self._snapshot = (dec_flags, dec_buffer + input_chunk)
+
+        return not eof
+
+    def _pack_cookie(self, position, dec_flags=0,
+                           bytes_to_feed=0, need_eof=0, chars_to_skip=0):
+        # The meaning of a tell() cookie is: seek to position, set the
+        # decoder flags to dec_flags, read bytes_to_feed bytes, feed them
+        # into the decoder with need_eof as the EOF flag, then skip
+        # chars_to_skip characters of the decoded result.  For most simple
+        # decoders, tell() will often just give a byte offset in the file.
+        return (position | (dec_flags<<64) | (bytes_to_feed<<128) |
+               (chars_to_skip<<192) | bool(need_eof)<<256)
+
+    def _unpack_cookie(self, bigint):
+        rest, position = divmod(bigint, 1<<64)
+        rest, dec_flags = divmod(rest, 1<<64)
+        rest, bytes_to_feed = divmod(rest, 1<<64)
+        need_eof, chars_to_skip = divmod(rest, 1<<64)
+        return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip
+
+    def tell(self):
+        if not self._seekable:
+            raise UnsupportedOperation("underlying stream is not seekable")
+        if not self._telling:
+            raise IOError("telling position disabled by next() call")
+        self.flush()
+        position = self.buffer.tell()
+        decoder = self._decoder
+        if decoder is None or self._snapshot is None:
+            if self._decoded_chars:
+                # This should never happen.
+                raise AssertionError("pending decoded text")
+            return position
+
+        # Skip backward to the snapshot point (see _read_chunk).
+        dec_flags, next_input = self._snapshot
+        position -= len(next_input)
+
+        # How many decoded characters have been used up since the snapshot?
+        chars_to_skip = self._decoded_chars_used
+        if chars_to_skip == 0:
+            # We haven't moved from the snapshot point.
+            return self._pack_cookie(position, dec_flags)
+
+        # Starting from the snapshot position, we will walk the decoder
+        # forward until it gives us enough decoded characters.
+        saved_state = decoder.getstate()
+        try:
+            # Note our initial start point.
+            decoder.setstate((b'', dec_flags))
+            start_pos = position
+            start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
+            need_eof = 0
+
+            # Feed the decoder one byte at a time.  As we go, note the
+            # nearest "safe start point" before the current location
+            # (a point where the decoder has nothing buffered, so seek()
+            # can safely start from there and advance to this location).
+            next_byte = bytearray(1)
+            for next_byte[0] in next_input:
+                bytes_fed += 1
+                chars_decoded += len(decoder.decode(next_byte))
+                dec_buffer, dec_flags = decoder.getstate()
+                if not dec_buffer and chars_decoded <= chars_to_skip:
+                    # Decoder buffer is empty, so this is a safe start point.
+                    start_pos += bytes_fed
+                    chars_to_skip -= chars_decoded
+                    start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
+                if chars_decoded >= chars_to_skip:
+                    break
+            else:
+                # We didn't get enough decoded data; signal EOF to get more.
+                chars_decoded += len(decoder.decode(b'', final=True))
+                need_eof = 1
+                if chars_decoded < chars_to_skip:
+                    raise IOError("can't reconstruct logical file position")
+
+            # The returned cookie corresponds to the last safe start point.
+            return self._pack_cookie(
+                start_pos, start_flags, bytes_fed, need_eof, chars_to_skip)
+        finally:
+            decoder.setstate(saved_state)
+
+    def truncate(self, pos=None):
+        self.flush()
+        if pos is None:
+            pos = self.tell()
+        return self.buffer.truncate(pos)
+
+    def detach(self):
+        if self.buffer is None:
+            raise ValueError("buffer is already detached")
+        self.flush()
+        buffer = self._buffer
+        self._buffer = None
+        return buffer
+
+    def seek(self, cookie, whence=0):
+        if self.closed:
+            raise ValueError("tell on closed file")
+        if not self._seekable:
+            raise UnsupportedOperation("underlying stream is not seekable")
+        if whence == 1: # seek relative to current position
+            if cookie != 0:
+                raise UnsupportedOperation("can't do nonzero cur-relative seeks")
+            # Seeking to the current position should attempt to
+            # sync the underlying buffer with the current position.
+            whence = 0
+            cookie = self.tell()
+        if whence == 2: # seek relative to end of file
+            if cookie != 0:
+                raise UnsupportedOperation("can't do nonzero end-relative seeks")
+            self.flush()
+            position = self.buffer.seek(0, 2)
+            self._set_decoded_chars('')
+            self._snapshot = None
+            if self._decoder:
+                self._decoder.reset()
+            return position
+        if whence != 0:
+            raise ValueError("invalid whence (%r, should be 0, 1 or 2)" %
+                             (whence,))
+        if cookie < 0:
+            raise ValueError("negative seek position %r" % (cookie,))
+        self.flush()
+
+        # The strategy of seek() is to go back to the safe start point
+        # and replay the effect of read(chars_to_skip) from there.
+        start_pos, dec_flags, bytes_to_feed, need_eof, chars_to_skip = \
+            self._unpack_cookie(cookie)
+
+        # Seek back to the safe start point.
+        self.buffer.seek(start_pos)
+        self._set_decoded_chars('')
+        self._snapshot = None
+
+        # Restore the decoder to its state from the safe start point.
+        if cookie == 0 and self._decoder:
+            self._decoder.reset()
+        elif self._decoder or dec_flags or chars_to_skip:
+            self._decoder = self._decoder or self._get_decoder()
+            self._decoder.setstate((b'', dec_flags))
+            self._snapshot = (dec_flags, b'')
+
+        if chars_to_skip:
+            # Just like _read_chunk, feed the decoder and save a snapshot.
+            input_chunk = self.buffer.read(bytes_to_feed)
+            self._set_decoded_chars(
+                self._decoder.decode(input_chunk, need_eof))
+            self._snapshot = (dec_flags, input_chunk)
+
+            # Skip chars_to_skip of the decoded characters.
+            if len(self._decoded_chars) < chars_to_skip:
+                raise IOError("can't restore logical file position")
+            self._decoded_chars_used = chars_to_skip
+
+        # Finally, reset the encoder (merely useful for proper BOM handling)
+        try:
+            encoder = self._encoder or self._get_encoder()
+        except LookupError:
+            # Sometimes the encoder doesn't exist
+            pass
+        else:
+            if cookie != 0:
+                encoder.setstate(0)
+            else:
+                encoder.reset()
+        return cookie
+
+    def read(self, n=None):
+        self._checkReadable()
+        if n is None:
+            n = -1
+        decoder = self._decoder or self._get_decoder()
+        try:
+            n.__index__
+        except AttributeError as err:
+            raise TypeError("an integer is required") from err
+        if n < 0:
+            # Read everything.
+            result = (self._get_decoded_chars() +
+                      decoder.decode(self.buffer.read(), final=True))
+            self._set_decoded_chars('')
+            self._snapshot = None
+            return result
+        else:
+            # Keep reading chunks until we have n characters to return.
+            eof = False
+            result = self._get_decoded_chars(n)
+            while len(result) < n and not eof:
+                eof = not self._read_chunk()
+                result += self._get_decoded_chars(n - len(result))
+            return result
+
+    def __next__(self):
+        self._telling = False
+        line = self.readline()
+        if not line:
+            self._snapshot = None
+            self._telling = self._seekable
+            raise StopIteration
+        return line
+
+    def readline(self, limit=None):
+        if self.closed:
+            raise ValueError("read from closed file")
+        if limit is None:
+            limit = -1
+        elif not isinstance(limit, int):
+            raise TypeError("limit must be an integer")
+
+        # Grab all the decoded text (we will rewind any extra bits later).
+        line = self._get_decoded_chars()
+
+        start = 0
+        # Make the decoder if it doesn't already exist.
+        if not self._decoder:
+            self._get_decoder()
+
+        pos = endpos = None
+        while True:
+            if self._readtranslate:
+                # Newlines are already translated, only search for \n
+                pos = line.find('\n', start)
+                if pos >= 0:
+                    endpos = pos + 1
+                    break
+                else:
+                    start = len(line)
+
+            elif self._readuniversal:
+                # Universal newline search. Find any of \r, \r\n, \n
+                # The decoder ensures that \r\n are not split in two pieces
+
+                # In C we'd look for these in parallel of course.
+                nlpos = line.find("\n", start)
+                crpos = line.find("\r", start)
+                if crpos == -1:
+                    if nlpos == -1:
+                        # Nothing found
+                        start = len(line)
+                    else:
+                        # Found \n
+                        endpos = nlpos + 1
+                        break
+                elif nlpos == -1:
+                    # Found lone \r
+                    endpos = crpos + 1
+                    break
+                elif nlpos < crpos:
+                    # Found \n
+                    endpos = nlpos + 1
+                    break
+                elif nlpos == crpos + 1:
+                    # Found \r\n
+                    endpos = crpos + 2
+                    break
+                else:
+                    # Found \r
+                    endpos = crpos + 1
+                    break
+            else:
+                # non-universal
+                pos = line.find(self._readnl)
+                if pos >= 0:
+                    endpos = pos + len(self._readnl)
+                    break
+
+            if limit >= 0 and len(line) >= limit:
+                endpos = limit  # reached length limit
+                break
+
+            # No line ending seen yet - get more data'
+            while self._read_chunk():
+                if self._decoded_chars:
+                    break
+            if self._decoded_chars:
+                line += self._get_decoded_chars()
+            else:
+                # end of file
+                self._set_decoded_chars('')
+                self._snapshot = None
+                return line
+
+        if limit >= 0 and endpos > limit:
+            endpos = limit  # don't exceed limit
+
+        # Rewind _decoded_chars to just after the line ending we found.
+        self._rewind_decoded_chars(len(line) - endpos)
+        return line[:endpos]
+
+    @property
+    def newlines(self):
+        return self._decoder.newlines if self._decoder else None
+
+
+class StringIO(TextIOWrapper):
+    """Text I/O implementation using an in-memory buffer.
+
+    The initial_value argument sets the value of object.  The newline
+    argument is like the one of TextIOWrapper's constructor.
+    """
+
+    def __init__(self, initial_value="", newline="\n"):
+        super(StringIO, self).__init__(BytesIO(),
+                                       encoding="utf-8",
+                                       errors="strict",
+                                       newline=newline)
+        # Issue #5645: make universal newlines semantics the same as in the
+        # C version, even under Windows.
+        if newline is None:
+            self._writetranslate = False
+        if initial_value is not None:
+            if not isinstance(initial_value, str):
+                raise TypeError("initial_value must be str or None, not {0}"
+                                .format(type(initial_value).__name__))
+                initial_value = str(initial_value)
+            self.write(initial_value)
+            self.seek(0)
+
+    def getvalue(self):
+        self.flush()
+        return self.buffer.getvalue().decode(self._encoding, self._errors)
+
+    def __repr__(self):
+        # TextIOWrapper tells the encoding in its repr. In StringIO,
+        # that's a implementation detail.
+        return object.__repr__(self)
+
+    @property
+    def errors(self):
+        return None
+
+    @property
+    def encoding(self):
+        return None
+
+    def detach(self):
+        # This doesn't make sense on StringIO.
+        self._unsupported("detach")
diff --git a/lib-python/3.2/_strptime.py b/lib-python/3.2/_strptime.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_strptime.py
@@ -0,0 +1,499 @@
+"""Strptime-related classes and functions.
+
+CLASSES:
+    LocaleTime -- Discovers and stores locale-specific time information
+    TimeRE -- Creates regexes for pattern matching a string of text containing
+                time information
+
+FUNCTIONS:
+    _getlang -- Figure out what language is being used for the locale
+    strptime -- Calculates the time struct represented by the passed-in string
+
+"""
+import time
+import locale
+import calendar
+from re import compile as re_compile
+from re import IGNORECASE, ASCII
+from re import escape as re_escape
+from datetime import (date as datetime_date,
+                      timedelta as datetime_timedelta,
+                      timezone as datetime_timezone)
+try:
+    from _thread import allocate_lock as _thread_allocate_lock
+except:
+    from _dummy_thread import allocate_lock as _thread_allocate_lock
+
+__all__ = []
+
+def _getlang():
+    # Figure out what the current language is set to.
+    return locale.getlocale(locale.LC_TIME)
+
+class LocaleTime(object):
+    """Stores and handles locale-specific information related to time.
+
+    ATTRIBUTES:
+        f_weekday -- full weekday names (7-item list)
+        a_weekday -- abbreviated weekday names (7-item list)
+        f_month -- full month names (13-item list; dummy value in [0], which
+                    is added by code)
+        a_month -- abbreviated month names (13-item list, dummy value in
+                    [0], which is added by code)
+        am_pm -- AM/PM representation (2-item list)
+        LC_date_time -- format string for date/time representation (string)
+        LC_date -- format string for date representation (string)
+        LC_time -- format string for time representation (string)
+        timezone -- daylight- and non-daylight-savings timezone representation
+                    (2-item list of sets)
+        lang -- Language used by instance (2-item tuple)
+    """
+
+    def __init__(self):
+        """Set all attributes.
+
+        Order of methods called matters for dependency reasons.
+
+        The locale language is set at the offset and then checked again before
+        exiting.  This is to make sure that the attributes were not set with a
+        mix of information from more than one locale.  This would most likely
+        happen when using threads where one thread calls a locale-dependent
+        function while another thread changes the locale while the function in
+        the other thread is still running.  Proper coding would call for
+        locks to prevent changing the locale while locale-dependent code is
+        running.  The check here is done in case someone does not think about
+        doing this.
+
+        Only other possible issue is if someone changed the timezone and did
+        not call tz.tzset .  That is an issue for the programmer, though,
+        since changing the timezone is worthless without that call.
+
+        """
+        self.lang = _getlang()
+        self.__calc_weekday()
+        self.__calc_month()
+        self.__calc_am_pm()
+        self.__calc_timezone()
+        self.__calc_date_time()
+        if _getlang() != self.lang:
+            raise ValueError("locale changed during initialization")
+
+    def __pad(self, seq, front):
+        # Add '' to seq to either the front (is True), else the back.
+        seq = list(seq)
+        if front:
+            seq.insert(0, '')
+        else:
+            seq.append('')
+        return seq
+
+    def __calc_weekday(self):
+        # Set self.a_weekday and self.f_weekday using the calendar
+        # module.
+        a_weekday = [calendar.day_abbr[i].lower() for i in range(7)]
+        f_weekday = [calendar.day_name[i].lower() for i in range(7)]
+        self.a_weekday = a_weekday
+        self.f_weekday = f_weekday
+
+    def __calc_month(self):
+        # Set self.f_month and self.a_month using the calendar module.
+        a_month = [calendar.month_abbr[i].lower() for i in range(13)]
+        f_month = [calendar.month_name[i].lower() for i in range(13)]
+        self.a_month = a_month
+        self.f_month = f_month
+
+    def __calc_am_pm(self):
+        # Set self.am_pm by using time.strftime().
+
+        # The magic date (1999,3,17,hour,44,55,2,76,0) is not really that
+        # magical; just happened to have used it everywhere else where a
+        # static date was needed.
+        am_pm = []
+        for hour in (1, 22):
+            time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0))
+            am_pm.append(time.strftime("%p", time_tuple).lower())
+        self.am_pm = am_pm
+
+    def __calc_date_time(self):
+        # Set self.date_time, self.date, & self.time by using
+        # time.strftime().
+
+        # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of
+        # overloaded numbers is minimized.  The order in which searches for
+        # values within the format string is very important; it eliminates
+        # possible ambiguity for what something represents.
+        time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
+        date_time = [None, None, None]
+        date_time[0] = time.strftime("%c", time_tuple).lower()
+        date_time[1] = time.strftime("%x", time_tuple).lower()
+        date_time[2] = time.strftime("%X", time_tuple).lower()
+        replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'),
+                    (self.f_month[3], '%B'), (self.a_weekday[2], '%a'),
+                    (self.a_month[3], '%b'), (self.am_pm[1], '%p'),
+                    ('1999', '%Y'), ('99', '%y'), ('22', '%H'),
+                    ('44', '%M'), ('55', '%S'), ('76', '%j'),
+                    ('17', '%d'), ('03', '%m'), ('3', '%m'),
+                    # '3' needed for when no leading zero.
+                    ('2', '%w'), ('10', '%I')]
+        replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone
+                                                for tz in tz_values])
+        for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')):
+            current_format = date_time[offset]
+            for old, new in replacement_pairs:
+                # Must deal with possible lack of locale info
+                # manifesting itself as the empty string (e.g., Swedish's
+                # lack of AM/PM info) or a platform returning a tuple of empty
+                # strings (e.g., MacOS 9 having timezone as ('','')).
+                if old:
+                    current_format = current_format.replace(old, new)
+            # If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
+            # 2005-01-03 occurs before the first Monday of the year.  Otherwise
+            # %U is used.
+            time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0))
+            if '00' in time.strftime(directive, time_tuple):
+                U_W = '%W'
+            else:
+                U_W = '%U'
+            date_time[offset] = current_format.replace('11', U_W)
+        self.LC_date_time = date_time[0]
+        self.LC_date = date_time[1]
+        self.LC_time = date_time[2]
+
+    def __calc_timezone(self):
+        # Set self.timezone by using time.tzname.
+        # Do not worry about possibility of time.tzname[0] == timetzname[1]
+        # and time.daylight; handle that in strptime .
+        try:
+            time.tzset()
+        except AttributeError:
+            pass
+        no_saving = frozenset(["utc", "gmt", time.tzname[0].lower()])
+        if time.daylight:
+            has_saving = frozenset([time.tzname[1].lower()])
+        else:
+            has_saving = frozenset()
+        self.timezone = (no_saving, has_saving)
+
+
+class TimeRE(dict):
+    """Handle conversion from format directives to regexes."""
+
+    def __init__(self, locale_time=None):
+        """Create keys/values.
+
+        Order of execution is important for dependency reasons.
+
+        """
+        if locale_time:
+            self.locale_time = locale_time
+        else:
+            self.locale_time = LocaleTime()
+        base = super()
+        base.__init__({
+            # The " \d" part of the regex is to make %c from ANSI C work
+            'd': r"(?P<d>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
+            'f': r"(?P<f>[0-9]{1,6})",
+            'H': r"(?P<H>2[0-3]|[0-1]\d|\d)",
+            'I': r"(?P<I>1[0-2]|0[1-9]|[1-9])",
+            'j': r"(?P<j>36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
+            'm': r"(?P<m>1[0-2]|0[1-9]|[1-9])",
+            'M': r"(?P<M>[0-5]\d|\d)",
+            'S': r"(?P<S>6[0-1]|[0-5]\d|\d)",
+            'U': r"(?P<U>5[0-3]|[0-4]\d|\d)",
+            'w': r"(?P<w>[0-6])",
+            # W is set below by using 'U'
+            'y': r"(?P<y>\d\d)",
+            #XXX: Does 'Y' need to worry about having less or more than
+            #     4 digits?
+            'Y': r"(?P<Y>\d\d\d\d)",
+            'z': r"(?P<z>[+-]\d\d[0-5]\d)",
+            'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
+            'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
+            'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
+            'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
+            'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
+            'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
+                                        for tz in tz_names),
+                                'Z'),
+            '%': '%'})
+        base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
+        base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
+        base.__setitem__('x', self.pattern(self.locale_time.LC_date))
+        base.__setitem__('X', self.pattern(self.locale_time.LC_time))
+
+    def __seqToRE(self, to_convert, directive):
+        """Convert a list to a regex string for matching a directive.
+
+        Want possible matching values to be from longest to shortest.  This
+        prevents the possibility of a match occuring for a value that also
+        a substring of a larger value that should have matched (e.g., 'abc'
+        matching when 'abcdef' should have been the match).
+
+        """
+        to_convert = sorted(to_convert, key=len, reverse=True)
+        for value in to_convert:
+            if value != '':
+                break
+        else:
+            return ''
+        regex = '|'.join(re_escape(stuff) for stuff in to_convert)
+        regex = '(?P<%s>%s' % (directive, regex)
+        return '%s)' % regex
+
+    def pattern(self, format):
+        """Return regex pattern for the format string.
+
+        Need to make sure that any characters that might be interpreted as
+        regex syntax are escaped.
+
+        """
+        processed_format = ''
+        # The sub() call escapes all characters that might be misconstrued
+        # as regex syntax.  Cannot use re.escape since we have to deal with
+        # format directives (%m, etc.).
+        regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
+        format = regex_chars.sub(r"\\\1", format)
+        whitespace_replacement = re_compile('\s+')
+        format = whitespace_replacement.sub('\s+', format)
+        while '%' in format:
+            directive_index = format.index('%')+1
+            processed_format = "%s%s%s" % (processed_format,
+                                           format[:directive_index-1],
+                                           self[format[directive_index]])
+            format = format[directive_index+1:]
+        return "%s%s" % (processed_format, format)
+
+    def compile(self, format):
+        """Return a compiled re object for the format string."""
+        return re_compile(self.pattern(format), IGNORECASE)
+
+_cache_lock = _thread_allocate_lock()
+# DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock
+# first!
+_TimeRE_cache = TimeRE()
+_CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache
+_regex_cache = {}
+
+def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon):
+    """Calculate the Julian day based on the year, week of the year, and day of
+    the week, with week_start_day representing whether the week of the year
+    assumes the week starts on Sunday or Monday (6 or 0)."""
+    first_weekday = datetime_date(year, 1, 1).weekday()
+    # If we are dealing with the %U directive (week starts on Sunday), it's
+    # easier to just shift the view to Sunday being the first day of the
+    # week.
+    if not week_starts_Mon:
+        first_weekday = (first_weekday + 1) % 7
+        day_of_week = (day_of_week + 1) % 7
+    # Need to watch out for a week 0 (when the first day of the year is not
+    # the same as that specified by %U or %W).
+    week_0_length = (7 - first_weekday) % 7
+    if week_of_year == 0:
+        return 1 + day_of_week - first_weekday
+    else:
+        days_to_week = week_0_length + (7 * (week_of_year - 1))
+        return 1 + days_to_week + day_of_week
+
+
+def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a 2-tuple consisting of a time struct and an int containing
+    the number of microseconds based on the input string and the
+    format string."""
+
+    for index, arg in enumerate([data_string, format]):
+        if not isinstance(arg, str):
+            msg = "strptime() argument {} must be str, not {}"
+            raise TypeError(msg.format(index, type(arg)))
+
+    global _TimeRE_cache, _regex_cache
+    with _cache_lock:
+
+        if _getlang() != _TimeRE_cache.locale_time.lang:
+            _TimeRE_cache = TimeRE()
+            _regex_cache.clear()
+        if len(_regex_cache) > _CACHE_MAX_SIZE:
+            _regex_cache.clear()
+        locale_time = _TimeRE_cache.locale_time
+        format_regex = _regex_cache.get(format)
+        if not format_regex:
+            try:
+                format_regex = _TimeRE_cache.compile(format)
+            # KeyError raised when a bad format is found; can be specified as
+            # \\, in which case it was a stray % but with a space after it
+            except KeyError as err:
+                bad_directive = err.args[0]
+                if bad_directive == "\\":
+                    bad_directive = "%"
+                del err
+                raise ValueError("'%s' is a bad directive in format '%s'" %
+                                    (bad_directive, format))
+            # IndexError only occurs when the format string is "%"
+            except IndexError:
+                raise ValueError("stray %% in format '%s'" % format)
+            _regex_cache[format] = format_regex
+    found = format_regex.match(data_string)
+    if not found:
+        raise ValueError("time data %r does not match format %r" %
+                         (data_string, format))
+    if len(data_string) != found.end():
+        raise ValueError("unconverted data remains: %s" %
+                          data_string[found.end():])
+
+    year = 1900
+    month = day = 1
+    hour = minute = second = fraction = 0
+    tz = -1
+    tzoffset = None
+    # Default to -1 to signify that values not known; not critical to have,
+    # though
+    week_of_year = -1
+    week_of_year_start = -1
+    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # values
+    weekday = julian = -1
+    found_dict = found.groupdict()
+    for group_key in found_dict.keys():
+        # Directives not explicitly handled below:
+        #   c, x, X
+        #      handled by making out of other directives
+        #   U, W
+        #      worthless without day of the week
+        if group_key == 'y':
+            year = int(found_dict['y'])
+            # Open Group specification for strptime() states that a %y
+            #value in the range of [00, 68] is in the century 2000, while
+            #[69,99] is in the century 1900
+            if year <= 68:
+                year += 2000
+            else:
+                year += 1900
+        elif group_key == 'Y':
+            year = int(found_dict['Y'])
+        elif group_key == 'm':
+            month = int(found_dict['m'])
+        elif group_key == 'B':
+            month = locale_time.f_month.index(found_dict['B'].lower())
+        elif group_key == 'b':
+            month = locale_time.a_month.index(found_dict['b'].lower())
+        elif group_key == 'd':
+            day = int(found_dict['d'])
+        elif group_key == 'H':
+            hour = int(found_dict['H'])
+        elif group_key == 'I':
+            hour = int(found_dict['I'])
+            ampm = found_dict.get('p', '').lower()
+            # If there was no AM/PM indicator, we'll treat this like AM
+            if ampm in ('', locale_time.am_pm[0]):
+                # We're in AM so the hour is correct unless we're
+                # looking at 12 midnight.
+                # 12 midnight == 12 AM == hour 0
+                if hour == 12:
+                    hour = 0
+            elif ampm == locale_time.am_pm[1]:
+                # We're in PM so we need to add 12 to the hour unless
+                # we're looking at 12 noon.
+                # 12 noon == 12 PM == hour 12
+                if hour != 12:
+                    hour += 12
+        elif group_key == 'M':
+            minute = int(found_dict['M'])
+        elif group_key == 'S':
+            second = int(found_dict['S'])
+        elif group_key == 'f':
+            s = found_dict['f']
+            # Pad to always return microseconds.
+            s += "0" * (6 - len(s))
+            fraction = int(s)
+        elif group_key == 'A':
+            weekday = locale_time.f_weekday.index(found_dict['A'].lower())
+        elif group_key == 'a':
+            weekday = locale_time.a_weekday.index(found_dict['a'].lower())
+        elif group_key == 'w':
+            weekday = int(found_dict['w'])
+            if weekday == 0:
+                weekday = 6
+            else:
+                weekday -= 1
+        elif group_key == 'j':
+            julian = int(found_dict['j'])
+        elif group_key in ('U', 'W'):
+            week_of_year = int(found_dict[group_key])
+            if group_key == 'U':
+                # U starts week on Sunday.
+                week_of_year_start = 6
+            else:
+                # W starts week on Monday.
+                week_of_year_start = 0
+        elif group_key == 'z':
+            z = found_dict['z']
+            tzoffset = int(z[1:3]) * 60 + int(z[3:5])
+            if z.startswith("-"):
+                tzoffset = -tzoffset
+        elif group_key == 'Z':
+            # Since -1 is default value only need to worry about setting tz if
+            # it can be something other than -1.
+            found_zone = found_dict['Z'].lower()
+            for value, tz_values in enumerate(locale_time.timezone):
+                if found_zone in tz_values:
+                    # Deal with bad locale setup where timezone names are the
+                    # same and yet time.daylight is true; too ambiguous to
+                    # be able to tell what timezone has daylight savings
+                    if (time.tzname[0] == time.tzname[1] and
+                       time.daylight and found_zone not in ("utc", "gmt")):
+                        break
+                    else:
+                        tz = value
+                        break
+    # If we know the week of the year and what day of that week, we can figure
+    # out the Julian day of the year.
+    if julian == -1 and week_of_year != -1 and weekday != -1:
+        week_starts_Mon = True if week_of_year_start == 0 else False
+        julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
+                                            week_starts_Mon)
+    # Cannot pre-calculate datetime_date() since can change in Julian
+    # calculation and thus could have different value for the day of the week
+    # calculation.
+    if julian == -1:
+        # Need to add 1 to result since first day of the year is 1, not 0.
+        julian = datetime_date(year, month, day).toordinal() - \
+                  datetime_date(year, 1, 1).toordinal() + 1
+    else:  # Assume that if they bothered to include Julian day it will
+           # be accurate.
+        datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal())
+        year = datetime_result.year
+        month = datetime_result.month
+        day = datetime_result.day
+    if weekday == -1:
+        weekday = datetime_date(year, month, day).weekday()
+    # Add timezone info
+    tzname = found_dict.get("Z")
+    if tzoffset is not None:
+        gmtoff = tzoffset * 60
+    else:
+        gmtoff = None
+
+    return (year, month, day,
+            hour, minute, second,
+            weekday, julian, tz, gmtoff, tzname), fraction
+
+def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a time struct based on the input string and the
+    format string."""
+    tt = _strptime(data_string, format)[0]
+    return time.struct_time(tt[:9])
+
+def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a class cls instance based on the input string and the
+    format string."""
+    tt, fraction = _strptime(data_string, format)
+    gmtoff, tzname = tt[-2:]
+    args = tt[:6] + (fraction,)
+    if gmtoff is not None:
+        tzdelta = datetime_timedelta(seconds=gmtoff)
+        if tzname:
+            tz = datetime_timezone(tzdelta, tzname)
+        else:
+            tz = datetime_timezone(tzdelta)
+        args += (tz,)
+
+    return cls(*args)
diff --git a/lib-python/3.2/_threading_local.py b/lib-python/3.2/_threading_local.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_threading_local.py
@@ -0,0 +1,246 @@
+"""Thread-local objects.
+
+(Note that this module provides a Python version of the threading.local
+ class.  Depending on the version of Python you're using, there may be a
+ faster one available.  You should always import the `local` class from
+ `threading`.)
+
+Thread-local objects support the management of thread-local data.
+If you have data that you want to be local to a thread, simply create
+a thread-local object and use its attributes:
+
+  >>> mydata = local()
+  >>> mydata.number = 42
+  >>> mydata.number
+  42
+
+You can also access the local-object's dictionary:
+
+  >>> mydata.__dict__
+  {'number': 42}
+  >>> mydata.__dict__.setdefault('widgets', [])
+  []
+  >>> mydata.widgets
+  []
+
+What's important about thread-local objects is that their data are
+local to a thread. If we access the data in a different thread:
+
+  >>> log = []
+  >>> def f():
+  ...     items = sorted(mydata.__dict__.items())
+  ...     log.append(items)
+  ...     mydata.number = 11
+  ...     log.append(mydata.number)
+
+  >>> import threading
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+  >>> log
+  [[], 11]
+
+we get different data.  Furthermore, changes made in the other thread
+don't affect data seen in this thread:
+
+  >>> mydata.number
+  42
+
+Of course, values you get from a local object, including a __dict__
+attribute, are for whatever thread was current at the time the
+attribute was read.  For that reason, you generally don't want to save
+these values across threads, as they apply only to the thread they
+came from.
+
+You can create custom local objects by subclassing the local class:
+
+  >>> class MyLocal(local):
+  ...     number = 2
+  ...     initialized = False
+  ...     def __init__(self, **kw):
+  ...         if self.initialized:
+  ...             raise SystemError('__init__ called too many times')
+  ...         self.initialized = True
+  ...         self.__dict__.update(kw)
+  ...     def squared(self):
+  ...         return self.number ** 2
+
+This can be useful to support default values, methods and
+initialization.  Note that if you define an __init__ method, it will be
+called each time the local object is used in a separate thread.  This
+is necessary to initialize each thread's dictionary.
+
+Now if we create a local object:
+
+  >>> mydata = MyLocal(color='red')
+
+Now we have a default number:
+
+  >>> mydata.number
+  2
+
+an initial color:
+
+  >>> mydata.color
+  'red'
+  >>> del mydata.color
+
+And a method that operates on the data:
+
+  >>> mydata.squared()
+  4
+
+As before, we can access the data in a separate thread:
+
+  >>> log = []
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+  >>> log
+  [[('color', 'red'), ('initialized', True)], 11]
+
+without affecting this thread's data:
+
+  >>> mydata.number
+  2
+  >>> mydata.color
+  Traceback (most recent call last):
+  ...
+  AttributeError: 'MyLocal' object has no attribute 'color'
+
+Note that subclasses can define slots, but they are not thread
+local. They are shared across threads:
+
+  >>> class MyLocal(local):
+  ...     __slots__ = 'number'
+
+  >>> mydata = MyLocal()
+  >>> mydata.number = 42
+  >>> mydata.color = 'red'
+
+So, the separate thread:
+
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+
+affects what we see:
+
+  >>> mydata.number
+  11
+
+>>> del mydata
+"""
+
+from weakref import ref
+from contextlib import contextmanager
+
+__all__ = ["local"]
+
+# We need to use objects from the threading module, but the threading
+# module may also want to use our `local` class, if support for locals
+# isn't compiled in to the `thread` module.  This creates potential problems
+# with circular imports.  For that reason, we don't import `threading`
+# until the bottom of this file (a hack sufficient to worm around the
+# potential problems).  Note that all platforms on CPython do have support
+# for locals in the `thread` module, and there is no circular import problem
+# then, so problems introduced by fiddling the order of imports here won't
+# manifest.
+
+class _localimpl:
+    """A class managing thread-local dicts"""
+    __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__'
+
+    def __init__(self):
+        # The key used in the Thread objects' attribute dicts.
+        # We keep it a string for speed but make it unlikely to clash with
+        # a "real" attribute.
+        self.key = '_threading_local._localimpl.' + str(id(self))
+        # { id(Thread) -> (ref(Thread), thread-local dict) }
+        self.dicts = {}
+
+    def get_dict(self):
+        """Return the dict for the current thread. Raises KeyError if none
+        defined."""
+        thread = current_thread()
+        return self.dicts[id(thread)][1]
+
+    def create_dict(self):
+        """Create a new dict for the current thread, and return it."""
+        localdict = {}
+        key = self.key
+        thread = current_thread()
+        idt = id(thread)
+        def local_deleted(_, key=key):
+            # When the localimpl is deleted, remove the thread attribute.
+            thread = wrthread()
+            if thread is not None:
+                del thread.__dict__[key]
+        def thread_deleted(_, idt=idt):
+            # When the thread is deleted, remove the local dict.
+            # Note that this is suboptimal if the thread object gets
+            # caught in a reference loop. We would like to be called
+            # as soon as the OS-level thread ends instead.
+            local = wrlocal()
+            if local is not None:
+                dct = local.dicts.pop(idt)
+        wrlocal = ref(self, local_deleted)
+        wrthread = ref(thread, thread_deleted)
+        thread.__dict__[key] = wrlocal
+        self.dicts[idt] = wrthread, localdict
+        return localdict
+
+
+ at contextmanager
+def _patch(self):
+    impl = object.__getattribute__(self, '_local__impl')
+    try:
+        dct = impl.get_dict()
+    except KeyError:
+        dct = impl.create_dict()
+        args, kw = impl.localargs
+        self.__init__(*args, **kw)
+    with impl.locallock:
+        object.__setattr__(self, '__dict__', dct)
+        yield
+
+
+class local:
+    __slots__ = '_local__impl', '__dict__'
+
+    def __new__(cls, *args, **kw):
+        if (args or kw) and (cls.__init__ is object.__init__):
+            raise TypeError("Initialization arguments are not supported")
+        self = object.__new__(cls)
+        impl = _localimpl()
+        impl.localargs = (args, kw)
+        impl.locallock = RLock()
+        object.__setattr__(self, '_local__impl', impl)
+        # We need to create the thread dict in anticipation of
+        # __init__ being called, to make sure we don't call it
+        # again ourselves.
+        impl.create_dict()
+        return self
+
+    def __getattribute__(self, name):
+        with _patch(self):
+            return object.__getattribute__(self, name)
+
+    def __setattr__(self, name, value):
+        if name == '__dict__':
+            raise AttributeError(
+                "%r object attribute '__dict__' is read-only"
+                % self.__class__.__name__)
+        with _patch(self):
+            return object.__setattr__(self, name, value)
+
+    def __delattr__(self, name):
+        if name == '__dict__':
+            raise AttributeError(
+                "%r object attribute '__dict__' is read-only"
+                % self.__class__.__name__)
+        with _patch(self):
+            return object.__delattr__(self, name)
+
+
+from threading import current_thread, RLock
diff --git a/lib-python/3.2/_weakrefset.py b/lib-python/3.2/_weakrefset.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_weakrefset.py
@@ -0,0 +1,214 @@
+# Access WeakSet through the weakref module.
+# This code is separated-out because it is needed
+# by abc.py to load everything else at startup.
+
+from _weakref import ref
+
+__all__ = ['WeakSet']
+
+
+class _IterationGuard:
+    # This context manager registers itself in the current iterators of the
+    # weak container, such as to delay all removals until the context manager
+    # exits.
+    # This technique should be relatively thread-safe (since sets are).
+
+    def __init__(self, weakcontainer):
+        # Don't create cycles
+        self.weakcontainer = ref(weakcontainer)
+
+    def __enter__(self):
+        w = self.weakcontainer()
+        if w is not None:
+            w._iterating.add(self)
+        return self
+
+    def __exit__(self, e, t, b):
+        w = self.weakcontainer()
+        if w is not None:
+            s = w._iterating
+            s.remove(self)
+            if not s:
+                w._commit_removals()
+
+
+class WeakSet:
+    def __init__(self, data=None):
+        self.data = set()
+        def _remove(item, selfref=ref(self)):
+            self = selfref()
+            if self is not None:
+                if self._iterating:
+                    self._pending_removals.append(item)
+                else:
+                    self.data.discard(item)
+        self._remove = _remove
+        # A list of keys to be removed
+        self._pending_removals = []
+        self._iterating = set()
+        if data is not None:
+            self.update(data)
+
+    def _commit_removals(self):
+        l = self._pending_removals
+        discard = self.data.discard
+        while l:
+            discard(l.pop())
+
+    def __iter__(self):
+        with _IterationGuard(self):
+            for itemref in self.data:
+                item = itemref()
+                if item is not None:
+                    yield item
+
+    def __len__(self):
+        return sum(x() is not None for x in self.data)
+
+    def __contains__(self, item):
+        try:
+            wr = ref(item)
+        except TypeError:
+            return False
+        return wr in self.data
+
+    def __reduce__(self):
+        return (self.__class__, (list(self),),
+                getattr(self, '__dict__', None))
+
+    def add(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.add(ref(item, self._remove))
+
+    def clear(self):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.clear()
+
+    def copy(self):
+        return self.__class__(self)
+
+    def pop(self):
+        if self._pending_removals:
+            self._commit_removals()
+        while True:
+            try:
+                itemref = self.data.pop()
+            except KeyError:
+                raise KeyError('pop from empty WeakSet')
+            item = itemref()
+            if item is not None:
+                return item
+
+    def remove(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.remove(ref(item))
+
+    def discard(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.discard(ref(item))
+
+    def update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if isinstance(other, self.__class__):
+            self.data.update(other.data)
+        else:
+            for element in other:
+                self.add(element)
+
+    def __ior__(self, other):
+        self.update(other)
+        return self
+
+    # Helper functions for simple delegating methods.
+    def _apply(self, other, method):
+        if not isinstance(other, self.__class__):
+            other = self.__class__(other)
+        newdata = method(other.data)
+        newset = self.__class__()
+        newset.data = newdata
+        return newset
+
+    def difference(self, other):
+        return self._apply(other, self.data.difference)
+    __sub__ = difference
+
+    def difference_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.difference_update(ref(item) for item in other)
+    def __isub__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.difference_update(ref(item) for item in other)
+        return self
+
+    def intersection(self, other):
+        return self._apply(other, self.data.intersection)
+    __and__ = intersection
+
+    def intersection_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.intersection_update(ref(item) for item in other)
+    def __iand__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.intersection_update(ref(item) for item in other)
+        return self
+
+    def issubset(self, other):
+        return self.data.issubset(ref(item) for item in other)
+    __lt__ = issubset
+
+    def __le__(self, other):
+        return self.data <= set(ref(item) for item in other)
+
+    def issuperset(self, other):
+        return self.data.issuperset(ref(item) for item in other)
+    __gt__ = issuperset
+
+    def __ge__(self, other):
+        return self.data >= set(ref(item) for item in other)
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return self.data == set(ref(item) for item in other)
+
+    def symmetric_difference(self, other):
+        return self._apply(other, self.data.symmetric_difference)
+    __xor__ = symmetric_difference
+
+    def symmetric_difference_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.symmetric_difference_update(ref(item) for item in other)
+    def __ixor__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.symmetric_difference_update(ref(item) for item in other)
+        return self
+
+    def union(self, other):
+        return self._apply(other, self.data.union)
+    __or__ = union
+
+    def isdisjoint(self, other):
+        return len(self.intersection(other)) == 0
diff --git a/lib-python/3.2/abc.py b/lib-python/3.2/abc.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/abc.py
@@ -0,0 +1,211 @@
+# Copyright 2007 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Abstract Base Classes (ABCs) according to PEP 3119."""
+
+from _weakrefset import WeakSet
+
+def abstractmethod(funcobj):
+    """A decorator indicating abstract methods.
+
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract methods are overridden.
+    The abstract methods can be called using any of the normal
+    'super' call mechanisms.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractmethod
+            def my_abstract_method(self, ...):
+                ...
+    """
+    funcobj.__isabstractmethod__ = True
+    return funcobj
+
+
+class abstractclassmethod(classmethod):
+    """A decorator indicating abstract classmethods.
+
+    Similar to abstractmethod.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractclassmethod
+            def my_abstract_classmethod(cls, ...):
+                ...
+    """
+
+    __isabstractmethod__ = True
+
+    def __init__(self, callable):
+        callable.__isabstractmethod__ = True
+        super().__init__(callable)
+
+
+class abstractstaticmethod(staticmethod):
+    """A decorator indicating abstract staticmethods.
+
+    Similar to abstractmethod.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractstaticmethod
+            def my_abstract_staticmethod(...):
+                ...
+    """
+
+    __isabstractmethod__ = True
+
+    def __init__(self, callable):
+        callable.__isabstractmethod__ = True
+        super().__init__(callable)
+
+
+class abstractproperty(property):
+    """A decorator indicating abstract properties.
+
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract properties are overridden.
+    The abstract properties can be called using any of the normal
+    'super' call mechanisms.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractproperty
+            def my_abstract_property(self):
+                ...
+
+    This defines a read-only property; you can also define a read-write
+    abstract property using the 'long' form of property declaration:
+
+        class C(metaclass=ABCMeta):
+            def getx(self): ...
+            def setx(self, value): ...
+            x = abstractproperty(getx, setx)
+    """
+    __isabstractmethod__ = True
+
+
+class ABCMeta(type):
+
+    """Metaclass for defining Abstract Base Classes (ABCs).
+
+    Use this metaclass to create an ABC.  An ABC can be subclassed
+    directly, and then acts as a mix-in class.  You can also register
+    unrelated concrete classes (even built-in classes) and unrelated
+    ABCs as 'virtual subclasses' -- these and their descendants will
+    be considered subclasses of the registering ABC by the built-in
+    issubclass() function, but the registering ABC won't show up in
+    their MRO (Method Resolution Order) nor will method
+    implementations defined by the registering ABC be callable (not
+    even via super()).
+
+    """
+
+    # A global counter that is incremented each time a class is
+    # registered as a virtual subclass of anything.  It forces the
+    # negative cache to be cleared before its next use.
+    _abc_invalidation_counter = 0
+
+    def __new__(mcls, name, bases, namespace):
+        cls = super().__new__(mcls, name, bases, namespace)
+        # Compute set of abstract method names
+        abstracts = {name
+                     for name, value in namespace.items()
+                     if getattr(value, "__isabstractmethod__", False)}
+        for base in bases:
+            for name in getattr(base, "__abstractmethods__", set()):
+                value = getattr(cls, name, None)
+                if getattr(value, "__isabstractmethod__", False):
+                    abstracts.add(name)
+        cls.__abstractmethods__ = frozenset(abstracts)
+        # Set up inheritance registry
+        cls._abc_registry = WeakSet()
+        cls._abc_cache = WeakSet()
+        cls._abc_negative_cache = WeakSet()
+        cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+        return cls
+
+    def register(cls, subclass):
+        """Register a virtual subclass of an ABC."""
+        if not isinstance(subclass, type):
+            raise TypeError("Can only register classes")
+        if issubclass(subclass, cls):
+            return  # Already a subclass
+        # Subtle: test for cycles *after* testing for "already a subclass";
+        # this means we allow X.register(X) and interpret it as a no-op.
+        if issubclass(cls, subclass):
+            # This would create a cycle, which is bad for the algorithm below
+            raise RuntimeError("Refusing to create an inheritance cycle")
+        cls._abc_registry.add(subclass)
+        ABCMeta._abc_invalidation_counter += 1  # Invalidate negative cache
+
+    def _dump_registry(cls, file=None):
+        """Debug helper to print the ABC registry."""
+        print("Class: %s.%s" % (cls.__module__, cls.__name__), file=file)
+        print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file)
+        for name in sorted(cls.__dict__.keys()):
+            if name.startswith("_abc_"):
+                value = getattr(cls, name)
+                print("%s: %r" % (name, value), file=file)
+
+    def __instancecheck__(cls, instance):
+        """Override for isinstance(instance, cls)."""
+        # Inline the cache checking
+        subclass = instance.__class__
+        if subclass in cls._abc_cache:
+            return True
+        subtype = type(instance)
+        if subtype is subclass:
+            if (cls._abc_negative_cache_version ==
+                ABCMeta._abc_invalidation_counter and
+                subclass in cls._abc_negative_cache):
+                return False
+            # Fall back to the subclass check.
+            return cls.__subclasscheck__(subclass)
+        return any(cls.__subclasscheck__(c) for c in {subclass, subtype})
+
+    def __subclasscheck__(cls, subclass):
+        """Override for issubclass(subclass, cls)."""
+        # Check cache
+        if subclass in cls._abc_cache:
+            return True
+        # Check negative cache; may have to invalidate
+        if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter:
+            # Invalidate the negative cache
+            cls._abc_negative_cache = WeakSet()
+            cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+        elif subclass in cls._abc_negative_cache:
+            return False
+        # Check the subclass hook
+        ok = cls.__subclasshook__(subclass)
+        if ok is not NotImplemented:
+            assert isinstance(ok, bool)
+            if ok:
+                cls._abc_cache.add(subclass)
+            else:
+                cls._abc_negative_cache.add(subclass)
+            return ok
+        # Check if it's a direct subclass
+        if cls in getattr(subclass, '__mro__', ()):
+            cls._abc_cache.add(subclass)
+            return True
+        # Check if it's a subclass of a registered class (recursive)
+        for rcls in cls._abc_registry:
+            if issubclass(subclass, rcls):
+                cls._abc_cache.add(subclass)
+                return True
+        # Check if it's a subclass of a subclass (recursive)
+        for scls in cls.__subclasses__():
+            if issubclass(subclass, scls):
+                cls._abc_cache.add(subclass)
+                return True
+        # No dice; update negative cache
+        cls._abc_negative_cache.add(subclass)
+        return False
diff --git a/lib-python/3.2/aifc.py b/lib-python/3.2/aifc.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/aifc.py
@@ -0,0 +1,891 @@
+"""Stuff to parse AIFF-C and AIFF files.
+
+Unless explicitly stated otherwise, the description below is true
+both for AIFF-C files and AIFF files.
+
+An AIFF-C file has the following structure.
+
+  +-----------------+
+  | FORM            |
+  +-----------------+
+  | <size>          |
+  +----+------------+
+  |    | AIFC       |
+  |    +------------+
+  |    | <chunks>   |
+  |    |    .       |
+  |    |    .       |
+  |    |    .       |
+  +----+------------+
+
+An AIFF file has the string "AIFF" instead of "AIFC".
+
+A chunk consists of an identifier (4 bytes) followed by a size (4 bytes,
+big endian order), followed by the data.  The size field does not include
+the size of the 8 byte header.
+
+The following chunk types are recognized.
+
+  FVER
+      <version number of AIFF-C defining document> (AIFF-C only).
+  MARK
+      <# of markers> (2 bytes)
+      list of markers:
+          <marker ID> (2 bytes, must be > 0)
+          <position> (4 bytes)
+          <marker name> ("pstring")
+  COMM
+      <# of channels> (2 bytes)
+      <# of sound frames> (4 bytes)
+      <size of the samples> (2 bytes)
+      <sampling frequency> (10 bytes, IEEE 80-bit extended
+          floating point)
+      in AIFF-C files only:
+      <compression type> (4 bytes)
+      <human-readable version of compression type> ("pstring")
+  SSND
+      <offset> (4 bytes, not used by this program)
+      <blocksize> (4 bytes, not used by this program)
+      <sound data>
+
+A pstring consists of 1 byte length, a string of characters, and 0 or 1
+byte pad to make the total length even.
+
+Usage.
+
+Reading AIFF files:
+  f = aifc.open(file, 'r')
+where file is either the name of a file or an open file pointer.
+The open file pointer must have methods read(), seek(), and close().
+In some types of audio files, if the setpos() method is not used,
+the seek() method is not necessary.
+
+This returns an instance of a class with the following public methods:
+  getnchannels()  -- returns number of audio channels (1 for
+             mono, 2 for stereo)
+  getsampwidth()  -- returns sample width in bytes
+  getframerate()  -- returns sampling frequency
+  getnframes()    -- returns number of audio frames
+  getcomptype()   -- returns compression type ('NONE' for AIFF files)
+  getcompname()   -- returns human-readable version of
+             compression type ('not compressed' for AIFF files)
+  getparams() -- returns a tuple consisting of all of the
+             above in the above order
+  getmarkers()    -- get the list of marks in the audio file or None
+             if there are no marks
+  getmark(id) -- get mark with the specified id (raises an error
+             if the mark does not exist)
+  readframes(n)   -- returns at most n frames of audio
+  rewind()    -- rewind to the beginning of the audio stream
+  setpos(pos) -- seek to the specified position
+  tell()      -- return the current position
+  close()     -- close the instance (make it unusable)
+The position returned by tell(), the position given to setpos() and
+the position of marks are all compatible and have nothing to do with
+the actual position in the file.
+The close() method is called automatically when the class instance
+is destroyed.
+
+Writing AIFF files:
+  f = aifc.open(file, 'w')
+where file is either the name of a file or an open file pointer.
+The open file pointer must have methods write(), tell(), seek(), and
+close().
+
+This returns an instance of a class with the following public methods:
+  aiff()      -- create an AIFF file (AIFF-C default)
+  aifc()      -- create an AIFF-C file
+  setnchannels(n) -- set the number of channels
+  setsampwidth(n) -- set the sample width
+  setframerate(n) -- set the frame rate
+  setnframes(n)   -- set the number of frames
+  setcomptype(type, name)
+          -- set the compression type and the
+             human-readable compression type
+  setparams(tuple)
+          -- set all parameters at once
+  setmark(id, pos, name)
+          -- add specified mark to the list of marks
+  tell()      -- return current position in output file (useful
+             in combination with setmark())
+  writeframesraw(data)
+          -- write audio frames without pathing up the
+             file header
+  writeframes(data)
+          -- write audio frames and patch up the file header
+  close()     -- patch up the file header and close the
+             output file
+You should set the parameters before the first writeframesraw or
+writeframes.  The total number of frames does not need to be set,
+but when it is set to the correct value, the header does not have to
+be patched up.
+It is best to first set all parameters, perhaps possibly the
+compression type, and then write audio frames using writeframesraw.
+When all frames have been written, either call writeframes('') or
+close() to patch up the sizes in the header.
+Marks can be added anytime.  If there are any marks, ypu must call
+close() after all frames have been written.
+The close() method is called automatically when the class instance
+is destroyed.
+
+When a file is opened with the extension '.aiff', an AIFF file is
+written, otherwise an AIFF-C file is written.  This default can be
+changed by calling aiff() or aifc() before the first writeframes or
+writeframesraw.
+"""
+
+import struct
+import builtins
+
+__all__ = ["Error", "open", "openfp"]
+
+class Error(Exception):
+    pass
+
+_AIFC_version = 0xA2805140     # Version 1 of AIFF-C
+
+def _read_long(file):
+    try:
+        return struct.unpack('>l', file.read(4))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_ulong(file):
+    try:
+        return struct.unpack('>L', file.read(4))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_short(file):
+    try:
+        return struct.unpack('>h', file.read(2))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_ushort(file):
+    try:
+        return struct.unpack('>H', file.read(2))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_string(file):
+    length = ord(file.read(1))
+    if length == 0:
+        data = b''
+    else:
+        data = file.read(length)
+    if length & 1 == 0:
+        dummy = file.read(1)
+    return data
+
+_HUGE_VAL = 1.79769313486231e+308 # See <limits.h>
+
+def _read_float(f): # 10 bytes
+    expon = _read_short(f) # 2 bytes
+    sign = 1
+    if expon < 0:
+        sign = -1
+        expon = expon + 0x8000
+    himant = _read_ulong(f) # 4 bytes
+    lomant = _read_ulong(f) # 4 bytes
+    if expon == himant == lomant == 0:
+        f = 0.0
+    elif expon == 0x7FFF:
+        f = _HUGE_VAL
+    else:
+        expon = expon - 16383
+        f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63)
+    return sign * f
+
+def _write_short(f, x):
+    f.write(struct.pack('>h', x))
+
+def _write_ushort(f, x):
+    f.write(struct.pack('>H', x))
+
+def _write_long(f, x):
+    f.write(struct.pack('>l', x))
+
+def _write_ulong(f, x):
+    f.write(struct.pack('>L', x))
+
+def _write_string(f, s):
+    if len(s) > 255:
+        raise ValueError("string exceeds maximum pstring length")
+    f.write(struct.pack('B', len(s)))
+    f.write(s)
+    if len(s) & 1 == 0:
+        f.write(b'\x00')
+
+def _write_float(f, x):
+    import math
+    if x < 0:
+        sign = 0x8000
+        x = x * -1
+    else:
+        sign = 0
+    if x == 0:
+        expon = 0
+        himant = 0
+        lomant = 0
+    else:
+        fmant, expon = math.frexp(x)
+        if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN
+            expon = sign|0x7FFF
+            himant = 0
+            lomant = 0
+        else:                   # Finite
+            expon = expon + 16382
+            if expon < 0:           # denormalized
+                fmant = math.ldexp(fmant, expon)
+                expon = 0
+            expon = expon | sign
+            fmant = math.ldexp(fmant, 32)
+            fsmant = math.floor(fmant)
+            himant = int(fsmant)
+            fmant = math.ldexp(fmant - fsmant, 32)
+            fsmant = math.floor(fmant)
+            lomant = int(fsmant)
+    _write_ushort(f, expon)
+    _write_ulong(f, himant)
+    _write_ulong(f, lomant)
+
+from chunk import Chunk
+
+class Aifc_read:
+    # Variables used in this class:
+    #
+    # These variables are available to the user though appropriate
+    # methods of this class:
+    # _file -- the open file with methods read(), close(), and seek()
+    #       set through the __init__() method
+    # _nchannels -- the number of audio channels
+    #       available through the getnchannels() method
+    # _nframes -- the number of audio frames
+    #       available through the getnframes() method
+    # _sampwidth -- the number of bytes per audio sample
+    #       available through the getsampwidth() method
+    # _framerate -- the sampling frequency
+    #       available through the getframerate() method
+    # _comptype -- the AIFF-C compression type ('NONE' if AIFF)
+    #       available through the getcomptype() method
+    # _compname -- the human-readable AIFF-C compression type
+    #       available through the getcomptype() method
+    # _markers -- the marks in the audio file
+    #       available through the getmarkers() and getmark()
+    #       methods
+    # _soundpos -- the position in the audio stream
+    #       available through the tell() method, set through the
+    #       setpos() method
+    #
+    # These variables are used internally only:
+    # _version -- the AIFF-C version number
+    # _decomp -- the decompressor from builtin module cl
+    # _comm_chunk_read -- 1 iff the COMM chunk has been read
+    # _aifc -- 1 iff reading an AIFF-C file
+    # _ssnd_seek_needed -- 1 iff positioned correctly in audio
+    #       file for readframes()
+    # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
+    # _framesize -- size of one frame in the file
+
+    def initfp(self, file):
+        self._version = 0
+        self._convert = None
+        self._markers = []
+        self._soundpos = 0
+        self._file = file
+        chunk = Chunk(file)
+        if chunk.getname() != b'FORM':
+            raise Error('file does not start with FORM id')
+        formdata = chunk.read(4)
+        if formdata == b'AIFF':
+            self._aifc = 0
+        elif formdata == b'AIFC':
+            self._aifc = 1
+        else:
+            raise Error('not an AIFF or AIFF-C file')
+        self._comm_chunk_read = 0
+        while 1:
+            self._ssnd_seek_needed = 1
+            try:
+                chunk = Chunk(self._file)
+            except EOFError:
+                break
+            chunkname = chunk.getname()
+            if chunkname == b'COMM':
+                self._read_comm_chunk(chunk)
+                self._comm_chunk_read = 1
+            elif chunkname == b'SSND':
+                self._ssnd_chunk = chunk
+                dummy = chunk.read(8)
+                self._ssnd_seek_needed = 0
+            elif chunkname == b'FVER':
+                self._version = _read_ulong(chunk)
+            elif chunkname == b'MARK':
+                self._readmark(chunk)
+            chunk.skip()
+        if not self._comm_chunk_read or not self._ssnd_chunk:
+            raise Error('COMM chunk and/or SSND chunk missing')
+
+    def __init__(self, f):
+        if isinstance(f, str):
+            f = builtins.open(f, 'rb')
+        # else, assume it is an open file object already
+        self.initfp(f)
+
+    #
+    # User visible methods.
+    #
+    def getfp(self):
+        return self._file
+
+    def rewind(self):
+        self._ssnd_seek_needed = 1
+        self._soundpos = 0
+
+    def close(self):
+        self._file.close()
+
+    def tell(self):
+        return self._soundpos
+
+    def getnchannels(self):
+        return self._nchannels
+
+    def getnframes(self):
+        return self._nframes
+
+    def getsampwidth(self):
+        return self._sampwidth
+
+    def getframerate(self):
+        return self._framerate
+
+    def getcomptype(self):
+        return self._comptype
+
+    def getcompname(self):
+        return self._compname
+
+##  def getversion(self):
+##      return self._version
+
+    def getparams(self):
+        return self.getnchannels(), self.getsampwidth(), \
+              self.getframerate(), self.getnframes(), \
+              self.getcomptype(), self.getcompname()
+
+    def getmarkers(self):
+        if len(self._markers) == 0:
+            return None
+        return self._markers
+
+    def getmark(self, id):
+        for marker in self._markers:
+            if id == marker[0]:
+                return marker
+        raise Error('marker {0!r} does not exist'.format(id))
+
+    def setpos(self, pos):
+        if pos < 0 or pos > self._nframes:
+            raise Error('position not in range')
+        self._soundpos = pos
+        self._ssnd_seek_needed = 1
+
+    def readframes(self, nframes):
+        if self._ssnd_seek_needed:
+            self._ssnd_chunk.seek(0)
+            dummy = self._ssnd_chunk.read(8)
+            pos = self._soundpos * self._framesize
+            if pos:
+                self._ssnd_chunk.seek(pos + 8)
+            self._ssnd_seek_needed = 0
+        if nframes == 0:
+            return b''
+        data = self._ssnd_chunk.read(nframes * self._framesize)
+        if self._convert and data:
+            data = self._convert(data)
+        self._soundpos = self._soundpos + len(data) // (self._nchannels
+                                                        * self._sampwidth)
+        return data
+
+    #
+    # Internal methods.
+    #
+
+    def _alaw2lin(self, data):
+        import audioop
+        return audioop.alaw2lin(data, 2)
+
+    def _ulaw2lin(self, data):
+        import audioop
+        return audioop.ulaw2lin(data, 2)
+
+    def _adpcm2lin(self, data):
+        import audioop
+        if not hasattr(self, '_adpcmstate'):
+            # first time
+            self._adpcmstate = None
+        data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
+        return data
+
+    def _read_comm_chunk(self, chunk):
+        self._nchannels = _read_short(chunk)
+        self._nframes = _read_long(chunk)
+        self._sampwidth = (_read_short(chunk) + 7) // 8
+        self._framerate = int(_read_float(chunk))
+        self._framesize = self._nchannels * self._sampwidth
+        if self._aifc:
+            #DEBUG: SGI's soundeditor produces a bad size :-(
+            kludge = 0
+            if chunk.chunksize == 18:
+                kludge = 1
+                print('Warning: bad COMM chunk size')
+                chunk.chunksize = 23
+            #DEBUG end
+            self._comptype = chunk.read(4)
+            #DEBUG start
+            if kludge:
+                length = ord(chunk.file.read(1))
+                if length & 1 == 0:
+                    length = length + 1
+                chunk.chunksize = chunk.chunksize + length
+                chunk.file.seek(-1, 1)
+            #DEBUG end
+            self._compname = _read_string(chunk)
+            if self._comptype != b'NONE':
+                if self._comptype == b'G722':
+                    self._convert = self._adpcm2lin
+                    self._framesize = self._framesize // 4
+                elif self._comptype in (b'ulaw', b'ULAW'):
+                    self._convert = self._ulaw2lin
+                    self._framesize = self._framesize // 2
+                elif self._comptype in (b'alaw', b'ALAW'):
+                    self._convert = self._alaw2lin
+                    self._framesize = self._framesize // 2
+                else:
+                    raise Error('unsupported compression type')
+        else:
+            self._comptype = b'NONE'
+            self._compname = b'not compressed'
+
+    def _readmark(self, chunk):
+        nmarkers = _read_short(chunk)
+        # Some files appear to contain invalid counts.
+        # Cope with this by testing for EOF.
+        try:
+            for i in range(nmarkers):
+                id = _read_short(chunk)
+                pos = _read_long(chunk)
+                name = _read_string(chunk)
+                if pos or name:
+                    # some files appear to have
+                    # dummy markers consisting of
+                    # a position 0 and name ''
+                    self._markers.append((id, pos, name))
+        except EOFError:
+            print('Warning: MARK chunk contains only', end=' ')
+            print(len(self._markers), end=' ')
+            if len(self._markers) == 1: print('marker', end=' ')
+            else: print('markers', end=' ')
+            print('instead of', nmarkers)
+
+class Aifc_write:
+    # Variables used in this class:
+    #
+    # These variables are user settable through appropriate methods
+    # of this class:
+    # _file -- the open file with methods write(), close(), tell(), seek()
+    #       set through the __init__() method
+    # _comptype -- the AIFF-C compression type ('NONE' in AIFF)
+    #       set through the setcomptype() or setparams() method
+    # _compname -- the human-readable AIFF-C compression type
+    #       set through the setcomptype() or setparams() method
+    # _nchannels -- the number of audio channels
+    #       set through the setnchannels() or setparams() method
+    # _sampwidth -- the number of bytes per audio sample
+    #       set through the setsampwidth() or setparams() method
+    # _framerate -- the sampling frequency
+    #       set through the setframerate() or setparams() method
+    # _nframes -- the number of audio frames written to the header
+    #       set through the setnframes() or setparams() method
+    # _aifc -- whether we're writing an AIFF-C file or an AIFF file
+    #       set through the aifc() method, reset through the
+    #       aiff() method
+    #
+    # These variables are used internally only:
+    # _version -- the AIFF-C version number
+    # _comp -- the compressor from builtin module cl
+    # _nframeswritten -- the number of audio frames actually written
+    # _datalength -- the size of the audio samples written to the header
+    # _datawritten -- the size of the audio samples actually written
+
+    def __init__(self, f):
+        if isinstance(f, str):
+            filename = f
+            f = builtins.open(f, 'wb')
+        else:
+            # else, assume it is an open file object already
+            filename = '???'
+        self.initfp(f)
+        if filename[-5:] == '.aiff':
+            self._aifc = 0
+        else:
+            self._aifc = 1
+
+    def initfp(self, file):
+        self._file = file
+        self._version = _AIFC_version
+        self._comptype = b'NONE'
+        self._compname = b'not compressed'
+        self._convert = None
+        self._nchannels = 0
+        self._sampwidth = 0
+        self._framerate = 0
+        self._nframes = 0
+        self._nframeswritten = 0
+        self._datawritten = 0
+        self._datalength = 0
+        self._markers = []
+        self._marklength = 0
+        self._aifc = 1      # AIFF-C is default
+
+    def __del__(self):
+        self.close()
+
+    #
+    # User visible methods.
+    #
+    def aiff(self):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._aifc = 0
+
+    def aifc(self):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._aifc = 1
+
+    def setnchannels(self, nchannels):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if nchannels < 1:
+            raise Error('bad # of channels')
+        self._nchannels = nchannels
+
+    def getnchannels(self):
+        if not self._nchannels:
+            raise Error('number of channels not set')
+        return self._nchannels
+
+    def setsampwidth(self, sampwidth):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if sampwidth < 1 or sampwidth > 4:
+            raise Error('bad sample width')
+        self._sampwidth = sampwidth
+
+    def getsampwidth(self):
+        if not self._sampwidth:
+            raise Error('sample width not set')
+        return self._sampwidth
+
+    def setframerate(self, framerate):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if framerate <= 0:
+            raise Error('bad frame rate')
+        self._framerate = framerate
+
+    def getframerate(self):
+        if not self._framerate:
+            raise Error('frame rate not set')
+        return self._framerate
+
+    def setnframes(self, nframes):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._nframes = nframes
+
+    def getnframes(self):
+        return self._nframeswritten
+
+    def setcomptype(self, comptype, compname):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if comptype not in (b'NONE', b'ulaw', b'ULAW',
+                            b'alaw', b'ALAW', b'G722'):
+            raise Error('unsupported compression type')
+        self._comptype = comptype
+        self._compname = compname
+
+    def getcomptype(self):
+        return self._comptype
+
+    def getcompname(self):
+        return self._compname
+
+##  def setversion(self, version):
+##      if self._nframeswritten:
+##          raise Error, 'cannot change parameters after starting to write'
+##      self._version = version
+
+    def setparams(self, params):
+        nchannels, sampwidth, framerate, nframes, comptype, compname = params
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if comptype not in (b'NONE', b'ulaw', b'ULAW',
+                            b'alaw', b'ALAW', b'G722'):
+            raise Error('unsupported compression type')
+        self.setnchannels(nchannels)
+        self.setsampwidth(sampwidth)
+        self.setframerate(framerate)
+        self.setnframes(nframes)
+        self.setcomptype(comptype, compname)
+
+    def getparams(self):
+        if not self._nchannels or not self._sampwidth or not self._framerate:
+            raise Error('not all parameters set')
+        return self._nchannels, self._sampwidth, self._framerate, \
+              self._nframes, self._comptype, self._compname
+
+    def setmark(self, id, pos, name):
+        if id <= 0:
+            raise Error('marker ID must be > 0')
+        if pos < 0:
+            raise Error('marker position must be >= 0')
+        if not isinstance(name, bytes):
+            raise Error('marker name must be bytes')
+        for i in range(len(self._markers)):
+            if id == self._markers[i][0]:
+                self._markers[i] = id, pos, name
+                return
+        self._markers.append((id, pos, name))
+
+    def getmark(self, id):
+        for marker in self._markers:
+            if id == marker[0]:
+                return marker
+        raise Error('marker {0!r} does not exist'.format(id))
+
+    def getmarkers(self):
+        if len(self._markers) == 0:
+            return None
+        return self._markers
+
+    def tell(self):
+        return self._nframeswritten
+
+    def writeframesraw(self, data):
+        self._ensure_header_written(len(data))
+        nframes = len(data) // (self._sampwidth * self._nchannels)
+        if self._convert:
+            data = self._convert(data)
+        self._file.write(data)
+        self._nframeswritten = self._nframeswritten + nframes
+        self._datawritten = self._datawritten + len(data)
+
+    def writeframes(self, data):
+        self.writeframesraw(data)
+        if self._nframeswritten != self._nframes or \
+              self._datalength != self._datawritten:
+            self._patchheader()
+
+    def close(self):
+        if self._file:
+            self._ensure_header_written(0)
+            if self._datawritten & 1:
+                # quick pad to even size
+                self._file.write(b'\x00')
+                self._datawritten = self._datawritten + 1
+            self._writemarkers()
+            if self._nframeswritten != self._nframes or \
+                  self._datalength != self._datawritten or \
+                  self._marklength:
+                self._patchheader()
+            # Prevent ref cycles
+            self._convert = None
+            self._file.close()
+            self._file = None
+
+    #
+    # Internal methods.
+    #
+
+    def _lin2alaw(self, data):
+        import audioop
+        return audioop.lin2alaw(data, 2)
+
+    def _lin2ulaw(self, data):
+        import audioop
+        return audioop.lin2ulaw(data, 2)
+
+    def _lin2adpcm(self, data):
+        import audioop
+        if not hasattr(self, '_adpcmstate'):
+            self._adpcmstate = None
+        data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
+        return data
+
+    def _ensure_header_written(self, datasize):
+        if not self._nframeswritten:
+            if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
+                if not self._sampwidth:
+                    self._sampwidth = 2
+                if self._sampwidth != 2:
+                    raise Error('sample width must be 2 when compressing '
+                                'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
+            if not self._nchannels:
+                raise Error('# channels not specified')
+            if not self._sampwidth:
+                raise Error('sample width not specified')
+            if not self._framerate:
+                raise Error('sampling rate not specified')
+            self._write_header(datasize)
+
+    def _init_compression(self):
+        if self._comptype == b'G722':
+            self._convert = self._lin2adpcm
+        elif self._comptype in (b'ulaw', b'ULAW'):
+            self._convert = self._lin2ulaw
+        elif self._comptype in (b'alaw', b'ALAW'):
+            self._convert = self._lin2alaw
+
+    def _write_header(self, initlength):
+        if self._aifc and self._comptype != b'NONE':
+            self._init_compression()
+        self._file.write(b'FORM')
+        if not self._nframes:
+            self._nframes = initlength // (self._nchannels * self._sampwidth)
+        self._datalength = self._nframes * self._nchannels * self._sampwidth
+        if self._datalength & 1:
+            self._datalength = self._datalength + 1
+        if self._aifc:
+            if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'):
+                self._datalength = self._datalength // 2
+                if self._datalength & 1:
+                    self._datalength = self._datalength + 1
+            elif self._comptype == b'G722':
+                self._datalength = (self._datalength + 3) // 4
+                if self._datalength & 1:
+                    self._datalength = self._datalength + 1
+        self._form_length_pos = self._file.tell()
+        commlength = self._write_form_length(self._datalength)
+        if self._aifc:
+            self._file.write(b'AIFC')
+            self._file.write(b'FVER')
+            _write_ulong(self._file, 4)
+            _write_ulong(self._file, self._version)
+        else:
+            self._file.write(b'AIFF')
+        self._file.write(b'COMM')
+        _write_ulong(self._file, commlength)
+        _write_short(self._file, self._nchannels)
+        self._nframes_pos = self._file.tell()
+        _write_ulong(self._file, self._nframes)
+        _write_short(self._file, self._sampwidth * 8)
+        _write_float(self._file, self._framerate)
+        if self._aifc:
+            self._file.write(self._comptype)
+            _write_string(self._file, self._compname)
+        self._file.write(b'SSND')
+        self._ssnd_length_pos = self._file.tell()
+        _write_ulong(self._file, self._datalength + 8)
+        _write_ulong(self._file, 0)
+        _write_ulong(self._file, 0)
+
+    def _write_form_length(self, datalength):
+        if self._aifc:
+            commlength = 18 + 5 + len(self._compname)
+            if commlength & 1:
+                commlength = commlength + 1
+            verslength = 12
+        else:
+            commlength = 18
+            verslength = 0
+        _write_ulong(self._file, 4 + verslength + self._marklength + \
+                     8 + commlength + 16 + datalength)
+        return commlength
+
+    def _patchheader(self):
+        curpos = self._file.tell()
+        if self._datawritten & 1:
+            datalength = self._datawritten + 1
+            self._file.write(b'\x00')
+        else:
+            datalength = self._datawritten
+        if datalength == self._datalength and \
+              self._nframes == self._nframeswritten and \
+              self._marklength == 0:
+            self._file.seek(curpos, 0)
+            return
+        self._file.seek(self._form_length_pos, 0)
+        dummy = self._write_form_length(datalength)
+        self._file.seek(self._nframes_pos, 0)
+        _write_ulong(self._file, self._nframeswritten)
+        self._file.seek(self._ssnd_length_pos, 0)
+        _write_ulong(self._file, datalength + 8)
+        self._file.seek(curpos, 0)
+        self._nframes = self._nframeswritten
+        self._datalength = datalength
+
+    def _writemarkers(self):
+        if len(self._markers) == 0:
+            return
+        self._file.write(b'MARK')
+        length = 2
+        for marker in self._markers:
+            id, pos, name = marker
+            length = length + len(name) + 1 + 6
+            if len(name) & 1 == 0:
+                length = length + 1
+        _write_ulong(self._file, length)
+        self._marklength = length + 8
+        _write_short(self._file, len(self._markers))
+        for marker in self._markers:
+            id, pos, name = marker
+            _write_short(self._file, id)
+            _write_ulong(self._file, pos)
+            _write_string(self._file, name)
+
+def open(f, mode=None):
+    if mode is None:
+        if hasattr(f, 'mode'):
+            mode = f.mode
+        else:
+            mode = 'rb'
+    if mode in ('r', 'rb'):
+        return Aifc_read(f)
+    elif mode in ('w', 'wb'):
+        return Aifc_write(f)
+    else:
+        raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
+
+openfp = open # B/W compatibility
+
+if __name__ == '__main__':
+    import sys
+    if not sys.argv[1:]:
+        sys.argv.append('/usr/demos/data/audio/bach.aiff')
+    fn = sys.argv[1]
+    f = open(fn, 'r')
+    print("Reading", fn)
+    print("nchannels =", f.getnchannels())
+    print("nframes   =", f.getnframes())
+    print("sampwidth =", f.getsampwidth())
+    print("framerate =", f.getframerate())
+    print("comptype  =", f.getcomptype())
+    print("compname  =", f.getcompname())
+    if sys.argv[2:]:
+        gn = sys.argv[2]
+        print("Writing", gn)
+        g = open(gn, 'w')
+        g.setparams(f.getparams())
+        while 1:
+            data = f.readframes(1024)
+            if not data:
+                break
+            g.writeframes(data)
+        g.close()
+        f.close()
+        print("Done.")
diff --git a/lib-python/3.2/antigravity.py b/lib-python/3.2/antigravity.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/antigravity.py
@@ -0,0 +1,17 @@
+
+import webbrowser
+import hashlib
+
+webbrowser.open("http://xkcd.com/353/")
+
+def geohash(latitude, longitude, datedow):
+    '''Compute geohash() using the Munroe algorithm.
+
+    >>> geohash(37.421542, -122.085589, b'2005-05-26-10458.68')
+    37.857713 -122.544543
+
+    '''
+    # http://xkcd.com/426/
+    h = hashlib.md5(datedow).hexdigest()
+    p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
+    print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:]))
diff --git a/lib-python/3.2/argparse.py b/lib-python/3.2/argparse.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/argparse.py
@@ -0,0 +1,2362 @@
+# Author: Steven J. Bethard <steven.bethard at gmail.com>.
+
+"""Command-line parsing library
+
+This module is an optparse-inspired command-line parsing library that:
+
+    - handles both optional and positional arguments
+    - produces highly informative usage messages
+    - supports parsers that dispatch to sub-parsers
+
+The following is a simple usage example that sums integers from the
+command-line and writes the result to a file::
+
+    parser = argparse.ArgumentParser(
+        description='sum the integers at the command line')
+    parser.add_argument(
+        'integers', metavar='int', nargs='+', type=int,
+        help='an integer to be summed')
+    parser.add_argument(
+        '--log', default=sys.stdout, type=argparse.FileType('w'),
+        help='the file where the sum should be written')
+    args = parser.parse_args()
+    args.log.write('%s' % sum(args.integers))
+    args.log.close()
+
+The module contains the following public classes:
+
+    - ArgumentParser -- The main entry point for command-line parsing. As the
+        example above shows, the add_argument() method is used to populate
+        the parser with actions for optional and positional arguments. Then
+        the parse_args() method is invoked to convert the args at the
+        command-line into an object with attributes.
+
+    - ArgumentError -- The exception raised by ArgumentParser objects when
+        there are errors with the parser's actions. Errors raised while
+        parsing the command-line are caught by ArgumentParser and emitted
+        as command-line messages.
+
+    - FileType -- A factory for defining types of files to be created. As the
+        example above shows, instances of FileType are typically passed as
+        the type= argument of add_argument() calls.
+
+    - Action -- The base class for parser actions. Typically actions are
+        selected by passing strings like 'store_true' or 'append_const' to
+        the action= argument of add_argument(). However, for greater
+        customization of ArgumentParser actions, subclasses of Action may
+        be defined and passed as the action= argument.
+
+    - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter,
+        ArgumentDefaultsHelpFormatter -- Formatter classes which
+        may be passed as the formatter_class= argument to the
+        ArgumentParser constructor. HelpFormatter is the default,
+        RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser
+        not to change the formatting for help text, and
+        ArgumentDefaultsHelpFormatter adds information about argument defaults
+        to the help.
+
+All other classes in this module are considered implementation details.
+(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
+considered public as object names -- the API of the formatter objects is
+still considered an implementation detail.)
+"""
+
+__version__ = '1.1'
+__all__ = [
+    'ArgumentParser',
+    'ArgumentError',
+    'ArgumentTypeError',
+    'FileType',
+    'HelpFormatter',
+    'ArgumentDefaultsHelpFormatter',
+    'RawDescriptionHelpFormatter',
+    'RawTextHelpFormatter',
+    'Namespace',
+    'Action',
+    'ONE_OR_MORE',
+    'OPTIONAL',
+    'PARSER',
+    'REMAINDER',
+    'SUPPRESS',
+    'ZERO_OR_MORE',
+]
+
+
+import collections as _collections
+import copy as _copy
+import os as _os
+import re as _re
+import sys as _sys
+import textwrap as _textwrap
+
+from gettext import gettext as _, ngettext
+
+
+SUPPRESS = '==SUPPRESS=='
+
+OPTIONAL = '?'
+ZERO_OR_MORE = '*'
+ONE_OR_MORE = '+'
+PARSER = 'A...'
+REMAINDER = '...'
+_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args'
+
+# =============================
+# Utility functions and classes
+# =============================
+
+class _AttributeHolder(object):
+    """Abstract base class that provides __repr__.
+
+    The __repr__ method returns a string in the format::
+        ClassName(attr=name, attr=name, ...)
+    The attributes are determined either by a class-level attribute,
+    '_kwarg_names', or by inspecting the instance __dict__.
+    """
+
+    def __repr__(self):
+        type_name = type(self).__name__
+        arg_strings = []
+        for arg in self._get_args():
+            arg_strings.append(repr(arg))
+        for name, value in self._get_kwargs():
+            arg_strings.append('%s=%r' % (name, value))
+        return '%s(%s)' % (type_name, ', '.join(arg_strings))
+
+    def _get_kwargs(self):
+        return sorted(self.__dict__.items())
+
+    def _get_args(self):
+        return []
+
+
+def _ensure_value(namespace, name, value):
+    if getattr(namespace, name, None) is None:
+        setattr(namespace, name, value)
+    return getattr(namespace, name)
+
+
+# ===============
+# Formatting Help
+# ===============
+
+class HelpFormatter(object):
+    """Formatter for generating usage messages and argument help strings.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def __init__(self,
+                 prog,
+                 indent_increment=2,
+                 max_help_position=24,
+                 width=None):
+
+        # default setting for width
+        if width is None:
+            try:
+                width = int(_os.environ['COLUMNS'])
+            except (KeyError, ValueError):
+                width = 80
+            width -= 2
+
+        self._prog = prog
+        self._indent_increment = indent_increment
+        self._max_help_position = max_help_position
+        self._width = width
+
+        self._current_indent = 0
+        self._level = 0
+        self._action_max_length = 0
+
+        self._root_section = self._Section(self, None)
+        self._current_section = self._root_section
+
+        self._whitespace_matcher = _re.compile(r'\s+')
+        self._long_break_matcher = _re.compile(r'\n\n\n+')
+
+    # ===============================
+    # Section and indentation methods
+    # ===============================
+    def _indent(self):
+        self._current_indent += self._indent_increment
+        self._level += 1
+
+    def _dedent(self):
+        self._current_indent -= self._indent_increment
+        assert self._current_indent >= 0, 'Indent decreased below 0.'
+        self._level -= 1
+
+    class _Section(object):
+
+        def __init__(self, formatter, parent, heading=None):
+            self.formatter = formatter
+            self.parent = parent
+            self.heading = heading
+            self.items = []
+
+        def format_help(self):
+            # format the indented section
+            if self.parent is not None:
+                self.formatter._indent()
+            join = self.formatter._join_parts
+            for func, args in self.items:
+                func(*args)
+            item_help = join([func(*args) for func, args in self.items])
+            if self.parent is not None:
+                self.formatter._dedent()
+
+            # return nothing if the section was empty
+            if not item_help:
+                return ''
+
+            # add the heading if the section was non-empty
+            if self.heading is not SUPPRESS and self.heading is not None:
+                current_indent = self.formatter._current_indent
+                heading = '%*s%s:\n' % (current_indent, '', self.heading)
+            else:
+                heading = ''
+
+            # join the section-initial newline, the heading and the help
+            return join(['\n', heading, item_help, '\n'])
+
+    def _add_item(self, func, args):
+        self._current_section.items.append((func, args))
+
+    # ========================
+    # Message building methods
+    # ========================
+    def start_section(self, heading):
+        self._indent()
+        section = self._Section(self, self._current_section, heading)
+        self._add_item(section.format_help, [])
+        self._current_section = section
+
+    def end_section(self):
+        self._current_section = self._current_section.parent
+        self._dedent()
+
+    def add_text(self, text):
+        if text is not SUPPRESS and text is not None:
+            self._add_item(self._format_text, [text])
+
+    def add_usage(self, usage, actions, groups, prefix=None):
+        if usage is not SUPPRESS:
+            args = usage, actions, groups, prefix
+            self._add_item(self._format_usage, args)
+
+    def add_argument(self, action):
+        if action.help is not SUPPRESS:
+
+            # find all invocations
+            get_invocation = self._format_action_invocation
+            invocations = [get_invocation(action)]
+            for subaction in self._iter_indented_subactions(action):
+                invocations.append(get_invocation(subaction))
+
+            # update the maximum item length
+            invocation_length = max([len(s) for s in invocations])
+            action_length = invocation_length + self._current_indent
+            self._action_max_length = max(self._action_max_length,
+                                          action_length)
+
+            # add the item to the list
+            self._add_item(self._format_action, [action])
+
+    def add_arguments(self, actions):
+        for action in actions:
+            self.add_argument(action)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+    def format_help(self):
+        help = self._root_section.format_help()
+        if help:
+            help = self._long_break_matcher.sub('\n\n', help)
+            help = help.strip('\n') + '\n'
+        return help
+
+    def _join_parts(self, part_strings):
+        return ''.join([part
+                        for part in part_strings
+                        if part and part is not SUPPRESS])
+
+    def _format_usage(self, usage, actions, groups, prefix):
+        if prefix is None:
+            prefix = _('usage: ')
+
+        # if usage is specified, use that
+        if usage is not None:
+            usage = usage % dict(prog=self._prog)
+
+        # if no optionals or positionals are available, usage is just prog
+        elif usage is None and not actions:
+            usage = '%(prog)s' % dict(prog=self._prog)
+
+        # if optionals and positionals are available, calculate usage
+        elif usage is None:
+            prog = '%(prog)s' % dict(prog=self._prog)
+
+            # split optionals from positionals
+            optionals = []
+            positionals = []
+            for action in actions:
+                if action.option_strings:
+                    optionals.append(action)
+                else:
+                    positionals.append(action)
+
+            # build full usage string
+            format = self._format_actions_usage
+            action_usage = format(optionals + positionals, groups)
+            usage = ' '.join([s for s in [prog, action_usage] if s])
+
+            # wrap the usage parts if it's too long
+            text_width = self._width - self._current_indent
+            if len(prefix) + len(usage) > text_width:
+
+                # break usage into wrappable parts
+                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
+                opt_usage = format(optionals, groups)
+                pos_usage = format(positionals, groups)
+                opt_parts = _re.findall(part_regexp, opt_usage)
+                pos_parts = _re.findall(part_regexp, pos_usage)
+                assert ' '.join(opt_parts) == opt_usage
+                assert ' '.join(pos_parts) == pos_usage
+
+                # helper for wrapping lines
+                def get_lines(parts, indent, prefix=None):
+                    lines = []
+                    line = []
+                    if prefix is not None:
+                        line_len = len(prefix) - 1
+                    else:
+                        line_len = len(indent) - 1
+                    for part in parts:
+                        if line_len + 1 + len(part) > text_width:
+                            lines.append(indent + ' '.join(line))
+                            line = []
+                            line_len = len(indent) - 1
+                        line.append(part)
+                        line_len += len(part) + 1
+                    if line:
+                        lines.append(indent + ' '.join(line))
+                    if prefix is not None:
+                        lines[0] = lines[0][len(indent):]
+                    return lines
+
+                # if prog is short, follow it with optionals or positionals
+                if len(prefix) + len(prog) <= 0.75 * text_width:
+                    indent = ' ' * (len(prefix) + len(prog) + 1)
+                    if opt_parts:
+                        lines = get_lines([prog] + opt_parts, indent, prefix)
+                        lines.extend(get_lines(pos_parts, indent))
+                    elif pos_parts:
+                        lines = get_lines([prog] + pos_parts, indent, prefix)
+                    else:
+                        lines = [prog]
+
+                # if prog is long, put it on its own line
+                else:
+                    indent = ' ' * len(prefix)
+                    parts = opt_parts + pos_parts
+                    lines = get_lines(parts, indent)
+                    if len(lines) > 1:
+                        lines = []
+                        lines.extend(get_lines(opt_parts, indent))
+                        lines.extend(get_lines(pos_parts, indent))
+                    lines = [prog] + lines
+
+                # join lines into usage
+                usage = '\n'.join(lines)
+
+        # prefix with 'usage:'
+        return '%s%s\n\n' % (prefix, usage)
+
+    def _format_actions_usage(self, actions, groups):
+        # find group indices and identify actions in groups
+        group_actions = set()
+        inserts = {}
+        for group in groups:
+            try:
+                start = actions.index(group._group_actions[0])
+            except ValueError:
+                continue
+            else:
+                end = start + len(group._group_actions)
+                if actions[start:end] == group._group_actions:
+                    for action in group._group_actions:
+                        group_actions.add(action)
+                    if not group.required:
+                        if start in inserts:
+                            inserts[start] += ' ['
+                        else:
+                            inserts[start] = '['
+                        inserts[end] = ']'
+                    else:
+                        if start in inserts:
+                            inserts[start] += ' ('
+                        else:
+                            inserts[start] = '('
+                        inserts[end] = ')'
+                    for i in range(start + 1, end):
+                        inserts[i] = '|'
+
+        # collect all actions format strings
+        parts = []
+        for i, action in enumerate(actions):
+
+            # suppressed arguments are marked with None
+            # remove | separators for suppressed arguments
+            if action.help is SUPPRESS:
+                parts.append(None)
+                if inserts.get(i) == '|':
+                    inserts.pop(i)
+                elif inserts.get(i + 1) == '|':
+                    inserts.pop(i + 1)
+
+            # produce all arg strings
+            elif not action.option_strings:
+                part = self._format_args(action, action.dest)
+
+                # if it's in a group, strip the outer []
+                if action in group_actions:
+                    if part[0] == '[' and part[-1] == ']':
+                        part = part[1:-1]
+
+                # add the action string to the list
+                parts.append(part)
+
+            # produce the first way to invoke the option in brackets
+            else:
+                option_string = action.option_strings[0]
+
+                # if the Optional doesn't take a value, format is:
+                #    -s or --long
+                if action.nargs == 0:
+                    part = '%s' % option_string
+
+                # if the Optional takes a value, format is:
+                #    -s ARGS or --long ARGS
+                else:
+                    default = action.dest.upper()
+                    args_string = self._format_args(action, default)
+                    part = '%s %s' % (option_string, args_string)
+
+                # make it look optional if it's not required or in a group
+                if not action.required and action not in group_actions:
+                    part = '[%s]' % part
+
+                # add the action string to the list
+                parts.append(part)
+
+        # insert things at the necessary indices
+        for i in sorted(inserts, reverse=True):
+            parts[i:i] = [inserts[i]]
+
+        # join all the action items with spaces
+        text = ' '.join([item for item in parts if item is not None])
+
+        # clean up separators for mutually exclusive groups
+        open = r'[\[(]'
+        close = r'[\])]'
+        text = _re.sub(r'(%s) ' % open, r'\1', text)
+        text = _re.sub(r' (%s)' % close, r'\1', text)
+        text = _re.sub(r'%s *%s' % (open, close), r'', text)
+        text = _re.sub(r'\(([^|]*)\)', r'\1', text)
+        text = text.strip()
+
+        # return the text
+        return text
+
+    def _format_text(self, text):
+        if '%(prog)' in text:
+            text = text % dict(prog=self._prog)
+        text_width = self._width - self._current_indent
+        indent = ' ' * self._current_indent
+        return self._fill_text(text, text_width, indent) + '\n\n'
+
+    def _format_action(self, action):
+        # determine the required width and the entry label
+        help_position = min(self._action_max_length + 2,
+                            self._max_help_position)
+        help_width = self._width - help_position
+        action_width = help_position - self._current_indent - 2
+        action_header = self._format_action_invocation(action)
+
+        # ho nelp; start on same line and add a final newline
+        if not action.help:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+
+        # short action name; start on the same line and pad two spaces
+        elif len(action_header) <= action_width:
+            tup = self._current_indent, '', action_width, action_header
+            action_header = '%*s%-*s  ' % tup
+            indent_first = 0
+
+        # long action name; start on the next line
+        else:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+            indent_first = help_position
+
+        # collect the pieces of the action help
+        parts = [action_header]
+
+        # if there was help for the action, add lines of help text
+        if action.help:
+            help_text = self._expand_help(action)
+            help_lines = self._split_lines(help_text, help_width)
+            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
+            for line in help_lines[1:]:
+                parts.append('%*s%s\n' % (help_position, '', line))
+
+        # or add a newline if the description doesn't end with one
+        elif not action_header.endswith('\n'):
+            parts.append('\n')
+
+        # if there are any sub-actions, add their help as well
+        for subaction in self._iter_indented_subactions(action):
+            parts.append(self._format_action(subaction))
+
+        # return a single string
+        return self._join_parts(parts)
+
+    def _format_action_invocation(self, action):
+        if not action.option_strings:
+            metavar, = self._metavar_formatter(action, action.dest)(1)
+            return metavar
+
+        else:
+            parts = []
+
+            # if the Optional doesn't take a value, format is:
+            #    -s, --long
+            if action.nargs == 0:
+                parts.extend(action.option_strings)
+
+            # if the Optional takes a value, format is:
+            #    -s ARGS, --long ARGS
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    parts.append('%s %s' % (option_string, args_string))
+
+            return ', '.join(parts)
+
+    def _metavar_formatter(self, action, default_metavar):
+        if action.metavar is not None:
+            result = action.metavar
+        elif action.choices is not None:
+            choice_strs = [str(choice) for choice in action.choices]
+            result = '{%s}' % ','.join(choice_strs)
+        else:
+            result = default_metavar
+
+        def format(tuple_size):
+            if isinstance(result, tuple):
+                return result
+            else:
+                return (result, ) * tuple_size
+        return format
+
+    def _format_args(self, action, default_metavar):
+        get_metavar = self._metavar_formatter(action, default_metavar)
+        if action.nargs is None:
+            result = '%s' % get_metavar(1)
+        elif action.nargs == OPTIONAL:
+            result = '[%s]' % get_metavar(1)
+        elif action.nargs == ZERO_OR_MORE:
+            result = '[%s [%s ...]]' % get_metavar(2)
+        elif action.nargs == ONE_OR_MORE:
+            result = '%s [%s ...]' % get_metavar(2)
+        elif action.nargs == REMAINDER:
+            result = '...'
+        elif action.nargs == PARSER:
+            result = '%s ...' % get_metavar(1)
+        else:
+            formats = ['%s' for _ in range(action.nargs)]
+            result = ' '.join(formats) % get_metavar(action.nargs)
+        return result
+
+    def _expand_help(self, action):
+        params = dict(vars(action), prog=self._prog)
+        for name in list(params):
+            if params[name] is SUPPRESS:
+                del params[name]
+        for name in list(params):
+            if hasattr(params[name], '__name__'):
+                params[name] = params[name].__name__
+        if params.get('choices') is not None:
+            choices_str = ', '.join([str(c) for c in params['choices']])
+            params['choices'] = choices_str
+        return self._get_help_string(action) % params
+
+    def _iter_indented_subactions(self, action):
+        try:
+            get_subactions = action._get_subactions
+        except AttributeError:
+            pass
+        else:
+            self._indent()
+            for subaction in get_subactions():
+                yield subaction
+            self._dedent()
+
+    def _split_lines(self, text, width):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.wrap(text, width)
+
+    def _fill_text(self, text, width, indent):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.fill(text, width, initial_indent=indent,
+                                           subsequent_indent=indent)
+
+    def _get_help_string(self, action):
+        return action.help
+
+
+class RawDescriptionHelpFormatter(HelpFormatter):
+    """Help message formatter which retains any formatting in descriptions.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _fill_text(self, text, width, indent):
+        return ''.join([indent + line for line in text.splitlines(True)])
+
+
+class RawTextHelpFormatter(RawDescriptionHelpFormatter):
+    """Help message formatter which retains formatting of all help text.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _split_lines(self, text, width):
+        return text.splitlines()
+
+
+class ArgumentDefaultsHelpFormatter(HelpFormatter):
+    """Help message formatter which adds default values to argument help.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _get_help_string(self, action):
+        help = action.help
+        if '%(default)' not in action.help:
+            if action.default is not SUPPRESS:
+                defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
+                if action.option_strings or action.nargs in defaulting_nargs:
+                    help += ' (default: %(default)s)'
+        return help
+
+
+# =====================
+# Options and Arguments
+# =====================
+
+def _get_action_name(argument):
+    if argument is None:
+        return None
+    elif argument.option_strings:
+        return  '/'.join(argument.option_strings)
+    elif argument.metavar not in (None, SUPPRESS):
+        return argument.metavar
+    elif argument.dest not in (None, SUPPRESS):
+        return argument.dest
+    else:
+        return None
+
+
+class ArgumentError(Exception):
+    """An error from creating or using an argument (optional or positional).
+
+    The string value of this exception is the message, augmented with
+    information about the argument that caused it.
+    """
+
+    def __init__(self, argument, message):
+        self.argument_name = _get_action_name(argument)
+        self.message = message
+
+    def __str__(self):
+        if self.argument_name is None:
+            format = '%(message)s'
+        else:
+            format = 'argument %(argument_name)s: %(message)s'
+        return format % dict(message=self.message,
+                             argument_name=self.argument_name)
+
+
+class ArgumentTypeError(Exception):
+    """An error from trying to convert a command line string to a type."""
+    pass
+
+
+# ==============
+# Action classes
+# ==============
+
+class Action(_AttributeHolder):
+    """Information about how to convert command line strings to Python objects.
+
+    Action objects are used by an ArgumentParser to represent the information
+    needed to parse a single argument from one or more strings from the
+    command line. The keyword arguments to the Action constructor are also
+    all attributes of Action instances.
+
+    Keyword Arguments:
+
+        - option_strings -- A list of command-line option strings which
+            should be associated with this action.
+
+        - dest -- The name of the attribute to hold the created object(s)
+
+        - nargs -- The number of command-line arguments that should be
+            consumed. By default, one argument will be consumed and a single
+            value will be produced.  Other values include:
+                - N (an integer) consumes N arguments (and produces a list)
+                - '?' consumes zero or one arguments
+                - '*' consumes zero or more arguments (and produces a list)
+                - '+' consumes one or more arguments (and produces a list)
+            Note that the difference between the default and nargs=1 is that
+            with the default, a single value will be produced, while with
+            nargs=1, a list containing a single value will be produced.
+
+        - const -- The value to be produced if the option is specified and the
+            option uses an action that takes no values.
+
+        - default -- The value to be produced if the option is not specified.
+
+        - type -- The type which the command-line arguments should be converted
+            to, should be one of 'string', 'int', 'float', 'complex' or a
+            callable object that accepts a single string argument. If None,
+            'string' is assumed.
+
+        - choices -- A container of values that should be allowed. If not None,
+            after a command-line argument has been converted to the appropriate
+            type, an exception will be raised if it is not a member of this
+            collection.
+
+        - required -- True if the action must always be specified at the
+            command line. This is only meaningful for optional command-line
+            arguments.
+
+        - help -- The help string describing the argument.
+
+        - metavar -- The name to be used for the option's argument with the
+            help string. If None, the 'dest' value will be used as the name.
+    """
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        self.option_strings = option_strings
+        self.dest = dest
+        self.nargs = nargs
+        self.const = const
+        self.default = default
+        self.type = type
+        self.choices = choices
+        self.required = required
+        self.help = help
+        self.metavar = metavar
+
+    def _get_kwargs(self):
+        names = [
+            'option_strings',
+            'dest',
+            'nargs',
+            'const',
+            'default',
+            'type',
+            'choices',
+            'help',
+            'metavar',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        raise NotImplementedError(_('.__call__() not defined'))
+
+
+class _StoreAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs for store actions must be > 0; if you '
+                             'have nothing to store, actions such as store '
+                             'true or store const may be more appropriate')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_StoreAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+
+class _StoreConstAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_StoreConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, self.const)
+
+
+class _StoreTrueAction(_StoreConstAction):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=False,
+                 required=False,
+                 help=None):
+        super(_StoreTrueAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=True,
+            default=default,
+            required=required,
+            help=help)
+
+
+class _StoreFalseAction(_StoreConstAction):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=True,
+                 required=False,
+                 help=None):
+        super(_StoreFalseAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=False,
+            default=default,
+            required=required,
+            help=help)
+
+
+class _AppendAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs for append actions must be > 0; if arg '
+                             'strings are not supplying the value to append, '
+                             'the append const action may be more appropriate')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_AppendAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = _copy.copy(_ensure_value(namespace, self.dest, []))
+        items.append(values)
+        setattr(namespace, self.dest, items)
+
+
+class _AppendConstAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_AppendConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = _copy.copy(_ensure_value(namespace, self.dest, []))
+        items.append(self.const)
+        setattr(namespace, self.dest, items)
+
+
+class _CountAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=None,
+                 required=False,
+                 help=None):
+        super(_CountAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        new_count = _ensure_value(namespace, self.dest, 0) + 1
+        setattr(namespace, self.dest, new_count)
+
+
+class _HelpAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help=None):
+        super(_HelpAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_help()
+        parser.exit()
+
+
+class _VersionAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 version=None,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help="show program's version number and exit"):
+        super(_VersionAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+        self.version = version
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        version = self.version
+        if version is None:
+            version = parser.version
+        formatter = parser._get_formatter()
+        formatter.add_text(version)
+        parser.exit(message=formatter.format_help())
+
+
+class _SubParsersAction(Action):
+
+    class _ChoicesPseudoAction(Action):
+
+        def __init__(self, name, aliases, help):
+            metavar = dest = name
+            if aliases:
+                metavar += ' (%s)' % ', '.join(aliases)
+            sup = super(_SubParsersAction._ChoicesPseudoAction, self)
+            sup.__init__(option_strings=[], dest=dest, help=help,
+                         metavar=metavar)
+
+    def __init__(self,
+                 option_strings,
+                 prog,
+                 parser_class,
+                 dest=SUPPRESS,
+                 help=None,
+                 metavar=None):
+
+        self._prog_prefix = prog
+        self._parser_class = parser_class
+        self._name_parser_map = _collections.OrderedDict()
+        self._choices_actions = []
+
+        super(_SubParsersAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=PARSER,
+            choices=self._name_parser_map,
+            help=help,
+            metavar=metavar)
+
+    def add_parser(self, name, **kwargs):
+        # set prog from the existing prefix
+        if kwargs.get('prog') is None:
+            kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
+
+        aliases = kwargs.pop('aliases', ())
+
+        # create a pseudo-action to hold the choice help
+        if 'help' in kwargs:
+            help = kwargs.pop('help')
+            choice_action = self._ChoicesPseudoAction(name, aliases, help)
+            self._choices_actions.append(choice_action)
+
+        # create the parser and add it to the map
+        parser = self._parser_class(**kwargs)
+        self._name_parser_map[name] = parser
+
+        # make parser available under aliases also
+        for alias in aliases:
+            self._name_parser_map[alias] = parser
+
+        return parser
+
+    def _get_subactions(self):
+        return self._choices_actions
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser_name = values[0]
+        arg_strings = values[1:]
+
+        # set the parser name if requested
+        if self.dest is not SUPPRESS:
+            setattr(namespace, self.dest, parser_name)
+
+        # select the parser
+        try:
+            parser = self._name_parser_map[parser_name]
+        except KeyError:
+            args = {'parser_name': parser_name,
+                    'choices': ', '.join(self._name_parser_map)}
+            msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
+            raise ArgumentError(self, msg)
+
+        # parse all the remaining options into the namespace
+        # store any unrecognized options on the object, so that the top
+        # level parser can decide what to do with them
+        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+        if arg_strings:
+            vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
+            getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
+
+
+# ==============
+# Type classes
+# ==============
+
+class FileType(object):
+    """Factory for creating file object types
+
+    Instances of FileType are typically passed as type= arguments to the
+    ArgumentParser add_argument() method.
+
+    Keyword Arguments:
+        - mode -- A string indicating how the file is to be opened. Accepts the
+            same values as the builtin open() function.
+        - bufsize -- The file's desired buffer size. Accepts the same values as
+            the builtin open() function.
+    """
+
+    def __init__(self, mode='r', bufsize=-1):
+        self._mode = mode
+        self._bufsize = bufsize
+
+    def __call__(self, string):
+        # the special argument "-" means sys.std{in,out}
+        if string == '-':
+            if 'r' in self._mode:
+                return _sys.stdin
+            elif 'w' in self._mode:
+                return _sys.stdout
+            else:
+                msg = _('argument "-" with mode %r') % self._mode
+                raise ValueError(msg)
+
+        # all other arguments are used as file names
+        try:
+            return open(string, self._mode, self._bufsize)
+        except IOError as e:
+            message = _("can't open '%s': %s")
+            raise ArgumentTypeError(message % (string, e))
+
+    def __repr__(self):
+        args = self._mode, self._bufsize
+        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
+        return '%s(%s)' % (type(self).__name__, args_str)
+
+# ===========================
+# Optional and Positional Parsing
+# ===========================
+
+class Namespace(_AttributeHolder):
+    """Simple object for storing attributes.
+
+    Implements equality by attribute names and values, and provides a simple
+    string representation.
+    """
+
+    def __init__(self, **kwargs):
+        for name in kwargs:
+            setattr(self, name, kwargs[name])
+
+    def __eq__(self, other):
+        return vars(self) == vars(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+
+class _ActionsContainer(object):
+
+    def __init__(self,
+                 description,
+                 prefix_chars,
+                 argument_default,
+                 conflict_handler):
+        super(_ActionsContainer, self).__init__()
+
+        self.description = description
+        self.argument_default = argument_default
+        self.prefix_chars = prefix_chars
+        self.conflict_handler = conflict_handler
+
+        # set up registries
+        self._registries = {}
+
+        # register actions
+        self.register('action', None, _StoreAction)
+        self.register('action', 'store', _StoreAction)
+        self.register('action', 'store_const', _StoreConstAction)
+        self.register('action', 'store_true', _StoreTrueAction)
+        self.register('action', 'store_false', _StoreFalseAction)
+        self.register('action', 'append', _AppendAction)
+        self.register('action', 'append_const', _AppendConstAction)
+        self.register('action', 'count', _CountAction)
+        self.register('action', 'help', _HelpAction)
+        self.register('action', 'version', _VersionAction)
+        self.register('action', 'parsers', _SubParsersAction)
+
+        # raise an exception if the conflict handler is invalid
+        self._get_handler()
+
+        # action storage
+        self._actions = []
+        self._option_string_actions = {}
+
+        # groups
+        self._action_groups = []
+        self._mutually_exclusive_groups = []
+
+        # defaults storage
+        self._defaults = {}
+
+        # determines whether an "option" looks like a negative number
+        self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')
+
+        # whether or not there are any optionals that look like negative
+        # numbers -- uses a list so it can be shared and edited
+        self._has_negative_number_optionals = []
+
+    # ====================
+    # Registration methods
+    # ====================
+    def register(self, registry_name, value, object):
+        registry = self._registries.setdefault(registry_name, {})
+        registry[value] = object
+
+    def _registry_get(self, registry_name, value, default=None):
+        return self._registries[registry_name].get(value, default)
+
+    # ==================================
+    # Namespace default accessor methods
+    # ==================================
+    def set_defaults(self, **kwargs):
+        self._defaults.update(kwargs)
+
+        # if these defaults match any existing arguments, replace
+        # the previous default on the object with the new one
+        for action in self._actions:
+            if action.dest in kwargs:
+                action.default = kwargs[action.dest]
+
+    def get_default(self, dest):
+        for action in self._actions:
+            if action.dest == dest and action.default is not None:
+                return action.default
+        return self._defaults.get(dest, None)
+
+
+    # =======================
+    # Adding argument actions
+    # =======================
+    def add_argument(self, *args, **kwargs):
+        """
+        add_argument(dest, ..., name=value, ...)
+        add_argument(option_string, option_string, ..., name=value, ...)
+        """
+
+        # if no positional args are supplied or only one is supplied and
+        # it doesn't look like an option string, parse a positional
+        # argument
+        chars = self.prefix_chars
+        if not args or len(args) == 1 and args[0][0] not in chars:
+            if args and 'dest' in kwargs:
+                raise ValueError('dest supplied twice for positional argument')
+            kwargs = self._get_positional_kwargs(*args, **kwargs)
+
+        # otherwise, we're adding an optional argument
+        else:
+            kwargs = self._get_optional_kwargs(*args, **kwargs)
+
+        # if no default was supplied, use the parser-level default
+        if 'default' not in kwargs:
+            dest = kwargs['dest']
+            if dest in self._defaults:
+                kwargs['default'] = self._defaults[dest]
+            elif self.argument_default is not None:
+                kwargs['default'] = self.argument_default
+
+        # create the action object, and add it to the parser
+        action_class = self._pop_action_class(kwargs)
+        if not callable(action_class):
+            raise ValueError('unknown action "%s"' % (action_class,))
+        action = action_class(**kwargs)
+
+        # raise an error if the action type is not callable
+        type_func = self._registry_get('type', action.type, action.type)
+        if not callable(type_func):
+            raise ValueError('%r is not callable' % (type_func,))
+
+        # raise an error if the metavar does not match the type
+        if hasattr(self, "_get_formatter"):
+            try:
+                self._get_formatter()._format_args(action, None)
+            except TypeError:
+                raise ValueError("length of metavar tuple does not match nargs")
+
+        return self._add_action(action)
+
+    def add_argument_group(self, *args, **kwargs):
+        group = _ArgumentGroup(self, *args, **kwargs)
+        self._action_groups.append(group)
+        return group
+
+    def add_mutually_exclusive_group(self, **kwargs):
+        group = _MutuallyExclusiveGroup(self, **kwargs)
+        self._mutually_exclusive_groups.append(group)
+        return group
+
+    def _add_action(self, action):
+        # resolve any conflicts
+        self._check_conflict(action)
+
+        # add to actions list
+        self._actions.append(action)
+        action.container = self
+
+        # index the action by any option strings it has
+        for option_string in action.option_strings:
+            self._option_string_actions[option_string] = action
+
+        # set the flag if any option strings look like negative numbers
+        for option_string in action.option_strings:
+            if self._negative_number_matcher.match(option_string):
+                if not self._has_negative_number_optionals:
+                    self._has_negative_number_optionals.append(True)
+
+        # return the created action
+        return action
+
+    def _remove_action(self, action):
+        self._actions.remove(action)
+
+    def _add_container_actions(self, container):
+        # collect groups by titles
+        title_group_map = {}
+        for group in self._action_groups:
+            if group.title in title_group_map:
+                msg = _('cannot merge actions - two groups are named %r')
+                raise ValueError(msg % (group.title))
+            title_group_map[group.title] = group
+
+        # map each action to its group
+        group_map = {}
+        for group in container._action_groups:
+
+            # if a group with the title exists, use that, otherwise
+            # create a new group matching the container's group
+            if group.title not in title_group_map:
+                title_group_map[group.title] = self.add_argument_group(
+                    title=group.title,
+                    description=group.description,
+                    conflict_handler=group.conflict_handler)
+
+            # map the actions to their new group
+            for action in group._group_actions:
+                group_map[action] = title_group_map[group.title]
+
+        # add container's mutually exclusive groups
+        # NOTE: if add_mutually_exclusive_group ever gains title= and
+        # description= then this code will need to be expanded as above
+        for group in container._mutually_exclusive_groups:
+            mutex_group = self.add_mutually_exclusive_group(
+                required=group.required)
+
+            # map the actions to their new mutex group
+            for action in group._group_actions:
+                group_map[action] = mutex_group
+
+        # add all actions to this container or their group
+        for action in container._actions:
+            group_map.get(action, self)._add_action(action)
+
+    def _get_positional_kwargs(self, dest, **kwargs):
+        # make sure required is not specified
+        if 'required' in kwargs:
+            msg = _("'required' is an invalid argument for positionals")
+            raise TypeError(msg)
+
+        # mark positional arguments as required if at least one is
+        # always required
+        if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]:
+            kwargs['required'] = True
+        if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs:
+            kwargs['required'] = True
+
+        # return the keyword arguments with no option strings
+        return dict(kwargs, dest=dest, option_strings=[])
+
+    def _get_optional_kwargs(self, *args, **kwargs):
+        # determine short and long option strings
+        option_strings = []
+        long_option_strings = []
+        for option_string in args:
+            # error on strings that don't start with an appropriate prefix
+            if not option_string[0] in self.prefix_chars:
+                args = {'option': option_string,
+                        'prefix_chars': self.prefix_chars}
+                msg = _('invalid option string %(option)r: '
+                        'must start with a character %(prefix_chars)r')
+                raise ValueError(msg % args)
+
+            # strings starting with two prefix characters are long options
+            option_strings.append(option_string)
+            if option_string[0] in self.prefix_chars:
+                if len(option_string) > 1:
+                    if option_string[1] in self.prefix_chars:
+                        long_option_strings.append(option_string)
+
+        # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
+        dest = kwargs.pop('dest', None)
+        if dest is None:
+            if long_option_strings:
+                dest_option_string = long_option_strings[0]
+            else:
+                dest_option_string = option_strings[0]
+            dest = dest_option_string.lstrip(self.prefix_chars)
+            if not dest:
+                msg = _('dest= is required for options like %r')
+                raise ValueError(msg % option_string)
+            dest = dest.replace('-', '_')
+
+        # return the updated keyword arguments
+        return dict(kwargs, dest=dest, option_strings=option_strings)
+
+    def _pop_action_class(self, kwargs, default=None):
+        action = kwargs.pop('action', default)
+        return self._registry_get('action', action, action)
+
+    def _get_handler(self):
+        # determine function from conflict handler string
+        handler_func_name = '_handle_conflict_%s' % self.conflict_handler
+        try:
+            return getattr(self, handler_func_name)
+        except AttributeError:
+            msg = _('invalid conflict_resolution value: %r')
+            raise ValueError(msg % self.conflict_handler)
+
+    def _check_conflict(self, action):
+
+        # find all options that conflict with this option
+        confl_optionals = []
+        for option_string in action.option_strings:
+            if option_string in self._option_string_actions:
+                confl_optional = self._option_string_actions[option_string]
+                confl_optionals.append((option_string, confl_optional))
+
+        # resolve any conflicts
+        if confl_optionals:
+            conflict_handler = self._get_handler()
+            conflict_handler(action, confl_optionals)
+
+    def _handle_conflict_error(self, action, conflicting_actions):
+        message = ngettext('conflicting option string: %s',
+                           'conflicting option strings: %s',
+                           len(conflicting_actions))
+        conflict_string = ', '.join([option_string
+                                     for option_string, action
+                                     in conflicting_actions])
+        raise ArgumentError(action, message % conflict_string)
+
+    def _handle_conflict_resolve(self, action, conflicting_actions):
+
+        # remove all conflicting options
+        for option_string, action in conflicting_actions:
+
+            # remove the conflicting option
+            action.option_strings.remove(option_string)
+            self._option_string_actions.pop(option_string, None)
+
+            # if the option now has no option string, remove it from the
+            # container holding it
+            if not action.option_strings:
+                action.container._remove_action(action)
+
+
+class _ArgumentGroup(_ActionsContainer):
+
+    def __init__(self, container, title=None, description=None, **kwargs):
+        # add any missing keyword arguments by checking the container
+        update = kwargs.setdefault
+        update('conflict_handler', container.conflict_handler)
+        update('prefix_chars', container.prefix_chars)
+        update('argument_default', container.argument_default)
+        super_init = super(_ArgumentGroup, self).__init__
+        super_init(description=description, **kwargs)
+
+        # group attributes
+        self.title = title
+        self._group_actions = []
+
+        # share most attributes with the container
+        self._registries = container._registries
+        self._actions = container._actions
+        self._option_string_actions = container._option_string_actions
+        self._defaults = container._defaults
+        self._has_negative_number_optionals = \
+            container._has_negative_number_optionals
+        self._mutually_exclusive_groups = container._mutually_exclusive_groups
+
+    def _add_action(self, action):
+        action = super(_ArgumentGroup, self)._add_action(action)
+        self._group_actions.append(action)
+        return action
+
+    def _remove_action(self, action):
+        super(_ArgumentGroup, self)._remove_action(action)
+        self._group_actions.remove(action)
+
+
+class _MutuallyExclusiveGroup(_ArgumentGroup):
+
+    def __init__(self, container, required=False):
+        super(_MutuallyExclusiveGroup, self).__init__(container)
+        self.required = required
+        self._container = container
+
+    def _add_action(self, action):
+        if action.required:
+            msg = _('mutually exclusive arguments must be optional')
+            raise ValueError(msg)
+        action = self._container._add_action(action)
+        self._group_actions.append(action)
+        return action
+
+    def _remove_action(self, action):
+        self._container._remove_action(action)
+        self._group_actions.remove(action)
+
+
+class ArgumentParser(_AttributeHolder, _ActionsContainer):
+    """Object for parsing command line strings into Python objects.
+
+    Keyword Arguments:
+        - prog -- The name of the program (default: sys.argv[0])
+        - usage -- A usage message (default: auto-generated from arguments)
+        - description -- A description of what the program does
+        - epilog -- Text following the argument descriptions
+        - parents -- Parsers whose arguments should be copied into this one
+        - formatter_class -- HelpFormatter class for printing help messages
+        - prefix_chars -- Characters that prefix optional arguments
+        - fromfile_prefix_chars -- Characters that prefix files containing
+            additional arguments
+        - argument_default -- The default value for all arguments
+        - conflict_handler -- String indicating how to handle conflicts
+        - add_help -- Add a -h/-help option
+    """
+
+    def __init__(self,
+                 prog=None,
+                 usage=None,
+                 description=None,
+                 epilog=None,
+                 version=None,
+                 parents=[],
+                 formatter_class=HelpFormatter,
+                 prefix_chars='-',
+                 fromfile_prefix_chars=None,
+                 argument_default=None,
+                 conflict_handler='error',
+                 add_help=True):
+
+        if version is not None:
+            import warnings
+            warnings.warn(
+                """The "version" argument to ArgumentParser is deprecated. """
+                """Please use """
+                """"add_argument(..., action='version', version="N", ...)" """
+                """instead""", DeprecationWarning)
+
+        superinit = super(ArgumentParser, self).__init__
+        superinit(description=description,
+                  prefix_chars=prefix_chars,
+                  argument_default=argument_default,
+                  conflict_handler=conflict_handler)
+
+        # default setting for prog
+        if prog is None:
+            prog = _os.path.basename(_sys.argv[0])
+
+        self.prog = prog
+        self.usage = usage
+        self.epilog = epilog
+        self.version = version
+        self.formatter_class = formatter_class
+        self.fromfile_prefix_chars = fromfile_prefix_chars
+        self.add_help = add_help
+
+        add_group = self.add_argument_group
+        self._positionals = add_group(_('positional arguments'))
+        self._optionals = add_group(_('optional arguments'))
+        self._subparsers = None
+
+        # register types
+        def identity(string):
+            return string
+        self.register('type', None, identity)
+
+        # add help and version arguments if necessary
+        # (using explicit default to override global argument_default)
+        default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
+        if self.add_help:
+            self.add_argument(
+                default_prefix+'h', default_prefix*2+'help',
+                action='help', default=SUPPRESS,
+                help=_('show this help message and exit'))
+        if self.version:
+            self.add_argument(
+                default_prefix+'v', default_prefix*2+'version',
+                action='version', default=SUPPRESS,
+                version=self.version,
+                help=_("show program's version number and exit"))
+
+        # add parent arguments and defaults
+        for parent in parents:
+            self._add_container_actions(parent)
+            try:
+                defaults = parent._defaults
+            except AttributeError:
+                pass
+            else:
+                self._defaults.update(defaults)
+
+    # =======================
+    # Pretty __repr__ methods
+    # =======================
+    def _get_kwargs(self):
+        names = [
+            'prog',
+            'usage',
+            'description',
+            'version',
+            'formatter_class',
+            'conflict_handler',
+            'add_help',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    # ==================================
+    # Optional/Positional adding methods
+    # ==================================
+    def add_subparsers(self, **kwargs):
+        if self._subparsers is not None:
+            self.error(_('cannot have multiple subparser arguments'))
+
+        # add the parser class to the arguments if it's not present
+        kwargs.setdefault('parser_class', type(self))
+
+        if 'title' in kwargs or 'description' in kwargs:
+            title = _(kwargs.pop('title', 'subcommands'))
+            description = _(kwargs.pop('description', None))
+            self._subparsers = self.add_argument_group(title, description)
+        else:
+            self._subparsers = self._positionals
+
+        # prog defaults to the usage message of this parser, skipping
+        # optional arguments and with no "usage:" prefix
+        if kwargs.get('prog') is None:
+            formatter = self._get_formatter()
+            positionals = self._get_positional_actions()
+            groups = self._mutually_exclusive_groups
+            formatter.add_usage(self.usage, positionals, groups, '')
+            kwargs['prog'] = formatter.format_help().strip()
+
+        # create the parsers action and add it to the positionals list
+        parsers_class = self._pop_action_class(kwargs, 'parsers')
+        action = parsers_class(option_strings=[], **kwargs)
+        self._subparsers._add_action(action)
+
+        # return the created parsers action
+        return action
+
+    def _add_action(self, action):
+        if action.option_strings:
+            self._optionals._add_action(action)
+        else:
+            self._positionals._add_action(action)
+        return action
+
+    def _get_optional_actions(self):
+        return [action
+                for action in self._actions
+                if action.option_strings]
+
+    def _get_positional_actions(self):
+        return [action
+                for action in self._actions
+                if not action.option_strings]
+
+    # =====================================
+    # Command line argument parsing methods
+    # =====================================
+    def parse_args(self, args=None, namespace=None):
+        args, argv = self.parse_known_args(args, namespace)
+        if argv:
+            msg = _('unrecognized arguments: %s')
+            self.error(msg % ' '.join(argv))
+        return args
+
+    def parse_known_args(self, args=None, namespace=None):
+        # args default to the system args
+        if args is None:
+            args = _sys.argv[1:]
+
+        # default Namespace built from parser defaults
+        if namespace is None:
+            namespace = Namespace()
+
+        # add any action defaults that aren't present
+        for action in self._actions:
+            if action.dest is not SUPPRESS:
+                if not hasattr(namespace, action.dest):
+                    if action.default is not SUPPRESS:
+                        default = action.default
+                        if isinstance(action.default, str):
+                            default = self._get_value(action, default)
+                        setattr(namespace, action.dest, default)
+
+        # add any parser defaults that aren't present
+        for dest in self._defaults:
+            if not hasattr(namespace, dest):
+                setattr(namespace, dest, self._defaults[dest])
+
+        # parse the arguments and exit if there are any errors
+        try:
+            namespace, args = self._parse_known_args(args, namespace)
+            if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
+                args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
+                delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
+            return namespace, args
+        except ArgumentError:
+            err = _sys.exc_info()[1]
+            self.error(str(err))
+
+    def _parse_known_args(self, arg_strings, namespace):
+        # replace arg strings that are file references
+        if self.fromfile_prefix_chars is not None:
+            arg_strings = self._read_args_from_files(arg_strings)
+
+        # map all mutually exclusive arguments to the other arguments
+        # they can't occur with
+        action_conflicts = {}
+        for mutex_group in self._mutually_exclusive_groups:
+            group_actions = mutex_group._group_actions
+            for i, mutex_action in enumerate(mutex_group._group_actions):
+                conflicts = action_conflicts.setdefault(mutex_action, [])
+                conflicts.extend(group_actions[:i])
+                conflicts.extend(group_actions[i + 1:])
+
+        # find all option indices, and determine the arg_string_pattern
+        # which has an 'O' if there is an option at an index,
+        # an 'A' if there is an argument, or a '-' if there is a '--'
+        option_string_indices = {}
+        arg_string_pattern_parts = []
+        arg_strings_iter = iter(arg_strings)
+        for i, arg_string in enumerate(arg_strings_iter):
+
+            # all args after -- are non-options
+            if arg_string == '--':
+                arg_string_pattern_parts.append('-')
+                for arg_string in arg_strings_iter:
+                    arg_string_pattern_parts.append('A')
+
+            # otherwise, add the arg to the arg strings
+            # and note the index if it was an option
+            else:
+                option_tuple = self._parse_optional(arg_string)
+                if option_tuple is None:
+                    pattern = 'A'
+                else:
+                    option_string_indices[i] = option_tuple
+                    pattern = 'O'
+                arg_string_pattern_parts.append(pattern)
+
+        # join the pieces together to form the pattern
+        arg_strings_pattern = ''.join(arg_string_pattern_parts)
+
+        # converts arg strings to the appropriate and then takes the action
+        seen_actions = set()
+        seen_non_default_actions = set()
+
+        def take_action(action, argument_strings, option_string=None):
+            seen_actions.add(action)
+            argument_values = self._get_values(action, argument_strings)
+
+            # error if this argument is not allowed with other previously
+            # seen arguments, assuming that actions that use the default
+            # value don't really count as "present"
+            if argument_values is not action.default:
+                seen_non_default_actions.add(action)
+                for conflict_action in action_conflicts.get(action, []):
+                    if conflict_action in seen_non_default_actions:
+                        msg = _('not allowed with argument %s')
+                        action_name = _get_action_name(conflict_action)
+                        raise ArgumentError(action, msg % action_name)
+
+            # take the action if we didn't receive a SUPPRESS value
+            # (e.g. from a default)
+            if argument_values is not SUPPRESS:
+                action(self, namespace, argument_values, option_string)
+
+        # function to convert arg_strings into an optional action
+        def consume_optional(start_index):
+
+            # get the optional identified at this index
+            option_tuple = option_string_indices[start_index]
+            action, option_string, explicit_arg = option_tuple
+
+            # identify additional optionals in the same arg string
+            # (e.g. -xyz is the same as -x -y -z if no args are required)
+            match_argument = self._match_argument
+            action_tuples = []
+            while True:
+
+                # if we found no optional action, skip it
+                if action is None:
+                    extras.append(arg_strings[start_index])
+                    return start_index + 1
+
+                # if there is an explicit argument, try to match the
+                # optional's string arguments to only this
+                if explicit_arg is not None:
+                    arg_count = match_argument(action, 'A')
+
+                    # if the action is a single-dash option and takes no
+                    # arguments, try to parse more single-dash options out
+                    # of the tail of the option string
+                    chars = self.prefix_chars
+                    if arg_count == 0 and option_string[1] not in chars:
+                        action_tuples.append((action, [], option_string))
+                        char = option_string[0]
+                        option_string = char + explicit_arg[0]
+                        new_explicit_arg = explicit_arg[1:] or None
+                        optionals_map = self._option_string_actions
+                        if option_string in optionals_map:
+                            action = optionals_map[option_string]
+                            explicit_arg = new_explicit_arg
+                        else:
+                            msg = _('ignored explicit argument %r')
+                            raise ArgumentError(action, msg % explicit_arg)
+
+                    # if the action expect exactly one argument, we've
+                    # successfully matched the option; exit the loop
+                    elif arg_count == 1:
+                        stop = start_index + 1
+                        args = [explicit_arg]
+                        action_tuples.append((action, args, option_string))
+                        break
+
+                    # error if a double-dash option did not use the
+                    # explicit argument
+                    else:
+                        msg = _('ignored explicit argument %r')
+                        raise ArgumentError(action, msg % explicit_arg)
+
+                # if there is no explicit argument, try to match the
+                # optional's string arguments with the following strings
+                # if successful, exit the loop
+                else:
+                    start = start_index + 1
+                    selected_patterns = arg_strings_pattern[start:]
+                    arg_count = match_argument(action, selected_patterns)
+                    stop = start + arg_count
+                    args = arg_strings[start:stop]
+                    action_tuples.append((action, args, option_string))
+                    break
+
+            # add the Optional to the list and return the index at which
+            # the Optional's string args stopped
+            assert action_tuples
+            for action, args, option_string in action_tuples:
+                take_action(action, args, option_string)
+            return stop
+
+        # the list of Positionals left to be parsed; this is modified
+        # by consume_positionals()
+        positionals = self._get_positional_actions()
+
+        # function to convert arg_strings into positional actions
+        def consume_positionals(start_index):
+            # match as many Positionals as possible
+            match_partial = self._match_arguments_partial
+            selected_pattern = arg_strings_pattern[start_index:]
+            arg_counts = match_partial(positionals, selected_pattern)
+
+            # slice off the appropriate arg strings for each Positional
+            # and add the Positional and its args to the list
+            for action, arg_count in zip(positionals, arg_counts):
+                args = arg_strings[start_index: start_index + arg_count]
+                start_index += arg_count
+                take_action(action, args)
+
+            # slice off the Positionals that we just parsed and return the
+            # index at which the Positionals' string args stopped
+            positionals[:] = positionals[len(arg_counts):]
+            return start_index
+
+        # consume Positionals and Optionals alternately, until we have
+        # passed the last option string
+        extras = []
+        start_index = 0
+        if option_string_indices:
+            max_option_string_index = max(option_string_indices)
+        else:
+            max_option_string_index = -1
+        while start_index <= max_option_string_index:
+
+            # consume any Positionals preceding the next option
+            next_option_string_index = min([
+                index
+                for index in option_string_indices
+                if index >= start_index])
+            if start_index != next_option_string_index:
+                positionals_end_index = consume_positionals(start_index)
+
+                # only try to parse the next optional if we didn't consume
+                # the option string during the positionals parsing
+                if positionals_end_index > start_index:
+                    start_index = positionals_end_index
+                    continue
+                else:
+                    start_index = positionals_end_index
+
+            # if we consumed all the positionals we could and we're not
+            # at the index of an option string, there were extra arguments
+            if start_index not in option_string_indices:
+                strings = arg_strings[start_index:next_option_string_index]
+                extras.extend(strings)
+                start_index = next_option_string_index
+
+            # consume the next optional and any arguments for it
+            start_index = consume_optional(start_index)
+
+        # consume any positionals following the last Optional
+        stop_index = consume_positionals(start_index)
+
+        # if we didn't consume all the argument strings, there were extras
+        extras.extend(arg_strings[stop_index:])
+
+        # if we didn't use all the Positional objects, there were too few
+        # arg strings supplied.
+        if positionals:
+            self.error(_('too few arguments'))
+
+        # make sure all required actions were present
+        for action in self._actions:
+            if action.required:
+                if action not in seen_actions:
+                    name = _get_action_name(action)
+                    self.error(_('argument %s is required') % name)
+
+        # make sure all required groups had one option present
+        for group in self._mutually_exclusive_groups:
+            if group.required:
+                for action in group._group_actions:
+                    if action in seen_non_default_actions:
+                        break
+
+                # if no actions were used, report the error
+                else:
+                    names = [_get_action_name(action)
+                             for action in group._group_actions
+                             if action.help is not SUPPRESS]
+                    msg = _('one of the arguments %s is required')
+                    self.error(msg % ' '.join(names))
+
+        # return the updated namespace and the extra arguments
+        return namespace, extras
+
+    def _read_args_from_files(self, arg_strings):
+        # expand arguments referencing files
+        new_arg_strings = []
+        for arg_string in arg_strings:
+
+            # for regular arguments, just add them back into the list
+            if arg_string[0] not in self.fromfile_prefix_chars:
+                new_arg_strings.append(arg_string)
+
+            # replace arguments referencing files with the file content
+            else:
+                try:
+                    args_file = open(arg_string[1:])
+                    try:
+                        arg_strings = []
+                        for arg_line in args_file.read().splitlines():
+                            for arg in self.convert_arg_line_to_args(arg_line):
+                                arg_strings.append(arg)
+                        arg_strings = self._read_args_from_files(arg_strings)
+                        new_arg_strings.extend(arg_strings)
+                    finally:
+                        args_file.close()
+                except IOError:
+                    err = _sys.exc_info()[1]
+                    self.error(str(err))
+
+        # return the modified argument list
+        return new_arg_strings
+
+    def convert_arg_line_to_args(self, arg_line):
+        return [arg_line]
+
+    def _match_argument(self, action, arg_strings_pattern):
+        # match the pattern for this action to the arg strings
+        nargs_pattern = self._get_nargs_pattern(action)
+        match = _re.match(nargs_pattern, arg_strings_pattern)
+
+        # raise an exception if we weren't able to find a match
+        if match is None:
+            nargs_errors = {
+                None: _('expected one argument'),
+                OPTIONAL: _('expected at most one argument'),
+                ONE_OR_MORE: _('expected at least one argument'),
+            }
+            default = ngettext('expected %s argument',
+                               'expected %s arguments',
+                               action.nargs) % action.nargs
+            msg = nargs_errors.get(action.nargs, default)
+            raise ArgumentError(action, msg)
+
+        # return the number of arguments matched
+        return len(match.group(1))
+
+    def _match_arguments_partial(self, actions, arg_strings_pattern):
+        # progressively shorten the actions list by slicing off the
+        # final actions until we find a match
+        result = []
+        for i in range(len(actions), 0, -1):
+            actions_slice = actions[:i]
+            pattern = ''.join([self._get_nargs_pattern(action)
+                               for action in actions_slice])
+            match = _re.match(pattern, arg_strings_pattern)
+            if match is not None:
+                result.extend([len(string) for string in match.groups()])
+                break
+
+        # return the list of arg string counts
+        return result
+
+    def _parse_optional(self, arg_string):
+        # if it's an empty string, it was meant to be a positional
+        if not arg_string:
+            return None
+
+        # if it doesn't start with a prefix, it was meant to be positional
+        if not arg_string[0] in self.prefix_chars:
+            return None
+
+        # if the option string is present in the parser, return the action
+        if arg_string in self._option_string_actions:
+            action = self._option_string_actions[arg_string]
+            return action, arg_string, None
+
+        # if it's just a single character, it was meant to be positional
+        if len(arg_string) == 1:
+            return None
+
+        # if the option string before the "=" is present, return the action
+        if '=' in arg_string:
+            option_string, explicit_arg = arg_string.split('=', 1)
+            if option_string in self._option_string_actions:
+                action = self._option_string_actions[option_string]
+                return action, option_string, explicit_arg
+
+        # search through all possible prefixes of the option string
+        # and all actions in the parser for possible interpretations
+        option_tuples = self._get_option_tuples(arg_string)
+
+        # if multiple actions match, the option string was ambiguous
+        if len(option_tuples) > 1:
+            options = ', '.join([option_string
+                for action, option_string, explicit_arg in option_tuples])
+            args = {'option': arg_string, 'matches': options}
+            msg = _('ambiguous option: %(option)s could match %(matches)s')
+            self.error(msg % args)
+
+        # if exactly one action matched, this segmentation is good,
+        # so return the parsed action
+        elif len(option_tuples) == 1:
+            option_tuple, = option_tuples
+            return option_tuple
+
+        # if it was not found as an option, but it looks like a negative
+        # number, it was meant to be positional
+        # unless there are negative-number-like options
+        if self._negative_number_matcher.match(arg_string):
+            if not self._has_negative_number_optionals:
+                return None
+
+        # if it contains a space, it was meant to be a positional
+        if ' ' in arg_string:
+            return None
+
+        # it was meant to be an optional but there is no such option
+        # in this parser (though it might be a valid option in a subparser)
+        return None, arg_string, None
+
+    def _get_option_tuples(self, option_string):
+        result = []
+
+        # option strings starting with two prefix characters are only
+        # split at the '='
+        chars = self.prefix_chars
+        if option_string[0] in chars and option_string[1] in chars:
+            if '=' in option_string:
+                option_prefix, explicit_arg = option_string.split('=', 1)
+            else:
+                option_prefix = option_string
+                explicit_arg = None
+            for option_string in self._option_string_actions:
+                if option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
+
+        # single character options can be concatenated with their arguments
+        # but multiple character options always have to have their argument
+        # separate
+        elif option_string[0] in chars and option_string[1] not in chars:
+            option_prefix = option_string
+            explicit_arg = None
+            short_option_prefix = option_string[:2]
+            short_explicit_arg = option_string[2:]
+
+            for option_string in self._option_string_actions:
+                if option_string == short_option_prefix:
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, short_explicit_arg
+                    result.append(tup)
+                elif option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
+
+        # shouldn't ever get here
+        else:
+            self.error(_('unexpected option string: %s') % option_string)
+
+        # return the collected option tuples
+        return result
+
+    def _get_nargs_pattern(self, action):
+        # in all examples below, we have to allow for '--' args
+        # which are represented as '-' in the pattern
+        nargs = action.nargs
+
+        # the default (None) is assumed to be a single argument
+        if nargs is None:
+            nargs_pattern = '(-*A-*)'
+
+        # allow zero or one arguments
+        elif nargs == OPTIONAL:
+            nargs_pattern = '(-*A?-*)'
+
+        # allow zero or more arguments
+        elif nargs == ZERO_OR_MORE:
+            nargs_pattern = '(-*[A-]*)'
+
+        # allow one or more arguments
+        elif nargs == ONE_OR_MORE:
+            nargs_pattern = '(-*A[A-]*)'
+
+        # allow any number of options or arguments
+        elif nargs == REMAINDER:
+            nargs_pattern = '([-AO]*)'
+
+        # allow one argument followed by any number of options or arguments
+        elif nargs == PARSER:
+            nargs_pattern = '(-*A[-AO]*)'
+
+        # all others should be integers
+        else:
+            nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)
+
+        # if this is an optional action, -- is not allowed
+        if action.option_strings:
+            nargs_pattern = nargs_pattern.replace('-*', '')
+            nargs_pattern = nargs_pattern.replace('-', '')
+
+        # return the pattern
+        return nargs_pattern
+
+    # ========================
+    # Value conversion methods
+    # ========================
+    def _get_values(self, action, arg_strings):
+        # for everything but PARSER args, strip out '--'
+        if action.nargs not in [PARSER, REMAINDER]:
+            arg_strings = [s for s in arg_strings if s != '--']
+
+        # optional argument produces a default when not present
+        if not arg_strings and action.nargs == OPTIONAL:
+            if action.option_strings:
+                value = action.const
+            else:
+                value = action.default
+            if isinstance(value, str):
+                value = self._get_value(action, value)
+                self._check_value(action, value)
+
+        # when nargs='*' on a positional, if there were no command-line
+        # args, use the default if it is anything other than None
+        elif (not arg_strings and action.nargs == ZERO_OR_MORE and
+              not action.option_strings):
+            if action.default is not None:
+                value = action.default
+            else:
+                value = arg_strings
+            self._check_value(action, value)
+
+        # single argument or optional argument produces a single value
+        elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
+            arg_string, = arg_strings
+            value = self._get_value(action, arg_string)
+            self._check_value(action, value)
+
+        # REMAINDER arguments convert all values, checking none
+        elif action.nargs == REMAINDER:
+            value = [self._get_value(action, v) for v in arg_strings]
+
+        # PARSER arguments convert all values, but check only the first
+        elif action.nargs == PARSER:
+            value = [self._get_value(action, v) for v in arg_strings]
+            self._check_value(action, value[0])
+
+        # all other types of nargs produce a list
+        else:
+            value = [self._get_value(action, v) for v in arg_strings]
+            for v in value:
+                self._check_value(action, v)
+
+        # return the converted value
+        return value
+
+    def _get_value(self, action, arg_string):
+        type_func = self._registry_get('type', action.type, action.type)
+        if not callable(type_func):
+            msg = _('%r is not callable')
+            raise ArgumentError(action, msg % type_func)
+
+        # convert the value to the appropriate type
+        try:
+            result = type_func(arg_string)
+
+        # ArgumentTypeErrors indicate errors
+        except ArgumentTypeError:
+            name = getattr(action.type, '__name__', repr(action.type))
+            msg = str(_sys.exc_info()[1])
+            raise ArgumentError(action, msg)
+
+        # TypeErrors or ValueErrors also indicate errors
+        except (TypeError, ValueError):
+            name = getattr(action.type, '__name__', repr(action.type))
+            args = {'type': name, 'value': arg_string}
+            msg = _('invalid %(type)s value: %(value)r')
+            raise ArgumentError(action, msg % args)
+
+        # return the converted value
+        return result
+
+    def _check_value(self, action, value):
+        # converted value must be one of the choices (if specified)
+        if action.choices is not None and value not in action.choices:
+            args = {'value': value,
+                    'choices': ', '.join(map(repr, action.choices))}
+            msg = _('invalid choice: %(value)r (choose from %(choices)s)')
+            raise ArgumentError(action, msg % args)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+    def format_usage(self):
+        formatter = self._get_formatter()
+        formatter.add_usage(self.usage, self._actions,
+                            self._mutually_exclusive_groups)
+        return formatter.format_help()
+
+    def format_help(self):
+        formatter = self._get_formatter()
+
+        # usage
+        formatter.add_usage(self.usage, self._actions,
+                            self._mutually_exclusive_groups)
+
+        # description
+        formatter.add_text(self.description)
+
+        # positionals, optionals and user-defined groups
+        for action_group in self._action_groups:
+            formatter.start_section(action_group.title)
+            formatter.add_text(action_group.description)
+            formatter.add_arguments(action_group._group_actions)
+            formatter.end_section()
+
+        # epilog
+        formatter.add_text(self.epilog)
+
+        # determine help from format above
+        return formatter.format_help()
+
+    def format_version(self):
+        import warnings
+        warnings.warn(
+            'The format_version method is deprecated -- the "version" '
+            'argument to ArgumentParser is no longer supported.',
+            DeprecationWarning)
+        formatter = self._get_formatter()
+        formatter.add_text(self.version)
+        return formatter.format_help()
+
+    def _get_formatter(self):
+        return self.formatter_class(prog=self.prog)
+
+    # =====================
+    # Help-printing methods
+    # =====================
+    def print_usage(self, file=None):
+        if file is None:
+            file = _sys.stdout
+        self._print_message(self.format_usage(), file)
+
+    def print_help(self, file=None):
+        if file is None:
+            file = _sys.stdout
+        self._print_message(self.format_help(), file)
+
+    def print_version(self, file=None):
+        import warnings
+        warnings.warn(
+            'The print_version method is deprecated -- the "version" '
+            'argument to ArgumentParser is no longer supported.',
+            DeprecationWarning)
+        self._print_message(self.format_version(), file)
+
+    def _print_message(self, message, file=None):
+        if message:
+            if file is None:
+                file = _sys.stderr
+            file.write(message)
+
+    # ===============
+    # Exiting methods
+    # ===============
+    def exit(self, status=0, message=None):
+        if message:
+            self._print_message(message, _sys.stderr)
+        _sys.exit(status)
+
+    def error(self, message):
+        """error(message: string)
+
+        Prints a usage message incorporating the message to stderr and
+        exits.
+
+        If you override this in a subclass, it should not return -- it
+        should either exit or raise an exception.
+        """
+        self.print_usage(_sys.stderr)
+        args = {'prog': self.prog, 'message': message}
+        self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
diff --git a/lib-python/3.2/ast.py b/lib-python/3.2/ast.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/ast.py
@@ -0,0 +1,318 @@
+"""
+    ast
+    ~~~
+
+    The `ast` module helps Python applications to process trees of the Python
+    abstract syntax grammar.  The abstract syntax itself might change with
+    each Python release; this module helps to find out programmatically what
+    the current grammar looks like and allows modifications of it.
+
+    An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
+    a flag to the `compile()` builtin function or by using the `parse()`
+    function from this module.  The result will be a tree of objects whose
+    classes all inherit from `ast.AST`.
+
+    A modified abstract syntax tree can be compiled into a Python code object
+    using the built-in `compile()` function.
+
+    Additionally various helper functions are provided that make working with
+    the trees simpler.  The main intention of the helper functions and this
+    module in general is to provide an easy to use interface for libraries
+    that work tightly with the python syntax (template engines for example).
+
+
+    :copyright: Copyright 2008 by Armin Ronacher.
+    :license: Python License.
+"""
+from _ast import *
+from _ast import __version__
+
+
+def parse(source, filename='<unknown>', mode='exec'):
+    """
+    Parse the source into an AST node.
+    Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
+    """
+    return compile(source, filename, mode, PyCF_ONLY_AST)
+
+
+def literal_eval(node_or_string):
+    """
+    Safely evaluate an expression node or a string containing a Python
+    expression.  The string or node provided may only consist of the following
+    Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
+    sets, booleans, and None.
+    """
+    _safe_names = {'None': None, 'True': True, 'False': False}
+    if isinstance(node_or_string, str):
+        node_or_string = parse(node_or_string, mode='eval')
+    if isinstance(node_or_string, Expression):
+        node_or_string = node_or_string.body
+    def _convert(node):
+        if isinstance(node, (Str, Bytes)):
+            return node.s
+        elif isinstance(node, Num):
+            return node.n
+        elif isinstance(node, Tuple):
+            return tuple(map(_convert, node.elts))
+        elif isinstance(node, List):
+            return list(map(_convert, node.elts))
+        elif isinstance(node, Set):
+            return set(map(_convert, node.elts))
+        elif isinstance(node, Dict):
+            return dict((_convert(k), _convert(v)) for k, v
+                        in zip(node.keys, node.values))
+        elif isinstance(node, Name):
+            if node.id in _safe_names:
+                return _safe_names[node.id]
+        elif isinstance(node, UnaryOp) and \
+             isinstance(node.op, (UAdd, USub)) and \
+             isinstance(node.operand, (Num, UnaryOp, BinOp)):
+            operand = _convert(node.operand)
+            if isinstance(node.op, UAdd):
+                return + operand
+            else:
+                return - operand
+        elif isinstance(node, BinOp) and \
+             isinstance(node.op, (Add, Sub)) and \
+             isinstance(node.right, (Num, UnaryOp, BinOp)) and \
+             isinstance(node.left, (Num, UnaryOp, BinOp)):
+            left = _convert(node.left)
+            right = _convert(node.right)
+            if isinstance(node.op, Add):
+                return left + right
+            else:
+                return left - right
+        raise ValueError('malformed node or string: ' + repr(node))
+    return _convert(node_or_string)
+
+
+def dump(node, annotate_fields=True, include_attributes=False):
+    """
+    Return a formatted dump of the tree in *node*.  This is mainly useful for
+    debugging purposes.  The returned string will show the names and the values
+    for fields.  This makes the code impossible to evaluate, so if evaluation is
+    wanted *annotate_fields* must be set to False.  Attributes such as line
+    numbers and column offsets are not dumped by default.  If this is wanted,
+    *include_attributes* can be set to True.
+    """
+    def _format(node):
+        if isinstance(node, AST):
+            fields = [(a, _format(b)) for a, b in iter_fields(node)]
+            rv = '%s(%s' % (node.__class__.__name__, ', '.join(
+                ('%s=%s' % field for field in fields)
+                if annotate_fields else
+                (b for a, b in fields)
+            ))
+            if include_attributes and node._attributes:
+                rv += fields and ', ' or ' '
+                rv += ', '.join('%s=%s' % (a, _format(getattr(node, a)))
+                                for a in node._attributes)
+            return rv + ')'
+        elif isinstance(node, list):
+            return '[%s]' % ', '.join(_format(x) for x in node)
+        return repr(node)
+    if not isinstance(node, AST):
+        raise TypeError('expected AST, got %r' % node.__class__.__name__)
+    return _format(node)
+
+
+def copy_location(new_node, old_node):
+    """
+    Copy source location (`lineno` and `col_offset` attributes) from
+    *old_node* to *new_node* if possible, and return *new_node*.
+    """
+    for attr in 'lineno', 'col_offset':
+        if attr in old_node._attributes and attr in new_node._attributes \
+           and hasattr(old_node, attr):
+            setattr(new_node, attr, getattr(old_node, attr))
+    return new_node
+
+
+def fix_missing_locations(node):
+    """
+    When you compile a node tree with compile(), the compiler expects lineno and
+    col_offset attributes for every node that supports them.  This is rather
+    tedious to fill in for generated nodes, so this helper adds these attributes
+    recursively where not already set, by setting them to the values of the
+    parent node.  It works recursively starting at *node*.
+    """
+    def _fix(node, lineno, col_offset):
+        if 'lineno' in node._attributes:
+            if not hasattr(node, 'lineno'):
+                node.lineno = lineno
+            else:
+                lineno = node.lineno
+        if 'col_offset' in node._attributes:
+            if not hasattr(node, 'col_offset'):
+                node.col_offset = col_offset
+            else:
+                col_offset = node.col_offset
+        for child in iter_child_nodes(node):
+            _fix(child, lineno, col_offset)
+    _fix(node, 1, 0)
+    return node
+
+
+def increment_lineno(node, n=1):
+    """
+    Increment the line number of each node in the tree starting at *node* by *n*.
+    This is useful to "move code" to a different location in a file.
+    """
+    for child in walk(node):
+        if 'lineno' in child._attributes:
+            child.lineno = getattr(child, 'lineno', 0) + n
+    return node
+
+
+def iter_fields(node):
+    """
+    Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields``
+    that is present on *node*.
+    """
+    for field in node._fields:
+        try:
+            yield field, getattr(node, field)
+        except AttributeError:
+            pass
+
+
+def iter_child_nodes(node):
+    """
+    Yield all direct child nodes of *node*, that is, all fields that are nodes
+    and all items of fields that are lists of nodes.
+    """
+    for name, field in iter_fields(node):
+        if isinstance(field, AST):
+            yield field
+        elif isinstance(field, list):
+            for item in field:
+                if isinstance(item, AST):
+                    yield item
+
+
+def get_docstring(node, clean=True):
+    """
+    Return the docstring for the given node or None if no docstring can
+    be found.  If the node provided does not have docstrings a TypeError
+    will be raised.
+    """
+    if not isinstance(node, (FunctionDef, ClassDef, Module)):
+        raise TypeError("%r can't have docstrings" % node.__class__.__name__)
+    if node.body and isinstance(node.body[0], Expr) and \
+       isinstance(node.body[0].value, Str):
+        if clean:
+            import inspect
+            return inspect.cleandoc(node.body[0].value.s)
+        return node.body[0].value.s
+
+
+def walk(node):
+    """
+    Recursively yield all descendant nodes in the tree starting at *node*
+    (including *node* itself), in no specified order.  This is useful if you
+    only want to modify nodes in place and don't care about the context.
+    """
+    from collections import deque
+    todo = deque([node])
+    while todo:
+        node = todo.popleft()
+        todo.extend(iter_child_nodes(node))
+        yield node
+
+
+class NodeVisitor(object):
+    """
+    A node visitor base class that walks the abstract syntax tree and calls a
+    visitor function for every node found.  This function may return a value
+    which is forwarded by the `visit` method.
+
+    This class is meant to be subclassed, with the subclass adding visitor
+    methods.
+
+    Per default the visitor functions for the nodes are ``'visit_'`` +
+    class name of the node.  So a `TryFinally` node visit function would
+    be `visit_TryFinally`.  This behavior can be changed by overriding
+    the `visit` method.  If no visitor function exists for a node
+    (return value `None`) the `generic_visit` visitor is used instead.
+
+    Don't use the `NodeVisitor` if you want to apply changes to nodes during
+    traversing.  For this a special visitor exists (`NodeTransformer`) that
+    allows modifications.
+    """
+
+    def visit(self, node):
+        """Visit a node."""
+        method = 'visit_' + node.__class__.__name__
+        visitor = getattr(self, method, self.generic_visit)
+        return visitor(node)
+
+    def generic_visit(self, node):
+        """Called if no explicit visitor function exists for a node."""
+        for field, value in iter_fields(node):
+            if isinstance(value, list):
+                for item in value:
+                    if isinstance(item, AST):
+                        self.visit(item)
+            elif isinstance(value, AST):
+                self.visit(value)
+
+
+class NodeTransformer(NodeVisitor):
+    """
+    A :class:`NodeVisitor` subclass that walks the abstract syntax tree and
+    allows modification of nodes.
+
+    The `NodeTransformer` will walk the AST and use the return value of the
+    visitor methods to replace or remove the old node.  If the return value of
+    the visitor method is ``None``, the node will be removed from its location,
+    otherwise it is replaced with the return value.  The return value may be the
+    original node in which case no replacement takes place.
+
+    Here is an example transformer that rewrites all occurrences of name lookups
+    (``foo``) to ``data['foo']``::
+
+       class RewriteName(NodeTransformer):
+
+           def visit_Name(self, node):
+               return copy_location(Subscript(
+                   value=Name(id='data', ctx=Load()),
+                   slice=Index(value=Str(s=node.id)),
+                   ctx=node.ctx
+               ), node)
+
+    Keep in mind that if the node you're operating on has child nodes you must
+    either transform the child nodes yourself or call the :meth:`generic_visit`
+    method for the node first.
+
+    For nodes that were part of a collection of statements (that applies to all
+    statement nodes), the visitor may also return a list of nodes rather than
+    just a single node.
+
+    Usually you use the transformer like this::
+
+       node = YourTransformer().visit(node)
+    """
+
+    def generic_visit(self, node):
+        for field, old_value in iter_fields(node):
+            old_value = getattr(node, field, None)
+            if isinstance(old_value, list):
+                new_values = []
+                for value in old_value:
+                    if isinstance(value, AST):
+                        value = self.visit(value)
+                        if value is None:
+                            continue
+                        elif not isinstance(value, AST):
+                            new_values.extend(value)
+                            continue
+                    new_values.append(value)
+                old_value[:] = new_values
+            elif isinstance(old_value, AST):
+                new_node = self.visit(old_value)
+                if new_node is None:
+                    delattr(node, field)
+                else:
+                    setattr(node, field, new_node)
+        return node
diff --git a/lib-python/3.2/asynchat.py b/lib-python/3.2/asynchat.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/asynchat.py
@@ -0,0 +1,336 @@
+# -*- Mode: Python; tab-width: 4 -*-
+#       Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
+#       Author: Sam Rushing <rushing at nightmare.com>
+
+# ======================================================================
+# Copyright 1996 by Sam Rushing
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+r"""A class supporting chat-style (command/response) protocols.
+
+This class adds support for 'chat' style protocols - where one side
+sends a 'command', and the other sends a response (examples would be
+the common internet protocols - smtp, nntp, ftp, etc..).
+
+The handle_read() method looks at the input stream for the current
+'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
+for multi-line output), calling self.found_terminator() on its
+receipt.
+
+for example:
+Say you build an async nntp client using this class.  At the start
+of the connection, you'll have self.terminator set to '\r\n', in
+order to process the single-line greeting.  Just before issuing a
+'LIST' command you'll set it to '\r\n.\r\n'.  The output of the LIST
+command will be accumulated (using your own 'collect_incoming_data'
+method) up to the terminator, and then control will be returned to
+you - by calling your self.found_terminator() method.
+"""
+import socket
+import asyncore
+from collections import deque
+
+def buffer(obj, start=None, stop=None):
+    # if memoryview objects gain slicing semantics,
+    # this function will change for the better
+    # memoryview used for the TypeError
+    memoryview(obj)
+    if start == None:
+        start = 0
+    if stop == None:
+        stop = len(obj)
+    x = obj[start:stop]
+    ## print("buffer type is: %s"%(type(x),))
+    return x
+
+class async_chat (asyncore.dispatcher):
+    """This is an abstract class.  You must derive from this class, and add
+    the two methods collect_incoming_data() and found_terminator()"""
+
+    # these are overridable defaults
+
+    ac_in_buffer_size       = 4096
+    ac_out_buffer_size      = 4096
+
+    # we don't want to enable the use of encoding by default, because that is a
+    # sign of an application bug that we don't want to pass silently
+
+    use_encoding            = 0
+    encoding                = 'latin1'
+
+    def __init__ (self, sock=None, map=None):
+        # for string terminator matching
+        self.ac_in_buffer = b''
+
+        # we use a list here rather than cStringIO for a few reasons...
+        # del lst[:] is faster than sio.truncate(0)
+        # lst = [] is faster than sio.truncate(0)
+        # cStringIO will be gaining unicode support in py3k, which
+        # will negatively affect the performance of bytes compared to
+        # a ''.join() equivalent
+        self.incoming = []
+
+        # we toss the use of the "simple producer" and replace it with
+        # a pure deque, which the original fifo was a wrapping of
+        self.producer_fifo = deque()
+        asyncore.dispatcher.__init__ (self, sock, map)
+
+    def collect_incoming_data(self, data):
+        raise NotImplementedError("must be implemented in subclass")
+
+    def _collect_incoming_data(self, data):
+        self.incoming.append(data)
+
+    def _get_data(self):
+        d = b''.join(self.incoming)
+        del self.incoming[:]
+        return d
+
+    def found_terminator(self):
+        raise NotImplementedError("must be implemented in subclass")
+
+    def set_terminator (self, term):
+        "Set the input delimiter.  Can be a fixed string of any length, an integer, or None"
+        if isinstance(term, str) and self.use_encoding:
+            term = bytes(term, self.encoding)
+        self.terminator = term
+
+    def get_terminator (self):
+        return self.terminator
+
+    # grab some more data from the socket,
+    # throw it to the collector method,
+    # check for the terminator,
+    # if found, transition to the next state.
+
+    def handle_read (self):
+
+        try:
+            data = self.recv (self.ac_in_buffer_size)
+        except socket.error as why:
+            self.handle_error()
+            return
+
+        if isinstance(data, str) and self.use_encoding:
+            data = bytes(str, self.encoding)
+        self.ac_in_buffer = self.ac_in_buffer + data
+
+        # Continue to search for self.terminator in self.ac_in_buffer,
+        # while calling self.collect_incoming_data.  The while loop
+        # is necessary because we might read several data+terminator
+        # combos with a single recv(4096).
+
+        while self.ac_in_buffer:
+            lb = len(self.ac_in_buffer)
+            terminator = self.get_terminator()
+            if not terminator:
+                # no terminator, collect it all
+                self.collect_incoming_data (self.ac_in_buffer)
+                self.ac_in_buffer = b''
+            elif isinstance(terminator, int):
+                # numeric terminator
+                n = terminator
+                if lb < n:
+                    self.collect_incoming_data (self.ac_in_buffer)
+                    self.ac_in_buffer = b''
+                    self.terminator = self.terminator - lb
+                else:
+                    self.collect_incoming_data (self.ac_in_buffer[:n])
+                    self.ac_in_buffer = self.ac_in_buffer[n:]
+                    self.terminator = 0
+                    self.found_terminator()
+            else:
+                # 3 cases:
+                # 1) end of buffer matches terminator exactly:
+                #    collect data, transition
+                # 2) end of buffer matches some prefix:
+                #    collect data to the prefix
+                # 3) end of buffer does not match any prefix:
+                #    collect data
+                terminator_len = len(terminator)
+                index = self.ac_in_buffer.find(terminator)
+                if index != -1:
+                    # we found the terminator
+                    if index > 0:
+                        # don't bother reporting the empty string (source of subtle bugs)
+                        self.collect_incoming_data (self.ac_in_buffer[:index])
+                    self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
+                    # This does the Right Thing if the terminator is changed here.
+                    self.found_terminator()
+                else:
+                    # check for a prefix of the terminator
+                    index = find_prefix_at_end (self.ac_in_buffer, terminator)
+                    if index:
+                        if index != lb:
+                            # we found a prefix, collect up to the prefix
+                            self.collect_incoming_data (self.ac_in_buffer[:-index])
+                            self.ac_in_buffer = self.ac_in_buffer[-index:]
+                        break
+                    else:
+                        # no prefix, collect it all
+                        self.collect_incoming_data (self.ac_in_buffer)
+                        self.ac_in_buffer = b''
+
+    def handle_write (self):
+        self.initiate_send()
+
+    def handle_close (self):
+        self.close()
+
+    def push (self, data):
+        sabs = self.ac_out_buffer_size
+        if len(data) > sabs:
+            for i in range(0, len(data), sabs):
+                self.producer_fifo.append(data[i:i+sabs])
+        else:
+            self.producer_fifo.append(data)
+        self.initiate_send()
+
+    def push_with_producer (self, producer):
+        self.producer_fifo.append(producer)
+        self.initiate_send()
+
+    def readable (self):
+        "predicate for inclusion in the readable for select()"
+        # cannot use the old predicate, it violates the claim of the
+        # set_terminator method.
+
+        # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
+        return 1
+
+    def writable (self):
+        "predicate for inclusion in the writable for select()"
+        return self.producer_fifo or (not self.connected)
+
+    def close_when_done (self):
+        "automatically close this channel once the outgoing queue is empty"
+        self.producer_fifo.append(None)
+
+    def initiate_send(self):
+        while self.producer_fifo and self.connected:
+            first = self.producer_fifo[0]
+            # handle empty string/buffer or None entry
+            if not first:
+                del self.producer_fifo[0]
+                if first is None:
+                    ## print("first is None")
+                    self.handle_close()
+                    return
+                ## print("first is not None")
+
+            # handle classic producer behavior
+            obs = self.ac_out_buffer_size
+            try:
+                data = buffer(first, 0, obs)
+            except TypeError:
+                data = first.more()
+                if data:
+                    self.producer_fifo.appendleft(data)
+                else:
+                    del self.producer_fifo[0]
+                continue
+
+            if isinstance(data, str) and self.use_encoding:
+                data = bytes(data, self.encoding)
+
+            # send the data
+            try:
+                num_sent = self.send(data)
+            except socket.error:
+                self.handle_error()
+                return
+
+            if num_sent:
+                if num_sent < len(data) or obs < len(first):
+                    self.producer_fifo[0] = first[num_sent:]
+                else:
+                    del self.producer_fifo[0]
+            # we tried to send some actual data
+            return
+
+    def discard_buffers (self):
+        # Emergencies only!
+        self.ac_in_buffer = b''
+        del self.incoming[:]
+        self.producer_fifo.clear()
+
+class simple_producer:
+
+    def __init__ (self, data, buffer_size=512):
+        self.data = data
+        self.buffer_size = buffer_size
+
+    def more (self):
+        if len (self.data) > self.buffer_size:
+            result = self.data[:self.buffer_size]
+            self.data = self.data[self.buffer_size:]
+            return result
+        else:
+            result = self.data
+            self.data = b''
+            return result
+
+class fifo:
+    def __init__ (self, list=None):
+        if not list:
+            self.list = deque()
+        else:
+            self.list = deque(list)
+
+    def __len__ (self):
+        return len(self.list)
+
+    def is_empty (self):
+        return not self.list
+
+    def first (self):
+        return self.list[0]
+
+    def push (self, data):
+        self.list.append(data)
+
+    def pop (self):
+        if self.list:
+            return (1, self.list.popleft())
+        else:
+            return (0, None)
+
+# Given 'haystack', see if any prefix of 'needle' is at its end.  This
+# assumes an exact match has already been checked.  Return the number of
+# characters matched.
+# for example:
+# f_p_a_e ("qwerty\r", "\r\n") => 1
+# f_p_a_e ("qwertydkjf", "\r\n") => 0
+# f_p_a_e ("qwerty\r\n", "\r\n") => <undefined>
+
+# this could maybe be made faster with a computed regex?
+# [answer: no; circa Python-2.0, Jan 2001]
+# new python:   28961/s
+# old python:   18307/s
+# re:        12820/s
+# regex:     14035/s
+
+def find_prefix_at_end (haystack, needle):
+    l = len(needle) - 1
+    while l and not haystack.endswith(needle[:l]):
+        l -= 1
+    return l
diff --git a/lib-python/3.2/asyncore.py b/lib-python/3.2/asyncore.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/asyncore.py
@@ -0,0 +1,663 @@
+# -*- Mode: Python -*-
+#   Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
+#   Author: Sam Rushing <rushing at nightmare.com>
+
+# ======================================================================
+# Copyright 1996 by Sam Rushing
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+"""Basic infrastructure for asynchronous socket service clients and servers.
+
+There are only two ways to have a program on a single processor do "more
+than one thing at a time".  Multi-threaded programming is the simplest and
+most popular way to do it, but there is another very different technique,
+that lets you have nearly all the advantages of multi-threading, without
+actually using multiple threads. it's really only practical if your program
+is largely I/O bound. If your program is CPU bound, then pre-emptive
+scheduled threads are probably what you really need. Network servers are
+rarely CPU-bound, however.
+
+If your operating system supports the select() system call in its I/O
+library (and nearly all do), then you can use it to juggle multiple
+communication channels at once; doing other work while your I/O is taking
+place in the "background."  Although this strategy can seem strange and
+complex, especially at first, it is in many ways easier to understand and
+control than multi-threaded programming. The module documented here solves
+many of the difficult problems for you, making the task of building
+sophisticated high-performance network servers and clients a snap.
+"""
+
+import select
+import socket
+import sys
+import time
+import warnings
+
+import os
+from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
+     ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
+     errorcode
+
+_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
+                           EBADF))
+
+try:
+    socket_map
+except NameError:
+    socket_map = {}
+
+def _strerror(err):
+    try:
+        return os.strerror(err)
+    except (ValueError, OverflowError, NameError):
+        if err in errorcode:
+            return errorcode[err]
+        return "Unknown error %s" %err
+
+class ExitNow(Exception):
+    pass
+
+_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit)
+
+def read(obj):
+    try:
+        obj.handle_read_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def write(obj):
+    try:
+        obj.handle_write_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def _exception(obj):
+    try:
+        obj.handle_expt_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def readwrite(obj, flags):
+    try:
+        if flags & select.POLLIN:
+            obj.handle_read_event()
+        if flags & select.POLLOUT:
+            obj.handle_write_event()
+        if flags & select.POLLPRI:
+            obj.handle_expt_event()
+        if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
+            obj.handle_close()
+    except socket.error as e:
+        if e.args[0] not in _DISCONNECTED:
+            obj.handle_error()
+        else:
+            obj.handle_close()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def poll(timeout=0.0, map=None):
+    if map is None:
+        map = socket_map
+    if map:
+        r = []; w = []; e = []
+        for fd, obj in list(map.items()):
+            is_r = obj.readable()
+            is_w = obj.writable()
+            if is_r:
+                r.append(fd)
+            # accepting sockets should not be writable
+            if is_w and not obj.accepting:
+                w.append(fd)
+            if is_r or is_w:
+                e.append(fd)
+        if [] == r == w == e:
+            time.sleep(timeout)
+            return
+
+        try:
+            r, w, e = select.select(r, w, e, timeout)
+        except select.error as err:
+            if err.args[0] != EINTR:
+                raise
+            else:
+                return
+
+        for fd in r:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            read(obj)
+
+        for fd in w:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            write(obj)
+
+        for fd in e:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            _exception(obj)
+
+def poll2(timeout=0.0, map=None):
+    # Use the poll() support added to the select module in Python 2.0
+    if map is None:
+        map = socket_map
+    if timeout is not None:
+        # timeout is in milliseconds
+        timeout = int(timeout*1000)
+    pollster = select.poll()
+    if map:
+        for fd, obj in list(map.items()):
+            flags = 0
+            if obj.readable():
+                flags |= select.POLLIN | select.POLLPRI
+            # accepting sockets should not be writable
+            if obj.writable() and not obj.accepting:
+                flags |= select.POLLOUT
+            if flags:
+                # Only check for exceptions if object was either readable
+                # or writable.
+                flags |= select.POLLERR | select.POLLHUP | select.POLLNVAL
+                pollster.register(fd, flags)
+        try:
+            r = pollster.poll(timeout)
+        except select.error as err:
+            if err.args[0] != EINTR:
+                raise
+            r = []
+        for fd, flags in r:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            readwrite(obj, flags)
+
+poll3 = poll2                           # Alias for backward compatibility
+
+def loop(timeout=30.0, use_poll=False, map=None, count=None):
+    if map is None:
+        map = socket_map
+
+    if use_poll and hasattr(select, 'poll'):
+        poll_fun = poll2
+    else:
+        poll_fun = poll
+
+    if count is None:
+        while map:
+            poll_fun(timeout, map)
+
+    else:
+        while map and count > 0:
+            poll_fun(timeout, map)
+            count = count - 1
+
+class dispatcher:
+
+    debug = False
+    connected = False
+    accepting = False
+    closing = False
+    addr = None
+    ignore_log_types = frozenset(['warning'])
+
+    def __init__(self, sock=None, map=None):
+        if map is None:
+            self._map = socket_map
+        else:
+            self._map = map
+
+        self._fileno = None
+
+        if sock:
+            # Set to nonblocking just to make sure for cases where we
+            # get a socket from a blocking source.
+            sock.setblocking(0)
+            self.set_socket(sock, map)
+            self.connected = True
+            # The constructor no longer requires that the socket
+            # passed be connected.
+            try:
+                self.addr = sock.getpeername()
+            except socket.error as err:
+                if err.args[0] == ENOTCONN:
+                    # To handle the case where we got an unconnected
+                    # socket.
+                    self.connected = False
+                else:
+                    # The socket is broken in some unknown way, alert
+                    # the user and remove it from the map (to prevent
+                    # polling of broken sockets).
+                    self.del_channel(map)
+                    raise
+        else:
+            self.socket = None
+
+    def __repr__(self):
+        status = [self.__class__.__module__+"."+self.__class__.__name__]
+        if self.accepting and self.addr:
+            status.append('listening')
+        elif self.connected:
+            status.append('connected')
+        if self.addr is not None:
+            try:
+                status.append('%s:%d' % self.addr)
+            except TypeError:
+                status.append(repr(self.addr))
+        return '<%s at %#x>' % (' '.join(status), id(self))
+
+    __str__ = __repr__
+
+    def add_channel(self, map=None):
+        #self.log_info('adding channel %s' % self)
+        if map is None:
+            map = self._map
+        map[self._fileno] = self
+
+    def del_channel(self, map=None):
+        fd = self._fileno
+        if map is None:
+            map = self._map
+        if fd in map:
+            #self.log_info('closing channel %d:%s' % (fd, self))
+            del map[fd]
+        self._fileno = None
+
+    def create_socket(self, family, type):
+        self.family_and_type = family, type
+        sock = socket.socket(family, type)
+        sock.setblocking(0)
+        self.set_socket(sock)
+
+    def set_socket(self, sock, map=None):
+        self.socket = sock
+##        self.__dict__['socket'] = sock
+        self._fileno = sock.fileno()
+        self.add_channel(map)
+
+    def set_reuse_addr(self):
+        # try to re-use a server port if possible
+        try:
+            self.socket.setsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEADDR,
+                self.socket.getsockopt(socket.SOL_SOCKET,
+                                       socket.SO_REUSEADDR) | 1
+                )
+        except socket.error:
+            pass
+
+    # ==================================================
+    # predicates for select()
+    # these are used as filters for the lists of sockets
+    # to pass to select().
+    # ==================================================
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return True
+
+    # ==================================================
+    # socket object methods.
+    # ==================================================
+
+    def listen(self, num):
+        self.accepting = True
+        if os.name == 'nt' and num > 5:
+            num = 5
+        return self.socket.listen(num)
+
+    def bind(self, addr):
+        self.addr = addr
+        return self.socket.bind(addr)
+
+    def connect(self, address):
+        self.connected = False
+        err = self.socket.connect_ex(address)
+        if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
+        or err == EINVAL and os.name in ('nt', 'ce'):
+            return
+        if err in (0, EISCONN):
+            self.addr = address
+            self.handle_connect_event()
+        else:
+            raise socket.error(err, errorcode[err])
+
+    def accept(self):
+        # XXX can return either an address pair or None
+        try:
+            conn, addr = self.socket.accept()
+        except TypeError:
+            return None
+        except socket.error as why:
+            if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
+                return None
+            else:
+                raise
+        else:
+            return conn, addr
+
+    def send(self, data):
+        try:
+            result = self.socket.send(data)
+            return result
+        except socket.error as why:
+            if why.args[0] == EWOULDBLOCK:
+                return 0
+            elif why.args[0] in _DISCONNECTED:
+                self.handle_close()
+                return 0
+            else:
+                raise
+
+    def recv(self, buffer_size):
+        try:
+            data = self.socket.recv(buffer_size)
+            if not data:
+                # a closed connection is indicated by signaling
+                # a read condition, and having recv() return 0.
+                self.handle_close()
+                return b''
+            else:
+                return data
+        except socket.error as why:
+            # winsock sometimes throws ENOTCONN
+            if why.args[0] in _DISCONNECTED:
+                self.handle_close()
+                return b''
+            else:
+                raise
+
+    def close(self):
+        self.connected = False
+        self.accepting = False
+        self.del_channel()
+        try:
+            self.socket.close()
+        except socket.error as why:
+            if why.args[0] not in (ENOTCONN, EBADF):
+                raise
+
+    # cheap inheritance, used to pass all other attribute
+    # references to the underlying socket object.
+    def __getattr__(self, attr):
+        try:
+            retattr = getattr(self.socket, attr)
+        except AttributeError:
+            raise AttributeError("%s instance has no attribute '%s'"
+                                 %(self.__class__.__name__, attr))
+        else:
+            msg = "%(me)s.%(attr)s is deprecated; use %(me)s.socket.%(attr)s " \
+                  "instead" % {'me' : self.__class__.__name__, 'attr' : attr}
+            warnings.warn(msg, DeprecationWarning, stacklevel=2)
+            return retattr
+
+    # log and log_info may be overridden to provide more sophisticated
+    # logging and warning methods. In general, log is for 'hit' logging
+    # and 'log_info' is for informational, warning and error logging.
+
+    def log(self, message):
+        sys.stderr.write('log: %s\n' % str(message))
+
+    def log_info(self, message, type='info'):
+        if type not in self.ignore_log_types:
+            print('%s: %s' % (type, message))
+
+    def handle_read_event(self):
+        if self.accepting:
+            # accepting sockets are never connected, they "spawn" new
+            # sockets that are connected
+            self.handle_accept()
+        elif not self.connected:
+            self.handle_connect_event()
+            self.handle_read()
+        else:
+            self.handle_read()
+
+    def handle_connect_event(self):
+        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+        if err != 0:
+            raise socket.error(err, _strerror(err))
+        self.handle_connect()
+        self.connected = True
+
+    def handle_write_event(self):
+        if self.accepting:
+            # Accepting sockets shouldn't get a write event.
+            # We will pretend it didn't happen.
+            return
+
+        if not self.connected:
+            #check for errors
+            err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+            if err != 0:
+                raise socket.error(err, _strerror(err))
+
+            self.handle_connect_event()
+        self.handle_write()
+
+    def handle_expt_event(self):
+        # handle_expt_event() is called if there might be an error on the
+        # socket, or if there is OOB data
+        # check for the error condition first
+        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+        if err != 0:
+            # we can get here when select.select() says that there is an
+            # exceptional condition on the socket
+            # since there is an error, we'll go ahead and close the socket
+            # like we would in a subclassed handle_read() that received no
+            # data
+            self.handle_close()
+        else:
+            self.handle_expt()
+
+    def handle_error(self):
+        nil, t, v, tbinfo = compact_traceback()
+
+        # sometimes a user repr method will crash.
+        try:
+            self_repr = repr(self)
+        except:
+            self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
+
+        self.log_info(
+            'uncaptured python exception, closing channel %s (%s:%s %s)' % (
+                self_repr,
+                t,
+                v,
+                tbinfo
+                ),
+            'error'
+            )
+        self.handle_close()
+
+    def handle_expt(self):
+        self.log_info('unhandled incoming priority event', 'warning')
+
+    def handle_read(self):
+        self.log_info('unhandled read event', 'warning')
+
+    def handle_write(self):
+        self.log_info('unhandled write event', 'warning')
+
+    def handle_connect(self):
+        self.log_info('unhandled connect event', 'warning')
+
+    def handle_accept(self):
+        pair = self.accept()
+        if pair is not None:
+            self.handle_accepted(*pair)
+
+    def handle_accepted(self, sock, addr):
+        sock.close()
+        self.log_info('unhandled accepted event', 'warning')
+
+    def handle_close(self):
+        self.log_info('unhandled close event', 'warning')
+        self.close()
+
+# ---------------------------------------------------------------------------
+# adds simple buffered output capability, useful for simple clients.
+# [for more sophisticated usage use asynchat.async_chat]
+# ---------------------------------------------------------------------------
+
+class dispatcher_with_send(dispatcher):
+
+    def __init__(self, sock=None, map=None):
+        dispatcher.__init__(self, sock, map)
+        self.out_buffer = b''
+
+    def initiate_send(self):
+        num_sent = 0
+        num_sent = dispatcher.send(self, self.out_buffer[:512])
+        self.out_buffer = self.out_buffer[num_sent:]
+
+    def handle_write(self):
+        self.initiate_send()
+
+    def writable(self):
+        return (not self.connected) or len(self.out_buffer)
+
+    def send(self, data):
+        if self.debug:
+            self.log_info('sending %s' % repr(data))
+        self.out_buffer = self.out_buffer + data
+        self.initiate_send()
+
+# ---------------------------------------------------------------------------
+# used for debugging.
+# ---------------------------------------------------------------------------
+
+def compact_traceback():
+    t, v, tb = sys.exc_info()
+    tbinfo = []
+    if not tb: # Must have a traceback
+        raise AssertionError("traceback does not exist")
+    while tb:
+        tbinfo.append((
+            tb.tb_frame.f_code.co_filename,
+            tb.tb_frame.f_code.co_name,
+            str(tb.tb_lineno)
+            ))
+        tb = tb.tb_next
+
+    # just to be safe
+    del tb
+
+    file, function, line = tbinfo[-1]
+    info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
+    return (file, function, line), t, v, info
+
+def close_all(map=None, ignore_all=False):
+    if map is None:
+        map = socket_map
+    for x in list(map.values()):
+        try:
+            x.close()
+        except OSError as x:
+            if x.args[0] == EBADF:
+                pass
+            elif not ignore_all:
+                raise
+        except _reraised_exceptions:
+            raise
+        except:
+            if not ignore_all:
+                raise
+    map.clear()
+
+# Asynchronous File I/O:
+#
+# After a little research (reading man pages on various unixen, and
+# digging through the linux kernel), I've determined that select()
+# isn't meant for doing asynchronous file i/o.
+# Heartening, though - reading linux/mm/filemap.c shows that linux
+# supports asynchronous read-ahead.  So _MOST_ of the time, the data
+# will be sitting in memory for us already when we go to read it.
+#
+# What other OS's (besides NT) support async file i/o?  [VMS?]
+#
+# Regardless, this is useful for pipes, and stdin/stdout...
+
+if os.name == 'posix':
+    import fcntl
+
+    class file_wrapper:
+        # Here we override just enough to make a file
+        # look like a socket for the purposes of asyncore.
+        # The passed fd is automatically os.dup()'d
+
+        def __init__(self, fd):
+            self.fd = os.dup(fd)
+
+        def recv(self, *args):
+            return os.read(self.fd, *args)
+
+        def send(self, *args):
+            return os.write(self.fd, *args)
+
+        def getsockopt(self, level, optname, buflen=None):
+            if (level == socket.SOL_SOCKET and
+                optname == socket.SO_ERROR and
+                not buflen):
+                return 0
+            raise NotImplementedError("Only asyncore specific behaviour "
+                                      "implemented.")
+
+        read = recv
+        write = send
+
+        def close(self):
+            os.close(self.fd)
+
+        def fileno(self):
+            return self.fd
+
+    class file_dispatcher(dispatcher):
+
+        def __init__(self, fd, map=None):
+            dispatcher.__init__(self, None, map)
+            self.connected = True
+            try:
+                fd = fd.fileno()
+            except AttributeError:
+                pass
+            self.set_file(fd)
+            # set it to non-blocking mode
+            flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0)
+            flags = flags | os.O_NONBLOCK
+            fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+        def set_file(self, fd):
+            self.socket = file_wrapper(fd)
+            self._fileno = self.socket.fileno()
+            self.add_channel()
diff --git a/lib-python/3.2/base64.py b/lib-python/3.2/base64.py
new file mode 100755
--- /dev/null
+++ b/lib-python/3.2/base64.py
@@ -0,0 +1,408 @@
+#! /usr/bin/env python3
+
+"""RFC 3548: Base16, Base32, Base64 Data Encodings"""
+
+# Modified 04-Oct-1995 by Jack Jansen to use binascii module
+# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
+# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere
+
+import re
+import struct
+import binascii
+
+
+__all__ = [
+    # Legacy interface exports traditional RFC 1521 Base64 encodings
+    'encode', 'decode', 'encodebytes', 'decodebytes',
+    # Generalized interface for other encodings
+    'b64encode', 'b64decode', 'b32encode', 'b32decode',
+    'b16encode', 'b16decode',
+    # Standard Base64 encoding
+    'standard_b64encode', 'standard_b64decode',
+    # Some common Base64 alternatives.  As referenced by RFC 3458, see thread
+    # starting at:
+    #
+    # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html
+    'urlsafe_b64encode', 'urlsafe_b64decode',
+    ]
+
+
+bytes_types = (bytes, bytearray)  # Types acceptable as binary data
+
+
+def _translate(s, altchars):
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    translation = bytearray(range(256))
+    for k, v in altchars.items():
+        translation[ord(k)] = v[0]
+    return s.translate(translation)
+
+
+
+# Base64 encoding/decoding uses binascii
+
+def b64encode(s, altchars=None):
+    """Encode a byte string using Base64.
+
+    s is the byte string to encode.  Optional altchars must be a byte
+    string of length 2 which specifies an alternative alphabet for the
+    '+' and '/' characters.  This allows an application to
+    e.g. generate url or filesystem safe Base64 strings.
+
+    The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    # Strip off the trailing newline
+    encoded = binascii.b2a_base64(s)[:-1]
+    if altchars is not None:
+        if not isinstance(altchars, bytes_types):
+            raise TypeError("expected bytes, not %s"
+                            % altchars.__class__.__name__)
+        assert len(altchars) == 2, repr(altchars)
+        return _translate(encoded, {'+': altchars[0:1], '/': altchars[1:2]})
+    return encoded
+
+
+def b64decode(s, altchars=None, validate=False):
+    """Decode a Base64 encoded byte string.
+
+    s is the byte string to decode.  Optional altchars must be a
+    string of length 2 which specifies the alternative alphabet used
+    instead of the '+' and '/' characters.
+
+    The decoded string is returned.  A binascii.Error is raised if s is
+    incorrectly padded.
+
+    If validate is False (the default), non-base64-alphabet characters are
+    discarded prior to the padding check.  If validate is True,
+    non-base64-alphabet characters in the input result in a binascii.Error.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    if altchars is not None:
+        if not isinstance(altchars, bytes_types):
+            raise TypeError("expected bytes, not %s"
+                            % altchars.__class__.__name__)
+        assert len(altchars) == 2, repr(altchars)
+        s = _translate(s, {chr(altchars[0]): b'+', chr(altchars[1]): b'/'})
+    if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
+        raise binascii.Error('Non-base64 digit found')
+    return binascii.a2b_base64(s)
+
+
+def standard_b64encode(s):
+    """Encode a byte string using the standard Base64 alphabet.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    return b64encode(s)
+
+def standard_b64decode(s):
+    """Decode a byte string encoded with the standard Base64 alphabet.
+
+    s is the byte string to decode.  The decoded byte string is
+    returned.  binascii.Error is raised if the input is incorrectly
+    padded or if there are non-alphabet characters present in the
+    input.
+    """
+    return b64decode(s)
+
+def urlsafe_b64encode(s):
+    """Encode a byte string using a url-safe Base64 alphabet.
+
+    s is the byte string to encode.  The encoded byte string is
+    returned.  The alphabet uses '-' instead of '+' and '_' instead of
+    '/'.
+    """
+    return b64encode(s, b'-_')
+
+def urlsafe_b64decode(s):
+    """Decode a byte string encoded with the standard Base64 alphabet.
+
+    s is the byte string to decode.  The decoded byte string is
+    returned.  binascii.Error is raised if the input is incorrectly
+    padded or if there are non-alphabet characters present in the
+    input.
+
+    The alphabet uses '-' instead of '+' and '_' instead of '/'.
+    """
+    return b64decode(s, b'-_')
+
+
+
+# Base32 encoding/decoding must be done in Python
+_b32alphabet = {
+    0: b'A',  9: b'J', 18: b'S', 27: b'3',
+    1: b'B', 10: b'K', 19: b'T', 28: b'4',
+    2: b'C', 11: b'L', 20: b'U', 29: b'5',
+    3: b'D', 12: b'M', 21: b'V', 30: b'6',
+    4: b'E', 13: b'N', 22: b'W', 31: b'7',
+    5: b'F', 14: b'O', 23: b'X',
+    6: b'G', 15: b'P', 24: b'Y',
+    7: b'H', 16: b'Q', 25: b'Z',
+    8: b'I', 17: b'R', 26: b'2',
+    }
+
+_b32tab = [v[0] for k, v in sorted(_b32alphabet.items())]
+_b32rev = dict([(v[0], k) for k, v in _b32alphabet.items()])
+
+
+def b32encode(s):
+    """Encode a byte string using Base32.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    quanta, leftover = divmod(len(s), 5)
+    # Pad the last quantum with zero bits if necessary
+    if leftover:
+        s = s + bytes(5 - leftover)  # Don't use += !
+        quanta += 1
+    encoded = bytes()
+    for i in range(quanta):
+        # c1 and c2 are 16 bits wide, c3 is 8 bits wide.  The intent of this
+        # code is to process the 40 bits in units of 5 bits.  So we take the 1
+        # leftover bit of c1 and tack it onto c2.  Then we take the 2 leftover
+        # bits of c2 and tack them onto c3.  The shifts and masks are intended
+        # to give us values of exactly 5 bits in width.
+        c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5])
+        c2 += (c1 & 1) << 16 # 17 bits wide
+        c3 += (c2 & 3) << 8  # 10 bits wide
+        encoded += bytes([_b32tab[c1 >> 11],         # bits 1 - 5
+                          _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10
+                          _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15
+                          _b32tab[c2 >> 12],         # bits 16 - 20 (1 - 5)
+                          _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10)
+                          _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15)
+                          _b32tab[c3 >> 5],          # bits 31 - 35 (1 - 5)
+                          _b32tab[c3 & 0x1f],        # bits 36 - 40 (1 - 5)
+                          ])
+    # Adjust for any leftover partial quanta
+    if leftover == 1:
+        return encoded[:-6] + b'======'
+    elif leftover == 2:
+        return encoded[:-4] + b'===='
+    elif leftover == 3:
+        return encoded[:-3] + b'==='
+    elif leftover == 4:
+        return encoded[:-1] + b'='
+    return encoded
+
+
+def b32decode(s, casefold=False, map01=None):
+    """Decode a Base32 encoded byte string.
+
+    s is the byte string to decode.  Optional casefold is a flag
+    specifying whether a lowercase alphabet is acceptable as input.
+    For security purposes, the default is False.
+
+    RFC 3548 allows for optional mapping of the digit 0 (zero) to the
+    letter O (oh), and for optional mapping of the digit 1 (one) to
+    either the letter I (eye) or letter L (el).  The optional argument
+    map01 when not None, specifies which letter the digit 1 should be
+    mapped to (when map01 is not None, the digit 0 is always mapped to
+    the letter O).  For security purposes the default is None, so that
+    0 and 1 are not allowed in the input.
+
+    The decoded byte string is returned.  binascii.Error is raised if
+    the input is incorrectly padded or if there are non-alphabet
+    characters present in the input.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    quanta, leftover = divmod(len(s), 8)
+    if leftover:
+        raise binascii.Error('Incorrect padding')
+    # Handle section 2.4 zero and one mapping.  The flag map01 will be either
+    # False, or the character to map the digit 1 (one) to.  It should be
+    # either L (el) or I (eye).
+    if map01 is not None:
+        if not isinstance(map01, bytes_types):
+            raise TypeError("expected bytes, not %s" % map01.__class__.__name__)
+        assert len(map01) == 1, repr(map01)
+        s = _translate(s, {b'0': b'O', b'1': map01})
+    if casefold:
+        s = s.upper()
+    # Strip off pad characters from the right.  We need to count the pad
+    # characters because this will tell us how many null bytes to remove from
+    # the end of the decoded string.
+    padchars = 0
+    mo = re.search(b'(?P<pad>[=]*)$', s)
+    if mo:
+        padchars = len(mo.group('pad'))
+        if padchars > 0:
+            s = s[:-padchars]
+    # Now decode the full quanta
+    parts = []
+    acc = 0
+    shift = 35
+    for c in s:
+        val = _b32rev.get(c)
+        if val is None:
+            raise TypeError('Non-base32 digit found')
+        acc += _b32rev[c] << shift
+        shift -= 5
+        if shift < 0:
+            parts.append(binascii.unhexlify(bytes('%010x' % acc, "ascii")))
+            acc = 0
+            shift = 35
+    # Process the last, partial quanta
+    last = binascii.unhexlify(bytes('%010x' % acc, "ascii"))
+    if padchars == 0:
+        last = b''                      # No characters
+    elif padchars == 1:
+        last = last[:-1]
+    elif padchars == 3:
+        last = last[:-2]
+    elif padchars == 4:
+        last = last[:-3]
+    elif padchars == 6:
+        last = last[:-4]
+    else:
+        raise binascii.Error('Incorrect padding')
+    parts.append(last)
+    return b''.join(parts)
+
+
+
+# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
+# lowercase.  The RFC also recommends against accepting input case
+# insensitively.
+def b16encode(s):
+    """Encode a byte string using Base16.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    return binascii.hexlify(s).upper()
+
+
+def b16decode(s, casefold=False):
+    """Decode a Base16 encoded byte string.
+
+    s is the byte string to decode.  Optional casefold is a flag
+    specifying whether a lowercase alphabet is acceptable as input.
+    For security purposes, the default is False.
+
+    The decoded byte string is returned.  binascii.Error is raised if
+    s were incorrectly padded or if there are non-alphabet characters
+    present in the string.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    if casefold:
+        s = s.upper()
+    if re.search(b'[^0-9A-F]', s):
+        raise binascii.Error('Non-base16 digit found')
+    return binascii.unhexlify(s)
+
+
+
+# Legacy interface.  This code could be cleaned up since I don't believe
+# binascii has any line length limitations.  It just doesn't seem worth it
+# though.  The files should be opened in binary mode.
+
+MAXLINESIZE = 76 # Excluding the CRLF
+MAXBINSIZE = (MAXLINESIZE//4)*3
+
+def encode(input, output):
+    """Encode a file; input and output are binary files."""
+    while True:
+        s = input.read(MAXBINSIZE)
+        if not s:
+            break
+        while len(s) < MAXBINSIZE:
+            ns = input.read(MAXBINSIZE-len(s))
+            if not ns:
+                break
+            s += ns
+        line = binascii.b2a_base64(s)
+        output.write(line)
+
+
+def decode(input, output):
+    """Decode a file; input and output are binary files."""
+    while True:
+        line = input.readline()
+        if not line:
+            break
+        s = binascii.a2b_base64(line)
+        output.write(s)
+
+
+def encodebytes(s):
+    """Encode a bytestring into a bytestring containing multiple lines
+    of base-64 data."""
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    pieces = []
+    for i in range(0, len(s), MAXBINSIZE):
+        chunk = s[i : i + MAXBINSIZE]
+        pieces.append(binascii.b2a_base64(chunk))
+    return b"".join(pieces)
+
+def encodestring(s):
+    """Legacy alias of encodebytes()."""
+    import warnings
+    warnings.warn("encodestring() is a deprecated alias, use encodebytes()",
+                  DeprecationWarning, 2)
+    return encodebytes(s)
+
+
+def decodebytes(s):
+    """Decode a bytestring of base-64 data into a bytestring."""
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    return binascii.a2b_base64(s)
+
+def decodestring(s):
+    """Legacy alias of decodebytes()."""
+    import warnings
+    warnings.warn("decodestring() is a deprecated alias, use decodebytes()",
+                  DeprecationWarning, 2)
+    return decodebytes(s)
+
+
+# Usable as a script...
+def main():
+    """Small main program"""
+    import sys, getopt
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'deut')
+    except getopt.error as msg:
+        sys.stdout = sys.stderr
+        print(msg)
+        print("""usage: %s [-d|-e|-u|-t] [file|-]
+        -d, -u: decode
+        -e: encode (default)
+        -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
+        sys.exit(2)
+    func = encode
+    for o, a in opts:
+        if o == '-e': func = encode
+        if o == '-d': func = decode
+        if o == '-u': func = decode
+        if o == '-t': test(); return
+    if args and args[0] != '-':
+        with open(args[0], 'rb') as f:
+            func(f, sys.stdout.buffer)
+    else:
+        func(sys.stdin.buffer, sys.stdout.buffer)
+
+
+def test():
+    s0 = b"Aladdin:open sesame"
+    print(repr(s0))
+    s1 = encodebytes(s0)
+    print(repr(s1))
+    s2 = decodebytes(s1)
+    print(repr(s2))
+    assert s0 == s2
+
+
+if __name__ == '__main__':
+    main()
diff --git a/lib-python/3.2/bdb.py b/lib-python/3.2/bdb.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/bdb.py
@@ -0,0 +1,634 @@
+"""Debugger basics"""
+
+import fnmatch
+import sys
+import os
+
+__all__ = ["BdbQuit", "Bdb", "Breakpoint"]
+
+class BdbQuit(Exception):
+    """Exception to give up completely."""
+
+
+class Bdb:
+    """Generic Python debugger base class.
+
+    This class takes care of details of the trace facility;
+    a derived class should implement user interaction.
+    The standard debugger class (pdb.Pdb) is an example.
+    """
+
+    def __init__(self, skip=None):
+        self.skip = set(skip) if skip else None
+        self.breaks = {}
+        self.fncache = {}
+
+    def canonic(self, filename):
+        if filename == "<" + filename[1:-1] + ">":
+            return filename
+        canonic = self.fncache.get(filename)
+        if not canonic:
+            canonic = os.path.abspath(filename)
+            canonic = os.path.normcase(canonic)
+            self.fncache[filename] = canonic
+        return canonic
+
+    def reset(self):
+        import linecache
+        linecache.checkcache()
+        self.botframe = None
+        self._set_stopinfo(None, None)
+
+    def trace_dispatch(self, frame, event, arg):
+        if self.quitting:
+            return # None
+        if event == 'line':
+            return self.dispatch_line(frame)
+        if event == 'call':
+            return self.dispatch_call(frame, arg)
+        if event == 'return':
+            return self.dispatch_return(frame, arg)
+        if event == 'exception':
+            return self.dispatch_exception(frame, arg)
+        if event == 'c_call':
+            return self.trace_dispatch
+        if event == 'c_exception':
+            return self.trace_dispatch
+        if event == 'c_return':
+            return self.trace_dispatch
+        print('bdb.Bdb.dispatch: unknown debugging event:', repr(event))
+        return self.trace_dispatch
+
+    def dispatch_line(self, frame):
+        if self.stop_here(frame) or self.break_here(frame):
+            self.user_line(frame)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_call(self, frame, arg):
+        # XXX 'arg' is no longer used
+        if self.botframe is None:
+            # First call of dispatch since reset()
+            self.botframe = frame.f_back # (CT) Note that this may also be None!
+            return self.trace_dispatch
+        if not (self.stop_here(frame) or self.break_anywhere(frame)):
+            # No need to trace this function
+            return # None
+        self.user_call(frame, arg)
+        if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_return(self, frame, arg):
+        if self.stop_here(frame) or frame == self.returnframe:
+            self.user_return(frame, arg)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_exception(self, frame, arg):
+        if self.stop_here(frame):
+            self.user_exception(frame, arg)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    # Normally derived classes don't override the following
+    # methods, but they may if they want to redefine the
+    # definition of stopping and breakpoints.
+
+    def is_skipped_module(self, module_name):
+        for pattern in self.skip:
+            if fnmatch.fnmatch(module_name, pattern):
+                return True
+        return False
+
+    def stop_here(self, frame):
+        # (CT) stopframe may now also be None, see dispatch_call.
+        # (CT) the former test for None is therefore removed from here.
+        if self.skip and \
+               self.is_skipped_module(frame.f_globals.get('__name__')):
+            return False
+        if frame is self.stopframe:
+            if self.stoplineno == -1:
+                return False
+            return frame.f_lineno >= self.stoplineno
+        while frame is not None and frame is not self.stopframe:
+            if frame is self.botframe:
+                return True
+            frame = frame.f_back
+        return False
+
+    def break_here(self, frame):
+        filename = self.canonic(frame.f_code.co_filename)
+        if filename not in self.breaks:
+            return False
+        lineno = frame.f_lineno
+        if lineno not in self.breaks[filename]:
+            # The line itself has no breakpoint, but maybe the line is the
+            # first line of a function with breakpoint set by function name.
+            lineno = frame.f_code.co_firstlineno
+            if lineno not in self.breaks[filename]:
+                return False
+
+        # flag says ok to delete temp. bp
+        (bp, flag) = effective(filename, lineno, frame)
+        if bp:
+            self.currentbp = bp.number
+            if (flag and bp.temporary):
+                self.do_clear(str(bp.number))
+            return True
+        else:
+            return False
+
+    def do_clear(self, arg):
+        raise NotImplementedError("subclass of bdb must implement do_clear()")
+
+    def break_anywhere(self, frame):
+        return self.canonic(frame.f_code.co_filename) in self.breaks
+
+    # Derived classes should override the user_* methods
+    # to gain control.
+
+    def user_call(self, frame, argument_list):
+        """This method is called when there is the remote possibility
+        that we ever need to stop in this function."""
+        pass
+
+    def user_line(self, frame):
+        """This method is called when we stop or break at this line."""
+        pass
+
+    def user_return(self, frame, return_value):
+        """This method is called when a return trap is set here."""
+        pass
+
+    def user_exception(self, frame, exc_info):
+        """This method is called if an exception occurs,
+        but only if we are to stop at or just below this level."""
+        pass
+
+    def _set_stopinfo(self, stopframe, returnframe, stoplineno=0):
+        self.stopframe = stopframe
+        self.returnframe = returnframe
+        self.quitting = False
+        # stoplineno >= 0 means: stop at line >= the stoplineno
+        # stoplineno -1 means: don't stop at all
+        self.stoplineno = stoplineno
+
+    # Derived classes and clients can call the following methods
+    # to affect the stepping state.
+
+    def set_until(self, frame, lineno=None):
+        """Stop when the line with the line no greater than the current one is
+        reached or when returning from current frame"""
+        # the name "until" is borrowed from gdb
+        if lineno is None:
+            lineno = frame.f_lineno + 1
+        self._set_stopinfo(frame, frame, lineno)
+
+    def set_step(self):
+        """Stop after one line of code."""
+        self._set_stopinfo(None, None)
+
+    def set_next(self, frame):
+        """Stop on the next line in or below the given frame."""
+        self._set_stopinfo(frame, None)
+
+    def set_return(self, frame):
+        """Stop when returning from the given frame."""
+        self._set_stopinfo(frame.f_back, frame)
+
+    def set_trace(self, frame=None):
+        """Start debugging from `frame`.
+
+        If frame is not specified, debugging starts from caller's frame.
+        """
+        if frame is None:
+            frame = sys._getframe().f_back
+        self.reset()
+        while frame:
+            frame.f_trace = self.trace_dispatch
+            self.botframe = frame
+            frame = frame.f_back
+        self.set_step()
+        sys.settrace(self.trace_dispatch)
+
+    def set_continue(self):
+        # Don't stop except at breakpoints or when finished
+        self._set_stopinfo(self.botframe, None, -1)
+        if not self.breaks:
+            # no breakpoints; run without debugger overhead
+            sys.settrace(None)
+            frame = sys._getframe().f_back
+            while frame and frame is not self.botframe:
+                del frame.f_trace
+                frame = frame.f_back
+
+    def set_quit(self):
+        self.stopframe = self.botframe
+        self.returnframe = None
+        self.quitting = True
+        sys.settrace(None)
+
+    # Derived classes and clients can call the following methods
+    # to manipulate breakpoints.  These methods return an
+    # error message is something went wrong, None if all is well.
+    # Set_break prints out the breakpoint line and file:lineno.
+    # Call self.get_*break*() to see the breakpoints or better
+    # for bp in Breakpoint.bpbynumber: if bp: bp.bpprint().
+
+    def set_break(self, filename, lineno, temporary=False, cond=None,
+                  funcname=None):
+        filename = self.canonic(filename)
+        import linecache # Import as late as possible
+        line = linecache.getline(filename, lineno)
+        if not line:
+            return 'Line %s:%d does not exist' % (filename, lineno)
+        list = self.breaks.setdefault(filename, [])
+        if lineno not in list:
+            list.append(lineno)
+        bp = Breakpoint(filename, lineno, temporary, cond, funcname)
+
+    def _prune_breaks(self, filename, lineno):
+        if (filename, lineno) not in Breakpoint.bplist:
+            self.breaks[filename].remove(lineno)
+        if not self.breaks[filename]:
+            del self.breaks[filename]
+
+    def clear_break(self, filename, lineno):
+        filename = self.canonic(filename)
+        if filename not in self.breaks:
+            return 'There are no breakpoints in %s' % filename
+        if lineno not in self.breaks[filename]:
+            return 'There is no breakpoint at %s:%d' % (filename, lineno)
+        # If there's only one bp in the list for that file,line
+        # pair, then remove the breaks entry
+        for bp in Breakpoint.bplist[filename, lineno][:]:
+            bp.deleteMe()
+        self._prune_breaks(filename, lineno)
+
+    def clear_bpbynumber(self, arg):
+        try:
+            bp = self.get_bpbynumber(arg)
+        except ValueError as err:
+            return str(err)
+        bp.deleteMe()
+        self._prune_breaks(bp.file, bp.line)
+
+    def clear_all_file_breaks(self, filename):
+        filename = self.canonic(filename)
+        if filename not in self.breaks:
+            return 'There are no breakpoints in %s' % filename
+        for line in self.breaks[filename]:
+            blist = Breakpoint.bplist[filename, line]
+            for bp in blist:
+                bp.deleteMe()
+        del self.breaks[filename]
+
+    def clear_all_breaks(self):
+        if not self.breaks:
+            return 'There are no breakpoints'
+        for bp in Breakpoint.bpbynumber:
+            if bp:
+                bp.deleteMe()
+        self.breaks = {}
+
+    def get_bpbynumber(self, arg):
+        if not arg:
+            raise ValueError('Breakpoint number expected')
+        try:
+            number = int(arg)
+        except ValueError:
+            raise ValueError('Non-numeric breakpoint number %s' % arg)
+        try:
+            bp = Breakpoint.bpbynumber[number]
+        except IndexError:
+            raise ValueError('Breakpoint number %d out of range' % number)
+        if bp is None:
+            raise ValueError('Breakpoint %d already deleted' % number)
+        return bp
+
+    def get_break(self, filename, lineno):
+        filename = self.canonic(filename)
+        return filename in self.breaks and \
+            lineno in self.breaks[filename]
+
+    def get_breaks(self, filename, lineno):
+        filename = self.canonic(filename)
+        return filename in self.breaks and \
+            lineno in self.breaks[filename] and \
+            Breakpoint.bplist[filename, lineno] or []

From noreply at buildbot.pypy.org  Sat Apr 14 11:34:44 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 11:34:44 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: merge from vendor/stdlib
Message-ID: <20120414093444.9C87C82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54358:83c48fa1563a
Date: 2012-04-14 11:27 +0200
http://bitbucket.org/pypy/pypy/changeset/83c48fa1563a/

Log:	merge from vendor/stdlib

diff too long, truncating to 10000 out of 558051 lines

diff --git a/lib-python/3.2/__future__.py b/lib-python/3.2/__future__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/__future__.py
@@ -0,0 +1,134 @@
+"""Record of phased-in incompatible language changes.
+
+Each line is of the form:
+
+    FeatureName = "_Feature(" OptionalRelease "," MandatoryRelease ","
+                              CompilerFlag ")"
+
+where, normally, OptionalRelease < MandatoryRelease, and both are 5-tuples
+of the same form as sys.version_info:
+
+    (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int
+     PY_MINOR_VERSION, # the 1; an int
+     PY_MICRO_VERSION, # the 0; an int
+     PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string
+     PY_RELEASE_SERIAL # the 3; an int
+    )
+
+OptionalRelease records the first release in which
+
+    from __future__ import FeatureName
+
+was accepted.
+
+In the case of MandatoryReleases that have not yet occurred,
+MandatoryRelease predicts the release in which the feature will become part
+of the language.
+
+Else MandatoryRelease records when the feature became part of the language;
+in releases at or after that, modules no longer need
+
+    from __future__ import FeatureName
+
+to use the feature in question, but may continue to use such imports.
+
+MandatoryRelease may also be None, meaning that a planned feature got
+dropped.
+
+Instances of class _Feature have two corresponding methods,
+.getOptionalRelease() and .getMandatoryRelease().
+
+CompilerFlag is the (bitfield) flag that should be passed in the fourth
+argument to the builtin function compile() to enable the feature in
+dynamically compiled code.  This flag is stored in the .compiler_flag
+attribute on _Future instances.  These values must match the appropriate
+#defines of CO_xxx flags in Include/compile.h.
+
+No feature line is ever to be deleted from this file.
+"""
+
+all_feature_names = [
+    "nested_scopes",
+    "generators",
+    "division",
+    "absolute_import",
+    "with_statement",
+    "print_function",
+    "unicode_literals",
+    "barry_as_FLUFL",
+]
+
+__all__ = ["all_feature_names"] + all_feature_names
+
+# The CO_xxx symbols are defined here under the same names used by
+# compile.h, so that an editor search will find them here.  However,
+# they're not exported in __all__, because they don't really belong to
+# this module.
+CO_NESTED            = 0x0010   # nested_scopes
+CO_GENERATOR_ALLOWED = 0        # generators (obsolete, was 0x1000)
+CO_FUTURE_DIVISION   = 0x2000   # division
+CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
+CO_FUTURE_WITH_STATEMENT  = 0x8000   # with statement
+CO_FUTURE_PRINT_FUNCTION  = 0x10000   # print function
+CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
+CO_FUTURE_BARRY_AS_BDFL = 0x40000
+
+class _Feature:
+    def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
+        self.optional = optionalRelease
+        self.mandatory = mandatoryRelease
+        self.compiler_flag = compiler_flag
+
+    def getOptionalRelease(self):
+        """Return first release in which this feature was recognized.
+
+        This is a 5-tuple, of the same form as sys.version_info.
+        """
+
+        return self.optional
+
+    def getMandatoryRelease(self):
+        """Return release in which this feature will become mandatory.
+
+        This is a 5-tuple, of the same form as sys.version_info, or, if
+        the feature was dropped, is None.
+        """
+
+        return self.mandatory
+
+    def __repr__(self):
+        return "_Feature" + repr((self.optional,
+                                  self.mandatory,
+                                  self.compiler_flag))
+
+nested_scopes = _Feature((2, 1, 0, "beta",  1),
+                         (2, 2, 0, "alpha", 0),
+                         CO_NESTED)
+
+generators = _Feature((2, 2, 0, "alpha", 1),
+                      (2, 3, 0, "final", 0),
+                      CO_GENERATOR_ALLOWED)
+
+division = _Feature((2, 2, 0, "alpha", 2),
+                    (3, 0, 0, "alpha", 0),
+                    CO_FUTURE_DIVISION)
+
+absolute_import = _Feature((2, 5, 0, "alpha", 1),
+                           (2, 7, 0, "alpha", 0),
+                           CO_FUTURE_ABSOLUTE_IMPORT)
+
+with_statement = _Feature((2, 5, 0, "alpha", 1),
+                          (2, 6, 0, "alpha", 0),
+                          CO_FUTURE_WITH_STATEMENT)
+
+print_function = _Feature((2, 6, 0, "alpha", 2),
+                          (3, 0, 0, "alpha", 0),
+                          CO_FUTURE_PRINT_FUNCTION)
+
+unicode_literals = _Feature((2, 6, 0, "alpha", 2),
+                            (3, 0, 0, "alpha", 0),
+                            CO_FUTURE_UNICODE_LITERALS)
+
+barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
+                         (3, 9, 0, "alpha", 0),
+                         CO_FUTURE_BARRY_AS_BDFL)
diff --git a/lib-python/3.2/__phello__.foo.py b/lib-python/3.2/__phello__.foo.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/__phello__.foo.py
@@ -0,0 +1,1 @@
+# This file exists as a helper for the test.test_frozen module.
diff --git a/lib-python/3.2/_abcoll.py b/lib-python/3.2/_abcoll.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_abcoll.py
@@ -0,0 +1,623 @@
+# Copyright 2007 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Abstract Base Classes (ABCs) for collections, according to PEP 3119.
+
+DON'T USE THIS MODULE DIRECTLY!  The classes here should be imported
+via collections; they are defined here only to alleviate certain
+bootstrapping issues.  Unit tests are in test_collections.
+"""
+
+from abc import ABCMeta, abstractmethod
+import sys
+
+__all__ = ["Hashable", "Iterable", "Iterator",
+           "Sized", "Container", "Callable",
+           "Set", "MutableSet",
+           "Mapping", "MutableMapping",
+           "MappingView", "KeysView", "ItemsView", "ValuesView",
+           "Sequence", "MutableSequence",
+           "ByteString",
+           ]
+
+
+### collection related types which are not exposed through builtin ###
+## iterators ##
+bytes_iterator = type(iter(b''))
+bytearray_iterator = type(iter(bytearray()))
+#callable_iterator = ???
+dict_keyiterator = type(iter({}.keys()))
+dict_valueiterator = type(iter({}.values()))
+dict_itemiterator = type(iter({}.items()))
+list_iterator = type(iter([]))
+list_reverseiterator = type(iter(reversed([])))
+range_iterator = type(iter(range(0)))
+set_iterator = type(iter(set()))
+str_iterator = type(iter(""))
+tuple_iterator = type(iter(()))
+zip_iterator = type(iter(zip()))
+## views ##
+dict_keys = type({}.keys())
+dict_values = type({}.values())
+dict_items = type({}.items())
+## misc ##
+dict_proxy = type(type.__dict__)
+
+
+### ONE-TRICK PONIES ###
+
+class Hashable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __hash__(self):
+        return 0
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Hashable:
+            for B in C.__mro__:
+                if "__hash__" in B.__dict__:
+                    if B.__dict__["__hash__"]:
+                        return True
+                    break
+        return NotImplemented
+
+
+class Iterable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __iter__(self):
+        while False:
+            yield None
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Iterable:
+            if any("__iter__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Iterator(Iterable):
+
+    @abstractmethod
+    def __next__(self):
+        raise StopIteration
+
+    def __iter__(self):
+        return self
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Iterator:
+            if (any("__next__" in B.__dict__ for B in C.__mro__) and
+                any("__iter__" in B.__dict__ for B in C.__mro__)):
+                return True
+        return NotImplemented
+
+Iterator.register(bytes_iterator)
+Iterator.register(bytearray_iterator)
+#Iterator.register(callable_iterator)
+Iterator.register(dict_keyiterator)
+Iterator.register(dict_valueiterator)
+Iterator.register(dict_itemiterator)
+Iterator.register(list_iterator)
+Iterator.register(list_reverseiterator)
+Iterator.register(range_iterator)
+Iterator.register(set_iterator)
+Iterator.register(str_iterator)
+Iterator.register(tuple_iterator)
+Iterator.register(zip_iterator)
+
+class Sized(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __len__(self):
+        return 0
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Sized:
+            if any("__len__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Container(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __contains__(self, x):
+        return False
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Container:
+            if any("__contains__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Callable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __call__(self, *args, **kwds):
+        return False
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Callable:
+            if any("__call__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+### SETS ###
+
+
+class Set(Sized, Iterable, Container):
+
+    """A set is a finite, iterable container.
+
+    This class provides concrete generic implementations of all
+    methods except for __contains__, __iter__ and __len__.
+
+    To override the comparisons (presumably for speed, as the
+    semantics are fixed), all you have to do is redefine __le__ and
+    then the other operations will automatically follow suit.
+    """
+
+    def __le__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        if len(self) > len(other):
+            return False
+        for elem in self:
+            if elem not in other:
+                return False
+        return True
+
+    def __lt__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return len(self) < len(other) and self.__le__(other)
+
+    def __gt__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return other < self
+
+    def __ge__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return other <= self
+
+    def __eq__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return len(self) == len(other) and self.__le__(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    @classmethod
+    def _from_iterable(cls, it):
+        '''Construct an instance of the class from any iterable input.
+
+        Must override this method if the class constructor signature
+        does not accept an iterable for an input.
+        '''
+        return cls(it)
+
+    def __and__(self, other):
+        if not isinstance(other, Iterable):
+            return NotImplemented
+        return self._from_iterable(value for value in other if value in self)
+
+    def isdisjoint(self, other):
+        for value in other:
+            if value in self:
+                return False
+        return True
+
+    def __or__(self, other):
+        if not isinstance(other, Iterable):
+            return NotImplemented
+        chain = (e for s in (self, other) for e in s)
+        return self._from_iterable(chain)
+
+    def __sub__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return self._from_iterable(value for value in self
+                                   if value not in other)
+
+    def __xor__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return (self - other) | (other - self)
+
+    def _hash(self):
+        """Compute the hash value of a set.
+
+        Note that we don't define __hash__: not all sets are hashable.
+        But if you define a hashable set type, its __hash__ should
+        call this function.
+
+        This must be compatible __eq__.
+
+        All sets ought to compare equal if they contain the same
+        elements, regardless of how they are implemented, and
+        regardless of the order of the elements; so there's not much
+        freedom for __eq__ or __hash__.  We match the algorithm used
+        by the built-in frozenset type.
+        """
+        MAX = sys.maxsize
+        MASK = 2 * MAX + 1
+        n = len(self)
+        h = 1927868237 * (n + 1)
+        h &= MASK
+        for x in self:
+            hx = hash(x)
+            h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167
+            h &= MASK
+        h = h * 69069 + 907133923
+        h &= MASK
+        if h > MAX:
+            h -= MASK + 1
+        if h == -1:
+            h = 590923713
+        return h
+
+Set.register(frozenset)
+
+
+class MutableSet(Set):
+
+    @abstractmethod
+    def add(self, value):
+        """Add an element."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def discard(self, value):
+        """Remove an element.  Do not raise an exception if absent."""
+        raise NotImplementedError
+
+    def remove(self, value):
+        """Remove an element. If not a member, raise a KeyError."""
+        if value not in self:
+            raise KeyError(value)
+        self.discard(value)
+
+    def pop(self):
+        """Return the popped value.  Raise KeyError if empty."""
+        it = iter(self)
+        try:
+            value = next(it)
+        except StopIteration:
+            raise KeyError
+        self.discard(value)
+        return value
+
+    def clear(self):
+        """This is slow (creates N new iterators!) but effective."""
+        try:
+            while True:
+                self.pop()
+        except KeyError:
+            pass
+
+    def __ior__(self, it):
+        for value in it:
+            self.add(value)
+        return self
+
+    def __iand__(self, it):
+        for value in (self - it):
+            self.discard(value)
+        return self
+
+    def __ixor__(self, it):
+        if it is self:
+            self.clear()
+        else:
+            if not isinstance(it, Set):
+                it = self._from_iterable(it)
+            for value in it:
+                if value in self:
+                    self.discard(value)
+                else:
+                    self.add(value)
+        return self
+
+    def __isub__(self, it):
+        if it is self:
+            self.clear()
+        else:
+            for value in it:
+                self.discard(value)
+        return self
+
+MutableSet.register(set)
+
+
+### MAPPINGS ###
+
+
+class Mapping(Sized, Iterable, Container):
+
+    @abstractmethod
+    def __getitem__(self, key):
+        raise KeyError
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def __contains__(self, key):
+        try:
+            self[key]
+        except KeyError:
+            return False
+        else:
+            return True
+
+    def keys(self):
+        return KeysView(self)
+
+    def items(self):
+        return ItemsView(self)
+
+    def values(self):
+        return ValuesView(self)
+
+    def __eq__(self, other):
+        if not isinstance(other, Mapping):
+            return NotImplemented
+        return dict(self.items()) == dict(other.items())
+
+    def __ne__(self, other):
+        return not (self == other)
+
+
+class MappingView(Sized):
+
+    def __init__(self, mapping):
+        self._mapping = mapping
+
+    def __len__(self):
+        return len(self._mapping)
+
+    def __repr__(self):
+        return '{0.__class__.__name__}({0._mapping!r})'.format(self)
+
+
+class KeysView(MappingView, Set):
+
+    @classmethod
+    def _from_iterable(self, it):
+        return set(it)
+
+    def __contains__(self, key):
+        return key in self._mapping
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield key
+
+KeysView.register(dict_keys)
+
+
+class ItemsView(MappingView, Set):
+
+    @classmethod
+    def _from_iterable(self, it):
+        return set(it)
+
+    def __contains__(self, item):
+        key, value = item
+        try:
+            v = self._mapping[key]
+        except KeyError:
+            return False
+        else:
+            return v == value
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield (key, self._mapping[key])
+
+ItemsView.register(dict_items)
+
+
+class ValuesView(MappingView):
+
+    def __contains__(self, value):
+        for key in self._mapping:
+            if value == self._mapping[key]:
+                return True
+        return False
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield self._mapping[key]
+
+ValuesView.register(dict_values)
+
+
+class MutableMapping(Mapping):
+
+    @abstractmethod
+    def __setitem__(self, key, value):
+        raise KeyError
+
+    @abstractmethod
+    def __delitem__(self, key):
+        raise KeyError
+
+    __marker = object()
+
+    def pop(self, key, default=__marker):
+        try:
+            value = self[key]
+        except KeyError:
+            if default is self.__marker:
+                raise
+            return default
+        else:
+            del self[key]
+            return value
+
+    def popitem(self):
+        try:
+            key = next(iter(self))
+        except StopIteration:
+            raise KeyError
+        value = self[key]
+        del self[key]
+        return key, value
+
+    def clear(self):
+        try:
+            while True:
+                self.popitem()
+        except KeyError:
+            pass
+
+    def update(*args, **kwds):
+        if len(args) > 2:
+            raise TypeError("update() takes at most 2 positional "
+                            "arguments ({} given)".format(len(args)))
+        elif not args:
+            raise TypeError("update() takes at least 1 argument (0 given)")
+        self = args[0]
+        other = args[1] if len(args) >= 2 else ()
+
+        if isinstance(other, Mapping):
+            for key in other:
+                self[key] = other[key]
+        elif hasattr(other, "keys"):
+            for key in other.keys():
+                self[key] = other[key]
+        else:
+            for key, value in other:
+                self[key] = value
+        for key, value in kwds.items():
+            self[key] = value
+
+    def setdefault(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            self[key] = default
+        return default
+
+MutableMapping.register(dict)
+
+
+### SEQUENCES ###
+
+
+class Sequence(Sized, Iterable, Container):
+
+    """All the operations on a read-only sequence.
+
+    Concrete subclasses must override __new__ or __init__,
+    __getitem__, and __len__.
+    """
+
+    @abstractmethod
+    def __getitem__(self, index):
+        raise IndexError
+
+    def __iter__(self):
+        i = 0
+        try:
+            while True:
+                v = self[i]
+                yield v
+                i += 1
+        except IndexError:
+            return
+
+    def __contains__(self, value):
+        for v in self:
+            if v == value:
+                return True
+        return False
+
+    def __reversed__(self):
+        for i in reversed(range(len(self))):
+            yield self[i]
+
+    def index(self, value):
+        for i, v in enumerate(self):
+            if v == value:
+                return i
+        raise ValueError
+
+    def count(self, value):
+        return sum(1 for v in self if v == value)
+
+Sequence.register(tuple)
+Sequence.register(str)
+Sequence.register(range)
+
+
+class ByteString(Sequence):
+
+    """This unifies bytes and bytearray.
+
+    XXX Should add all their methods.
+    """
+
+ByteString.register(bytes)
+ByteString.register(bytearray)
+
+
+class MutableSequence(Sequence):
+
+    @abstractmethod
+    def __setitem__(self, index, value):
+        raise IndexError
+
+    @abstractmethod
+    def __delitem__(self, index):
+        raise IndexError
+
+    @abstractmethod
+    def insert(self, index, value):
+        raise IndexError
+
+    def append(self, value):
+        self.insert(len(self), value)
+
+    def reverse(self):
+        n = len(self)
+        for i in range(n//2):
+            self[i], self[n-i-1] = self[n-i-1], self[i]
+
+    def extend(self, values):
+        for v in values:
+            self.append(v)
+
+    def pop(self, index=-1):
+        v = self[index]
+        del self[index]
+        return v
+
+    def remove(self, value):
+        del self[self.index(value)]
+
+    def __iadd__(self, values):
+        self.extend(values)
+        return self
+
+MutableSequence.register(list)
+MutableSequence.register(bytearray)  # Multiply inheriting, see ByteString
diff --git a/lib-python/3.2/_compat_pickle.py b/lib-python/3.2/_compat_pickle.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_compat_pickle.py
@@ -0,0 +1,81 @@
+# This module is used to map the old Python 2 names to the new names used in
+# Python 3 for the pickle module.  This needed to make pickle streams
+# generated with Python 2 loadable by Python 3.
+
+# This is a copy of lib2to3.fixes.fix_imports.MAPPING.  We cannot import
+# lib2to3 and use the mapping defined there, because lib2to3 uses pickle.
+# Thus, this could cause the module to be imported recursively.
+IMPORT_MAPPING = {
+    'StringIO':  'io',
+    'cStringIO': 'io',
+    'cPickle': 'pickle',
+    '__builtin__' : 'builtins',
+    'copy_reg': 'copyreg',
+    'Queue': 'queue',
+    'SocketServer': 'socketserver',
+    'ConfigParser': 'configparser',
+    'repr': 'reprlib',
+    'FileDialog': 'tkinter.filedialog',
+    'tkFileDialog': 'tkinter.filedialog',
+    'SimpleDialog': 'tkinter.simpledialog',
+    'tkSimpleDialog': 'tkinter.simpledialog',
+    'tkColorChooser': 'tkinter.colorchooser',
+    'tkCommonDialog': 'tkinter.commondialog',
+    'Dialog': 'tkinter.dialog',
+    'Tkdnd': 'tkinter.dnd',
+    'tkFont': 'tkinter.font',
+    'tkMessageBox': 'tkinter.messagebox',
+    'ScrolledText': 'tkinter.scrolledtext',
+    'Tkconstants': 'tkinter.constants',
+    'Tix': 'tkinter.tix',
+    'ttk': 'tkinter.ttk',
+    'Tkinter': 'tkinter',
+    'markupbase': '_markupbase',
+    '_winreg': 'winreg',
+    'thread': '_thread',
+    'dummy_thread': '_dummy_thread',
+    'dbhash': 'dbm.bsd',
+    'dumbdbm': 'dbm.dumb',
+    'dbm': 'dbm.ndbm',
+    'gdbm': 'dbm.gnu',
+    'xmlrpclib': 'xmlrpc.client',
+    'DocXMLRPCServer': 'xmlrpc.server',
+    'SimpleXMLRPCServer': 'xmlrpc.server',
+    'httplib': 'http.client',
+    'htmlentitydefs' : 'html.entities',
+    'HTMLParser' : 'html.parser',
+    'Cookie': 'http.cookies',
+    'cookielib': 'http.cookiejar',
+    'BaseHTTPServer': 'http.server',
+    'SimpleHTTPServer': 'http.server',
+    'CGIHTTPServer': 'http.server',
+    'test.test_support': 'test.support',
+    'commands': 'subprocess',
+    'UserString' : 'collections',
+    'UserList' : 'collections',
+    'urlparse' : 'urllib.parse',
+    'robotparser' : 'urllib.robotparser',
+    'whichdb': 'dbm',
+    'anydbm': 'dbm'
+}
+
+
+# This contains rename rules that are easy to handle.  We ignore the more
+# complex stuff (e.g. mapping the names in the urllib and types modules).
+# These rules should be run before import names are fixed.
+NAME_MAPPING = {
+    ('__builtin__', 'xrange'):     ('builtins', 'range'),
+    ('__builtin__', 'reduce'):     ('functools', 'reduce'),
+    ('__builtin__', 'intern'):     ('sys', 'intern'),
+    ('__builtin__', 'unichr'):     ('builtins', 'chr'),
+    ('__builtin__', 'basestring'): ('builtins', 'str'),
+    ('__builtin__', 'long'):       ('builtins', 'int'),
+    ('itertools', 'izip'):         ('builtins', 'zip'),
+    ('itertools', 'imap'):         ('builtins', 'map'),
+    ('itertools', 'ifilter'):      ('builtins', 'filter'),
+    ('itertools', 'ifilterfalse'): ('itertools', 'filterfalse'),
+}
+
+# Same, but for 3.x to 2.x
+REVERSE_IMPORT_MAPPING = dict((v, k) for (k, v) in IMPORT_MAPPING.items())
+REVERSE_NAME_MAPPING = dict((v, k) for (k, v) in NAME_MAPPING.items())
diff --git a/lib-python/3.2/_dummy_thread.py b/lib-python/3.2/_dummy_thread.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_dummy_thread.py
@@ -0,0 +1,155 @@
+"""Drop-in replacement for the thread module.
+
+Meant to be used as a brain-dead substitute so that threaded code does
+not need to be rewritten for when the thread module is not present.
+
+Suggested usage is::
+
+    try:
+        import _thread
+    except ImportError:
+        import _dummy_thread as _thread
+
+"""
+# Exports only things specified by thread documentation;
+# skipping obsolete synonyms allocate(), start_new(), exit_thread().
+__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
+           'interrupt_main', 'LockType']
+
+# A dummy value
+TIMEOUT_MAX = 2**31
+
+# NOTE: this module can be imported early in the extension building process,
+# and so top level imports of other modules should be avoided.  Instead, all
+# imports are done when needed on a function-by-function basis.  Since threads
+# are disabled, the import lock should not be an issue anyway (??).
+
+class error(Exception):
+    """Dummy implementation of _thread.error."""
+
+    def __init__(self, *args):
+        self.args = args
+
+def start_new_thread(function, args, kwargs={}):
+    """Dummy implementation of _thread.start_new_thread().
+
+    Compatibility is maintained by making sure that ``args`` is a
+    tuple and ``kwargs`` is a dictionary.  If an exception is raised
+    and it is SystemExit (which can be done by _thread.exit()) it is
+    caught and nothing is done; all other exceptions are printed out
+    by using traceback.print_exc().
+
+    If the executed function calls interrupt_main the KeyboardInterrupt will be
+    raised when the function returns.
+
+    """
+    if type(args) != type(tuple()):
+        raise TypeError("2nd arg must be a tuple")
+    if type(kwargs) != type(dict()):
+        raise TypeError("3rd arg must be a dict")
+    global _main
+    _main = False
+    try:
+        function(*args, **kwargs)
+    except SystemExit:
+        pass
+    except:
+        import traceback
+        traceback.print_exc()
+    _main = True
+    global _interrupt
+    if _interrupt:
+        _interrupt = False
+        raise KeyboardInterrupt
+
+def exit():
+    """Dummy implementation of _thread.exit()."""
+    raise SystemExit
+
+def get_ident():
+    """Dummy implementation of _thread.get_ident().
+
+    Since this module should only be used when _threadmodule is not
+    available, it is safe to assume that the current process is the
+    only thread.  Thus a constant can be safely returned.
+    """
+    return -1
+
+def allocate_lock():
+    """Dummy implementation of _thread.allocate_lock()."""
+    return LockType()
+
+def stack_size(size=None):
+    """Dummy implementation of _thread.stack_size()."""
+    if size is not None:
+        raise error("setting thread stack size not supported")
+    return 0
+
+class LockType(object):
+    """Class implementing dummy implementation of _thread.LockType.
+
+    Compatibility is maintained by maintaining self.locked_status
+    which is a boolean that stores the state of the lock.  Pickling of
+    the lock, though, should not be done since if the _thread module is
+    then used with an unpickled ``lock()`` from here problems could
+    occur from this class not having atomic methods.
+
+    """
+
+    def __init__(self):
+        self.locked_status = False
+
+    def acquire(self, waitflag=None, timeout=-1):
+        """Dummy implementation of acquire().
+
+        For blocking calls, self.locked_status is automatically set to
+        True and returned appropriately based on value of
+        ``waitflag``.  If it is non-blocking, then the value is
+        actually checked and not set if it is already acquired.  This
+        is all done so that threading.Condition's assert statements
+        aren't triggered and throw a little fit.
+
+        """
+        if waitflag is None or waitflag:
+            self.locked_status = True
+            return True
+        else:
+            if not self.locked_status:
+                self.locked_status = True
+                return True
+            else:
+                if timeout > 0:
+                    import time
+                    time.sleep(timeout)
+                return False
+
+    __enter__ = acquire
+
+    def __exit__(self, typ, val, tb):
+        self.release()
+
+    def release(self):
+        """Release the dummy lock."""
+        # XXX Perhaps shouldn't actually bother to test?  Could lead
+        #     to problems for complex, threaded code.
+        if not self.locked_status:
+            raise error
+        self.locked_status = False
+        return True
+
+    def locked(self):
+        return self.locked_status
+
+# Used to signal that interrupt_main was called in a "thread"
+_interrupt = False
+# True when not executing in a "thread"
+_main = True
+
+def interrupt_main():
+    """Set _interrupt flag to True to have start_new_thread raise
+    KeyboardInterrupt upon exiting."""
+    if _main:
+        raise KeyboardInterrupt
+    else:
+        global _interrupt
+        _interrupt = True
diff --git a/lib-python/3.2/_markupbase.py b/lib-python/3.2/_markupbase.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_markupbase.py
@@ -0,0 +1,395 @@
+"""Shared support for scanning document type declarations in HTML and XHTML.
+
+This module is used as a foundation for the html.parser module.  It has no
+documented public API and should not be used directly.
+
+"""
+
+import re
+
+_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
+_declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
+_commentclose = re.compile(r'--\s*>')
+_markedsectionclose = re.compile(r']\s*]\s*>')
+
+# An analysis of the MS-Word extensions is available at
+# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
+
+_msmarkedsectionclose = re.compile(r']\s*>')
+
+del re
+
+
+class ParserBase:
+    """Parser base class which provides some common support methods used
+    by the SGML/HTML and XHTML parsers."""
+
+    def __init__(self):
+        if self.__class__ is ParserBase:
+            raise RuntimeError(
+                "_markupbase.ParserBase must be subclassed")
+
+    def error(self, message):
+        raise NotImplementedError(
+            "subclasses of ParserBase must override error()")
+
+    def reset(self):
+        self.lineno = 1
+        self.offset = 0
+
+    def getpos(self):
+        """Return current line number and offset."""
+        return self.lineno, self.offset
+
+    # Internal -- update line number and offset.  This should be
+    # called for each piece of data exactly once, in order -- in other
+    # words the concatenation of all the input strings to this
+    # function should be exactly the entire input.
+    def updatepos(self, i, j):
+        if i >= j:
+            return j
+        rawdata = self.rawdata
+        nlines = rawdata.count("\n", i, j)
+        if nlines:
+            self.lineno = self.lineno + nlines
+            pos = rawdata.rindex("\n", i, j) # Should not fail
+            self.offset = j-(pos+1)
+        else:
+            self.offset = self.offset + j-i
+        return j
+
+    _decl_otherchars = ''
+
+    # Internal -- parse declaration (for use by subclasses).
+    def parse_declaration(self, i):
+        # This is some sort of declaration; in "HTML as
+        # deployed," this should only be the document type
+        # declaration ("<!DOCTYPE html...>").
+        # ISO 8879:1986, however, has more complex
+        # declaration syntax for elements in <!...>, including:
+        # --comment--
+        # [marked section]
+        # name in the following list: ENTITY, DOCTYPE, ELEMENT,
+        # ATTLIST, NOTATION, SHORTREF, USEMAP,
+        # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
+        rawdata = self.rawdata
+        j = i + 2
+        assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
+        if rawdata[j:j+1] == ">":
+            # the empty comment <!>
+            return j + 1
+        if rawdata[j:j+1] in ("-", ""):
+            # Start of comment followed by buffer boundary,
+            # or just a buffer boundary.
+            return -1
+        # A simple, practical version could look like: ((name|stringlit) S*) + '>'
+        n = len(rawdata)
+        if rawdata[j:j+2] == '--': #comment
+            # Locate --.*-- as the body of the comment
+            return self.parse_comment(i)
+        elif rawdata[j] == '[': #marked section
+            # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
+            # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
+            # Note that this is extended by Microsoft Office "Save as Web" function
+            # to include [if...] and [endif].
+            return self.parse_marked_section(i)
+        else: #all other declaration elements
+            decltype, j = self._scan_name(j, i)
+        if j < 0:
+            return j
+        if decltype == "doctype":
+            self._decl_otherchars = ''
+        while j < n:
+            c = rawdata[j]
+            if c == ">":
+                # end of declaration syntax
+                data = rawdata[i+2:j]
+                if decltype == "doctype":
+                    self.handle_decl(data)
+                else:
+                    # According to the HTML5 specs sections "8.2.4.44 Bogus
+                    # comment state" and "8.2.4.45 Markup declaration open
+                    # state", a comment token should be emitted.
+                    # Calling unknown_decl provides more flexibility though.
+                    self.unknown_decl(data)
+                return j + 1
+            if c in "\"'":
+                m = _declstringlit_match(rawdata, j)
+                if not m:
+                    return -1 # incomplete
+                j = m.end()
+            elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
+                name, j = self._scan_name(j, i)
+            elif c in self._decl_otherchars:
+                j = j + 1
+            elif c == "[":
+                # this could be handled in a separate doctype parser
+                if decltype == "doctype":
+                    j = self._parse_doctype_subset(j + 1, i)
+                elif decltype in {"attlist", "linktype", "link", "element"}:
+                    # must tolerate []'d groups in a content model in an element declaration
+                    # also in data attribute specifications of attlist declaration
+                    # also link type declaration subsets in linktype declarations
+                    # also link attribute specification lists in link declarations
+                    self.error("unsupported '[' char in %s declaration" % decltype)
+                else:
+                    self.error("unexpected '[' char in declaration")
+            else:
+                self.error(
+                    "unexpected %r char in declaration" % rawdata[j])
+            if j < 0:
+                return j
+        return -1 # incomplete
+
+    # Internal -- parse a marked section
+    # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
+    def parse_marked_section(self, i, report=1):
+        rawdata= self.rawdata
+        assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()"
+        sectName, j = self._scan_name( i+3, i )
+        if j < 0:
+            return j
+        if sectName in {"temp", "cdata", "ignore", "include", "rcdata"}:
+            # look for standard ]]> ending
+            match= _markedsectionclose.search(rawdata, i+3)
+        elif sectName in {"if", "else", "endif"}:
+            # look for MS Office ]> ending
+            match= _msmarkedsectionclose.search(rawdata, i+3)
+        else:
+            self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
+        if not match:
+            return -1
+        if report:
+            j = match.start(0)
+            self.unknown_decl(rawdata[i+3: j])
+        return match.end(0)
+
+    # Internal -- parse comment, return length or -1 if not terminated
+    def parse_comment(self, i, report=1):
+        rawdata = self.rawdata
+        if rawdata[i:i+4] != '<!--':
+            self.error('unexpected call to parse_comment()')
+        match = _commentclose.search(rawdata, i+4)
+        if not match:
+            return -1
+        if report:
+            j = match.start(0)
+            self.handle_comment(rawdata[i+4: j])
+        return match.end(0)
+
+    # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
+    # returning the index just past any whitespace following the trailing ']'.
+    def _parse_doctype_subset(self, i, declstartpos):
+        rawdata = self.rawdata
+        n = len(rawdata)
+        j = i
+        while j < n:
+            c = rawdata[j]
+            if c == "<":
+                s = rawdata[j:j+2]
+                if s == "<":
+                    # end of buffer; incomplete
+                    return -1
+                if s != "<!":
+                    self.updatepos(declstartpos, j + 1)
+                    self.error("unexpected char in internal subset (in %r)" % s)
+                if (j + 2) == n:
+                    # end of buffer; incomplete
+                    return -1
+                if (j + 4) > n:
+                    # end of buffer; incomplete
+                    return -1
+                if rawdata[j:j+4] == "<!--":
+                    j = self.parse_comment(j, report=0)
+                    if j < 0:
+                        return j
+                    continue
+                name, j = self._scan_name(j + 2, declstartpos)
+                if j == -1:
+                    return -1
+                if name not in {"attlist", "element", "entity", "notation"}:
+                    self.updatepos(declstartpos, j + 2)
+                    self.error(
+                        "unknown declaration %r in internal subset" % name)
+                # handle the individual names
+                meth = getattr(self, "_parse_doctype_" + name)
+                j = meth(j, declstartpos)
+                if j < 0:
+                    return j
+            elif c == "%":
+                # parameter entity reference
+                if (j + 1) == n:
+                    # end of buffer; incomplete
+                    return -1
+                s, j = self._scan_name(j + 1, declstartpos)
+                if j < 0:
+                    return j
+                if rawdata[j] == ";":
+                    j = j + 1
+            elif c == "]":
+                j = j + 1
+                while j < n and rawdata[j].isspace():
+                    j = j + 1
+                if j < n:
+                    if rawdata[j] == ">":
+                        return j
+                    self.updatepos(declstartpos, j)
+                    self.error("unexpected char after internal subset")
+                else:
+                    return -1
+            elif c.isspace():
+                j = j + 1
+            else:
+                self.updatepos(declstartpos, j)
+                self.error("unexpected char %r in internal subset" % c)
+        # end of buffer reached
+        return -1
+
+    # Internal -- scan past <!ELEMENT declarations
+    def _parse_doctype_element(self, i, declstartpos):
+        name, j = self._scan_name(i, declstartpos)
+        if j == -1:
+            return -1
+        # style content model; just skip until '>'
+        rawdata = self.rawdata
+        if '>' in rawdata[j:]:
+            return rawdata.find(">", j) + 1
+        return -1
+
+    # Internal -- scan past <!ATTLIST declarations
+    def _parse_doctype_attlist(self, i, declstartpos):
+        rawdata = self.rawdata
+        name, j = self._scan_name(i, declstartpos)
+        c = rawdata[j:j+1]
+        if c == "":
+            return -1
+        if c == ">":
+            return j + 1
+        while 1:
+            # scan a series of attribute descriptions; simplified:
+            #   name type [value] [#constraint]
+            name, j = self._scan_name(j, declstartpos)
+            if j < 0:
+                return j
+            c = rawdata[j:j+1]
+            if c == "":
+                return -1
+            if c == "(":
+                # an enumerated type; look for ')'
+                if ")" in rawdata[j:]:
+                    j = rawdata.find(")", j) + 1
+                else:
+                    return -1
+                while rawdata[j:j+1].isspace():
+                    j = j + 1
+                if not rawdata[j:]:
+                    # end of buffer, incomplete
+                    return -1
+            else:
+                name, j = self._scan_name(j, declstartpos)
+            c = rawdata[j:j+1]
+            if not c:
+                return -1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if m:
+                    j = m.end()
+                else:
+                    return -1
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+            if c == "#":
+                if rawdata[j:] == "#":
+                    # end of buffer
+                    return -1
+                name, j = self._scan_name(j + 1, declstartpos)
+                if j < 0:
+                    return j
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+            if c == '>':
+                # all done
+                return j + 1
+
+    # Internal -- scan past <!NOTATION declarations
+    def _parse_doctype_notation(self, i, declstartpos):
+        name, j = self._scan_name(i, declstartpos)
+        if j < 0:
+            return j
+        rawdata = self.rawdata
+        while 1:
+            c = rawdata[j:j+1]
+            if not c:
+                # end of buffer; incomplete
+                return -1
+            if c == '>':
+                return j + 1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if not m:
+                    return -1
+                j = m.end()
+            else:
+                name, j = self._scan_name(j, declstartpos)
+                if j < 0:
+                    return j
+
+    # Internal -- scan past <!ENTITY declarations
+    def _parse_doctype_entity(self, i, declstartpos):
+        rawdata = self.rawdata
+        if rawdata[i:i+1] == "%":
+            j = i + 1
+            while 1:
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+                if c.isspace():
+                    j = j + 1
+                else:
+                    break
+        else:
+            j = i
+        name, j = self._scan_name(j, declstartpos)
+        if j < 0:
+            return j
+        while 1:
+            c = self.rawdata[j:j+1]
+            if not c:
+                return -1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if m:
+                    j = m.end()
+                else:
+                    return -1    # incomplete
+            elif c == ">":
+                return j + 1
+            else:
+                name, j = self._scan_name(j, declstartpos)
+                if j < 0:
+                    return j
+
+    # Internal -- scan a name token and the new position and the token, or
+    # return -1 if we've reached the end of the buffer.
+    def _scan_name(self, i, declstartpos):
+        rawdata = self.rawdata
+        n = len(rawdata)
+        if i == n:
+            return None, -1
+        m = _declname_match(rawdata, i)
+        if m:
+            s = m.group()
+            name = s.strip()
+            if (i + len(s)) == n:
+                return None, -1  # end of buffer
+            return name.lower(), m.end()
+        else:
+            self.updatepos(declstartpos, i)
+            self.error("expected name token at %r"
+                       % rawdata[declstartpos:declstartpos+20])
+
+    # To be overridden -- handlers for unknown objects
+    def unknown_decl(self, data):
+        pass
diff --git a/lib-python/3.2/_pyio.py b/lib-python/3.2/_pyio.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_pyio.py
@@ -0,0 +1,2033 @@
+"""
+Python implementation of the io module.
+"""
+
+import os
+import abc
+import codecs
+import warnings
+import errno
+# Import _thread instead of threading to reduce startup cost
+try:
+    from _thread import allocate_lock as Lock
+except ImportError:
+    from _dummy_thread import allocate_lock as Lock
+
+import io
+from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
+from errno import EINTR
+
+# open() uses st_blksize whenever we can
+DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
+
+# NOTE: Base classes defined here are registered with the "official" ABCs
+# defined in io.py. We don't use real inheritance though, because we don't
+# want to inherit the C implementations.
+
+
+class BlockingIOError(IOError):
+
+    """Exception raised when I/O would block on a non-blocking I/O stream."""
+
+    def __init__(self, errno, strerror, characters_written=0):
+        super().__init__(errno, strerror)
+        if not isinstance(characters_written, int):
+            raise TypeError("characters_written must be a integer")
+        self.characters_written = characters_written
+
+
+def open(file, mode="r", buffering=-1, encoding=None, errors=None,
+         newline=None, closefd=True):
+
+    r"""Open file and return a stream.  Raise IOError upon failure.
+
+    file is either a text or byte string giving the name (and the path
+    if the file isn't in the current working directory) of the file to
+    be opened or an integer file descriptor of the file to be
+    wrapped. (If a file descriptor is given, it is closed when the
+    returned I/O object is closed, unless closefd is set to False.)
+
+    mode is an optional string that specifies the mode in which the file
+    is opened. It defaults to 'r' which means open for reading in text
+    mode.  Other common values are 'w' for writing (truncating the file if
+    it already exists), and 'a' for appending (which on some Unix systems,
+    means that all writes append to the end of the file regardless of the
+    current seek position). In text mode, if encoding is not specified the
+    encoding used is platform dependent. (For reading and writing raw
+    bytes use binary mode and leave encoding unspecified.) The available
+    modes are:
+
+    ========= ===============================================================
+    Character Meaning
+    --------- ---------------------------------------------------------------
+    'r'       open for reading (default)
+    'w'       open for writing, truncating the file first
+    'a'       open for writing, appending to the end of the file if it exists
+    'b'       binary mode
+    't'       text mode (default)
+    '+'       open a disk file for updating (reading and writing)
+    'U'       universal newline mode (for backwards compatibility; unneeded
+              for new code)
+    ========= ===============================================================
+
+    The default mode is 'rt' (open for reading text). For binary random
+    access, the mode 'w+b' opens and truncates the file to 0 bytes, while
+    'r+b' opens the file without truncation.
+
+    Python distinguishes between files opened in binary and text modes,
+    even when the underlying operating system doesn't. Files opened in
+    binary mode (appending 'b' to the mode argument) return contents as
+    bytes objects without any decoding. In text mode (the default, or when
+    't' is appended to the mode argument), the contents of the file are
+    returned as strings, the bytes having been first decoded using a
+    platform-dependent encoding or using the specified encoding if given.
+
+    buffering is an optional integer used to set the buffering policy.
+    Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
+    line buffering (only usable in text mode), and an integer > 1 to indicate
+    the size of a fixed-size chunk buffer.  When no buffering argument is
+    given, the default buffering policy works as follows:
+
+    * Binary files are buffered in fixed-size chunks; the size of the buffer
+      is chosen using a heuristic trying to determine the underlying device's
+      "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
+      On many systems, the buffer will typically be 4096 or 8192 bytes long.
+
+    * "Interactive" text files (files for which isatty() returns True)
+      use line buffering.  Other text files use the policy described above
+      for binary files.
+
+    encoding is the str name of the encoding used to decode or encode the
+    file. This should only be used in text mode. The default encoding is
+    platform dependent, but any encoding supported by Python can be
+    passed.  See the codecs module for the list of supported encodings.
+
+    errors is an optional string that specifies how encoding errors are to
+    be handled---this argument should not be used in binary mode. Pass
+    'strict' to raise a ValueError exception if there is an encoding error
+    (the default of None has the same effect), or pass 'ignore' to ignore
+    errors. (Note that ignoring encoding errors can lead to data loss.)
+    See the documentation for codecs.register for a list of the permitted
+    encoding error strings.
+
+    newline is a string controlling how universal newlines works (it only
+    applies to text mode). It can be None, '', '\n', '\r', and '\r\n'.  It works
+    as follows:
+
+    * On input, if newline is None, universal newlines mode is
+      enabled. Lines in the input can end in '\n', '\r', or '\r\n', and
+      these are translated into '\n' before being returned to the
+      caller. If it is '', universal newline mode is enabled, but line
+      endings are returned to the caller untranslated. If it has any of
+      the other legal values, input lines are only terminated by the given
+      string, and the line ending is returned to the caller untranslated.
+
+    * On output, if newline is None, any '\n' characters written are
+      translated to the system default line separator, os.linesep. If
+      newline is '', no translation takes place. If newline is any of the
+      other legal values, any '\n' characters written are translated to
+      the given string.
+
+    closedfd is a bool. If closefd is False, the underlying file descriptor will
+    be kept open when the file is closed. This does not work when a file name is
+    given and must be True in that case.
+
+    open() returns a file object whose type depends on the mode, and
+    through which the standard file operations such as reading and writing
+    are performed. When open() is used to open a file in a text mode ('w',
+    'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open
+    a file in a binary mode, the returned class varies: in read binary
+    mode, it returns a BufferedReader; in write binary and append binary
+    modes, it returns a BufferedWriter, and in read/write mode, it returns
+    a BufferedRandom.
+
+    It is also possible to use a string or bytearray as a file for both
+    reading and writing. For strings StringIO can be used like a file
+    opened in a text mode, and for bytes a BytesIO can be used like a file
+    opened in a binary mode.
+    """
+    if not isinstance(file, (str, bytes, int)):
+        raise TypeError("invalid file: %r" % file)
+    if not isinstance(mode, str):
+        raise TypeError("invalid mode: %r" % mode)
+    if not isinstance(buffering, int):
+        raise TypeError("invalid buffering: %r" % buffering)
+    if encoding is not None and not isinstance(encoding, str):
+        raise TypeError("invalid encoding: %r" % encoding)
+    if errors is not None and not isinstance(errors, str):
+        raise TypeError("invalid errors: %r" % errors)
+    modes = set(mode)
+    if modes - set("arwb+tU") or len(mode) > len(modes):
+        raise ValueError("invalid mode: %r" % mode)
+    reading = "r" in modes
+    writing = "w" in modes
+    appending = "a" in modes
+    updating = "+" in modes
+    text = "t" in modes
+    binary = "b" in modes
+    if "U" in modes:
+        if writing or appending:
+            raise ValueError("can't use U and writing mode at once")
+        reading = True
+    if text and binary:
+        raise ValueError("can't have text and binary mode at once")
+    if reading + writing + appending > 1:
+        raise ValueError("can't have read/write/append mode at once")
+    if not (reading or writing or appending):
+        raise ValueError("must have exactly one of read/write/append mode")
+    if binary and encoding is not None:
+        raise ValueError("binary mode doesn't take an encoding argument")
+    if binary and errors is not None:
+        raise ValueError("binary mode doesn't take an errors argument")
+    if binary and newline is not None:
+        raise ValueError("binary mode doesn't take a newline argument")
+    raw = FileIO(file,
+                 (reading and "r" or "") +
+                 (writing and "w" or "") +
+                 (appending and "a" or "") +
+                 (updating and "+" or ""),
+                 closefd)
+    line_buffering = False
+    if buffering == 1 or buffering < 0 and raw.isatty():
+        buffering = -1
+        line_buffering = True
+    if buffering < 0:
+        buffering = DEFAULT_BUFFER_SIZE
+        try:
+            bs = os.fstat(raw.fileno()).st_blksize
+        except (os.error, AttributeError):
+            pass
+        else:
+            if bs > 1:
+                buffering = bs
+    if buffering < 0:
+        raise ValueError("invalid buffering size")
+    if buffering == 0:
+        if binary:
+            return raw
+        raise ValueError("can't have unbuffered text I/O")
+    if updating:
+        buffer = BufferedRandom(raw, buffering)
+    elif writing or appending:
+        buffer = BufferedWriter(raw, buffering)
+    elif reading:
+        buffer = BufferedReader(raw, buffering)
+    else:
+        raise ValueError("unknown mode: %r" % mode)
+    if binary:
+        return buffer
+    text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
+    text.mode = mode
+    return text
+
+
+class DocDescriptor:
+    """Helper for builtins.open.__doc__
+    """
+    def __get__(self, obj, typ):
+        return (
+            "open(file, mode='r', buffering=-1, encoding=None, "
+                 "errors=None, newline=None, closefd=True)\n\n" +
+            open.__doc__)
+
+class OpenWrapper:
+    """Wrapper for builtins.open
+
+    Trick so that open won't become a bound method when stored
+    as a class variable (as dbm.dumb does).
+
+    See initstdio() in Python/pythonrun.c.
+    """
+    __doc__ = DocDescriptor()
+
+    def __new__(cls, *args, **kwargs):
+        return open(*args, **kwargs)
+
+
+# In normal operation, both `UnsupportedOperation`s should be bound to the
+# same object.
+try:
+    UnsupportedOperation = io.UnsupportedOperation
+except AttributeError:
+    class UnsupportedOperation(ValueError, IOError):
+        pass
+
+
+class IOBase(metaclass=abc.ABCMeta):
+
+    """The abstract base class for all I/O classes, acting on streams of
+    bytes. There is no public constructor.
+
+    This class provides dummy implementations for many methods that
+    derived classes can override selectively; the default implementations
+    represent a file that cannot be read, written or seeked.
+
+    Even though IOBase does not declare read, readinto, or write because
+    their signatures will vary, implementations and clients should
+    consider those methods part of the interface. Also, implementations
+    may raise UnsupportedOperation when operations they do not support are
+    called.
+
+    The basic type used for binary data read from or written to a file is
+    bytes. bytearrays are accepted too, and in some cases (such as
+    readinto) needed. Text I/O classes work with str data.
+
+    Note that calling any method (even inquiries) on a closed stream is
+    undefined. Implementations may raise IOError in this case.
+
+    IOBase (and its subclasses) support the iterator protocol, meaning
+    that an IOBase object can be iterated over yielding the lines in a
+    stream.
+
+    IOBase also supports the :keyword:`with` statement. In this example,
+    fp is closed after the suite of the with statement is complete:
+
+    with open('spam.txt', 'r') as fp:
+        fp.write('Spam and eggs!')
+    """
+
+    ### Internal ###
+
+    def _unsupported(self, name):
+        """Internal: raise an IOError exception for unsupported operations."""
+        raise UnsupportedOperation("%s.%s() not supported" %
+                                   (self.__class__.__name__, name))
+
+    ### Positioning ###
+
+    def seek(self, pos, whence=0):
+        """Change stream position.
+
+        Change the stream position to byte offset offset. offset is
+        interpreted relative to the position indicated by whence.  Values
+        for whence are ints:
+
+        * 0 -- start of stream (the default); offset should be zero or positive
+        * 1 -- current stream position; offset may be negative
+        * 2 -- end of stream; offset is usually negative
+
+        Return an int indicating the new absolute position.
+        """
+        self._unsupported("seek")
+
+    def tell(self):
+        """Return an int indicating the current stream position."""
+        return self.seek(0, 1)
+
+    def truncate(self, pos=None):
+        """Truncate file to size bytes.
+
+        Size defaults to the current IO position as reported by tell().  Return
+        the new size.
+        """
+        self._unsupported("truncate")
+
+    ### Flush and close ###
+
+    def flush(self):
+        """Flush write buffers, if applicable.
+
+        This is not implemented for read-only and non-blocking streams.
+        """
+        self._checkClosed()
+        # XXX Should this return the number of bytes written???
+
+    __closed = False
+
+    def close(self):
+        """Flush and close the IO object.
+
+        This method has no effect if the file is already closed.
+        """
+        if not self.__closed:
+            self.flush()
+            self.__closed = True
+
+    def __del__(self):
+        """Destructor.  Calls close()."""
+        # The try/except block is in case this is called at program
+        # exit time, when it's possible that globals have already been
+        # deleted, and then the close() call might fail.  Since
+        # there's nothing we can do about such failures and they annoy
+        # the end users, we suppress the traceback.
+        try:
+            self.close()
+        except:
+            pass
+
+    ### Inquiries ###
+
+    def seekable(self):
+        """Return a bool indicating whether object supports random access.
+
+        If False, seek(), tell() and truncate() will raise UnsupportedOperation.
+        This method may need to do a test seek().
+        """
+        return False
+
+    def _checkSeekable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not seekable
+        """
+        if not self.seekable():
+            raise UnsupportedOperation("File or stream is not seekable."
+                                       if msg is None else msg)
+
+    def readable(self):
+        """Return a bool indicating whether object was opened for reading.
+
+        If False, read() will raise UnsupportedOperation.
+        """
+        return False
+
+    def _checkReadable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not readable
+        """
+        if not self.readable():
+            raise UnsupportedOperation("File or stream is not readable."
+                                       if msg is None else msg)
+
+    def writable(self):
+        """Return a bool indicating whether object was opened for writing.
+
+        If False, write() and truncate() will raise UnsupportedOperation.
+        """
+        return False
+
+    def _checkWritable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not writable
+        """
+        if not self.writable():
+            raise UnsupportedOperation("File or stream is not writable."
+                                       if msg is None else msg)
+
+    @property
+    def closed(self):
+        """closed: bool.  True iff the file has been closed.
+
+        For backwards compatibility, this is a property, not a predicate.
+        """
+        return self.__closed
+
+    def _checkClosed(self, msg=None):
+        """Internal: raise an ValueError if file is closed
+        """
+        if self.closed:
+            raise ValueError("I/O operation on closed file."
+                             if msg is None else msg)
+
+    ### Context manager ###
+
+    def __enter__(self):  # That's a forward reference
+        """Context management protocol.  Returns self (an instance of IOBase)."""
+        self._checkClosed()
+        return self
+
+    def __exit__(self, *args):
+        """Context management protocol.  Calls close()"""
+        self.close()
+
+    ### Lower-level APIs ###
+
+    # XXX Should these be present even if unimplemented?
+
+    def fileno(self):
+        """Returns underlying file descriptor (an int) if one exists.
+
+        An IOError is raised if the IO object does not use a file descriptor.
+        """
+        self._unsupported("fileno")
+
+    def isatty(self):
+        """Return a bool indicating whether this is an 'interactive' stream.
+
+        Return False if it can't be determined.
+        """
+        self._checkClosed()
+        return False
+
+    ### Readline[s] and writelines ###
+
+    def readline(self, limit=-1):
+        r"""Read and return a line of bytes from the stream.
+
+        If limit is specified, at most limit bytes will be read.
+        Limit should be an int.
+
+        The line terminator is always b'\n' for binary files; for text
+        files, the newlines argument to open can be used to select the line
+        terminator(s) recognized.
+        """
+        # For backwards compatibility, a (slowish) readline().
+        if hasattr(self, "peek"):
+            def nreadahead():
+                readahead = self.peek(1)
+                if not readahead:
+                    return 1
+                n = (readahead.find(b"\n") + 1) or len(readahead)
+                if limit >= 0:
+                    n = min(n, limit)
+                return n
+        else:
+            def nreadahead():
+                return 1
+        if limit is None:
+            limit = -1
+        elif not isinstance(limit, int):
+            raise TypeError("limit must be an integer")
+        res = bytearray()
+        while limit < 0 or len(res) < limit:
+            b = self.read(nreadahead())
+            if not b:
+                break
+            res += b
+            if res.endswith(b"\n"):
+                break
+        return bytes(res)
+
+    def __iter__(self):
+        self._checkClosed()
+        return self
+
+    def __next__(self):
+        line = self.readline()
+        if not line:
+            raise StopIteration
+        return line
+
+    def readlines(self, hint=None):
+        """Return a list of lines from the stream.
+
+        hint can be specified to control the number of lines read: no more
+        lines will be read if the total size (in bytes/characters) of all
+        lines so far exceeds hint.
+        """
+        if hint is None or hint <= 0:
+            return list(self)
+        n = 0
+        lines = []
+        for line in self:
+            lines.append(line)
+            n += len(line)
+            if n >= hint:
+                break
+        return lines
+
+    def writelines(self, lines):
+        self._checkClosed()
+        for line in lines:
+            self.write(line)
+
+io.IOBase.register(IOBase)
+
+
+class RawIOBase(IOBase):
+
+    """Base class for raw binary I/O."""
+
+    # The read() method is implemented by calling readinto(); derived
+    # classes that want to support read() only need to implement
+    # readinto() as a primitive operation.  In general, readinto() can be
+    # more efficient than read().
+
+    # (It would be tempting to also provide an implementation of
+    # readinto() in terms of read(), in case the latter is a more suitable
+    # primitive operation, but that would lead to nasty recursion in case
+    # a subclass doesn't implement either.)
+
+    def read(self, n=-1):
+        """Read and return up to n bytes, where n is an int.
+
+        Returns an empty bytes object on EOF, or None if the object is
+        set not to block and has no data to read.
+        """
+        if n is None:
+            n = -1
+        if n < 0:
+            return self.readall()
+        b = bytearray(n.__index__())
+        n = self.readinto(b)
+        if n is None:
+            return None
+        del b[n:]
+        return bytes(b)
+
+    def readall(self):
+        """Read until EOF, using multiple read() call."""
+        res = bytearray()
+        while True:
+            data = self.read(DEFAULT_BUFFER_SIZE)
+            if not data:
+                break
+            res += data
+        if res:
+            return bytes(res)
+        else:
+            # b'' or None
+            return data
+
+    def readinto(self, b):
+        """Read up to len(b) bytes into bytearray b.
+
+        Returns an int representing the number of bytes read (0 for EOF), or
+        None if the object is set not to block and has no data to read.
+        """
+        self._unsupported("readinto")
+
+    def write(self, b):
+        """Write the given buffer to the IO stream.
+
+        Returns the number of bytes written, which may be less than len(b).
+        """
+        self._unsupported("write")
+
+io.RawIOBase.register(RawIOBase)
+from _io import FileIO
+RawIOBase.register(FileIO)
+
+
+class BufferedIOBase(IOBase):
+
+    """Base class for buffered IO objects.
+
+    The main difference with RawIOBase is that the read() method
+    supports omitting the size argument, and does not have a default
+    implementation that defers to readinto().
+
+    In addition, read(), readinto() and write() may raise
+    BlockingIOError if the underlying raw stream is in non-blocking
+    mode and not ready; unlike their raw counterparts, they will never
+    return None.
+
+    A typical implementation should not inherit from a RawIOBase
+    implementation, but wrap one.
+    """
+
+    def read(self, n=None):
+        """Read and return up to n bytes, where n is an int.
+
+        If the argument is omitted, None, or negative, reads and
+        returns all data until EOF.
+
+        If the argument is positive, and the underlying raw stream is
+        not 'interactive', multiple raw reads may be issued to satisfy
+        the byte count (unless EOF is reached first).  But for
+        interactive raw streams (XXX and for pipes?), at most one raw
+        read will be issued, and a short result does not imply that
+        EOF is imminent.
+
+        Returns an empty bytes array on EOF.
+
+        Raises BlockingIOError if the underlying raw stream has no
+        data at the moment.
+        """
+        self._unsupported("read")
+
+    def read1(self, n=None):
+        """Read up to n bytes with at most one read() system call,
+        where n is an int.
+        """
+        self._unsupported("read1")
+
+    def readinto(self, b):
+        """Read up to len(b) bytes into bytearray b.
+
+        Like read(), this may issue multiple reads to the underlying raw
+        stream, unless the latter is 'interactive'.
+
+        Returns an int representing the number of bytes read (0 for EOF).
+
+        Raises BlockingIOError if the underlying raw stream has no
+        data at the moment.
+        """
+        # XXX This ought to work with anything that supports the buffer API
+        data = self.read(len(b))
+        n = len(data)
+        try:
+            b[:n] = data
+        except TypeError as err:
+            import array
+            if not isinstance(b, array.array):
+                raise err
+            b[:n] = array.array('b', data)
+        return n
+
+    def write(self, b):
+        """Write the given bytes buffer to the IO stream.
+
+        Return the number of bytes written, which is never less than
+        len(b).
+
+        Raises BlockingIOError if the buffer is full and the
+        underlying raw stream cannot accept more data at the moment.
+        """
+        self._unsupported("write")
+
+    def detach(self):
+        """
+        Separate the underlying raw stream from the buffer and return it.
+
+        After the raw stream has been detached, the buffer is in an unusable
+        state.
+        """
+        self._unsupported("detach")
+
+io.BufferedIOBase.register(BufferedIOBase)
+
+
+class _BufferedIOMixin(BufferedIOBase):
+
+    """A mixin implementation of BufferedIOBase with an underlying raw stream.
+
+    This passes most requests on to the underlying raw stream.  It
+    does *not* provide implementations of read(), readinto() or
+    write().
+    """
+
+    def __init__(self, raw):
+        self._raw = raw
+
+    ### Positioning ###
+
+    def seek(self, pos, whence=0):
+        new_position = self.raw.seek(pos, whence)
+        if new_position < 0:
+            raise IOError("seek() returned an invalid position")
+        return new_position
+
+    def tell(self):
+        pos = self.raw.tell()
+        if pos < 0:
+            raise IOError("tell() returned an invalid position")
+        return pos
+
+    def truncate(self, pos=None):
+        # Flush the stream.  We're mixing buffered I/O with lower-level I/O,
+        # and a flush may be necessary to synch both views of the current
+        # file state.
+        self.flush()
+
+        if pos is None:
+            pos = self.tell()
+        # XXX: Should seek() be used, instead of passing the position
+        # XXX  directly to truncate?
+        return self.raw.truncate(pos)
+
+    ### Flush and close ###
+
+    def flush(self):
+        if self.closed:
+            raise ValueError("flush of closed file")
+        self.raw.flush()
+
+    def close(self):
+        if self.raw is not None and not self.closed:
+            try:
+                # may raise BlockingIOError or BrokenPipeError etc
+                self.flush()
+            finally:
+                self.raw.close()
+
+    def detach(self):
+        if self.raw is None:
+            raise ValueError("raw stream already detached")
+        self.flush()
+        raw = self._raw
+        self._raw = None
+        return raw
+
+    ### Inquiries ###
+
+    def seekable(self):
+        return self.raw.seekable()
+
+    def readable(self):
+        return self.raw.readable()
+
+    def writable(self):
+        return self.raw.writable()
+
+    @property
+    def raw(self):
+        return self._raw
+
+    @property
+    def closed(self):
+        return self.raw.closed
+
+    @property
+    def name(self):
+        return self.raw.name
+
+    @property
+    def mode(self):
+        return self.raw.mode
+
+    def __getstate__(self):
+        raise TypeError("can not serialize a '{0}' object"
+                        .format(self.__class__.__name__))
+
+    def __repr__(self):
+        clsname = self.__class__.__name__
+        try:
+            name = self.name
+        except AttributeError:
+            return "<_pyio.{0}>".format(clsname)
+        else:
+            return "<_pyio.{0} name={1!r}>".format(clsname, name)
+
+    ### Lower-level APIs ###
+
+    def fileno(self):
+        return self.raw.fileno()
+
+    def isatty(self):
+        return self.raw.isatty()
+
+
+class BytesIO(BufferedIOBase):
+
+    """Buffered I/O implementation using an in-memory bytes buffer."""
+
+    def __init__(self, initial_bytes=None):
+        buf = bytearray()
+        if initial_bytes is not None:
+            buf += initial_bytes
+        self._buffer = buf
+        self._pos = 0
+
+    def __getstate__(self):
+        if self.closed:
+            raise ValueError("__getstate__ on closed file")
+        return self.__dict__.copy()
+
+    def getvalue(self):
+        """Return the bytes value (contents) of the buffer
+        """
+        if self.closed:
+            raise ValueError("getvalue on closed file")
+        return bytes(self._buffer)
+
+    def getbuffer(self):
+        """Return a readable and writable view of the buffer.
+        """
+        return memoryview(self._buffer)
+
+    def read(self, n=None):
+        if self.closed:
+            raise ValueError("read from closed file")
+        if n is None:
+            n = -1
+        if n < 0:
+            n = len(self._buffer)
+        if len(self._buffer) <= self._pos:
+            return b""
+        newpos = min(len(self._buffer), self._pos + n)
+        b = self._buffer[self._pos : newpos]
+        self._pos = newpos
+        return bytes(b)
+
+    def read1(self, n):
+        """This is the same as read.
+        """
+        return self.read(n)
+
+    def write(self, b):
+        if self.closed:
+            raise ValueError("write to closed file")
+        if isinstance(b, str):
+            raise TypeError("can't write str to binary stream")
+        n = len(b)
+        if n == 0:
+            return 0
+        pos = self._pos
+        if pos > len(self._buffer):
+            # Inserts null bytes between the current end of the file
+            # and the new write position.
+            padding = b'\x00' * (pos - len(self._buffer))
+            self._buffer += padding
+        self._buffer[pos:pos + n] = b
+        self._pos += n
+        return n
+
+    def seek(self, pos, whence=0):
+        if self.closed:
+            raise ValueError("seek on closed file")
+        try:
+            pos.__index__
+        except AttributeError as err:
+            raise TypeError("an integer is required") from err
+        if whence == 0:
+            if pos < 0:
+                raise ValueError("negative seek position %r" % (pos,))
+            self._pos = pos
+        elif whence == 1:
+            self._pos = max(0, self._pos + pos)
+        elif whence == 2:
+            self._pos = max(0, len(self._buffer) + pos)
+        else:
+            raise ValueError("invalid whence value")
+        return self._pos
+
+    def tell(self):
+        if self.closed:
+            raise ValueError("tell on closed file")
+        return self._pos
+
+    def truncate(self, pos=None):
+        if self.closed:
+            raise ValueError("truncate on closed file")
+        if pos is None:
+            pos = self._pos
+        else:
+            try:
+                pos.__index__
+            except AttributeError as err:
+                raise TypeError("an integer is required") from err
+            if pos < 0:
+                raise ValueError("negative truncate position %r" % (pos,))
+        del self._buffer[pos:]
+        return pos
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return True
+
+    def seekable(self):
+        return True
+
+
+class BufferedReader(_BufferedIOMixin):
+
+    """BufferedReader(raw[, buffer_size])
+
+    A buffer for a readable, sequential BaseRawIO object.
+
+    The constructor creates a BufferedReader for the given readable raw
+    stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE
+    is used.
+    """
+
+    def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
+        """Create a new buffered reader using the given readable raw IO object.
+        """
+        if not raw.readable():
+            raise IOError('"raw" argument must be readable.')
+
+        _BufferedIOMixin.__init__(self, raw)
+        if buffer_size <= 0:
+            raise ValueError("invalid buffer size")
+        self.buffer_size = buffer_size
+        self._reset_read_buf()
+        self._read_lock = Lock()
+
+    def _reset_read_buf(self):
+        self._read_buf = b""
+        self._read_pos = 0
+
+    def read(self, n=None):
+        """Read n bytes.
+
+        Returns exactly n bytes of data unless the underlying raw IO
+        stream reaches EOF or if the call would block in non-blocking
+        mode. If n is negative, read until EOF or until read() would
+        block.
+        """
+        if n is not None and n < -1:
+            raise ValueError("invalid number of bytes to read")
+        with self._read_lock:
+            return self._read_unlocked(n)
+
+    def _read_unlocked(self, n=None):
+        nodata_val = b""
+        empty_values = (b"", None)
+        buf = self._read_buf
+        pos = self._read_pos
+
+        # Special case for when the number of bytes to read is unspecified.
+        if n is None or n == -1:
+            self._reset_read_buf()
+            chunks = [buf[pos:]]  # Strip the consumed bytes.
+            current_size = 0
+            while True:
+                # Read until EOF or until read() would block.
+                try:
+                    chunk = self.raw.read()
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
+                if chunk in empty_values:
+                    nodata_val = chunk
+                    break
+                current_size += len(chunk)
+                chunks.append(chunk)
+            return b"".join(chunks) or nodata_val
+
+        # The number of bytes to read is specified, return at most n bytes.
+        avail = len(buf) - pos  # Length of the available buffered data.
+        if n <= avail:
+            # Fast path: the data to read is fully buffered.
+            self._read_pos += n
+            return buf[pos:pos+n]
+        # Slow path: read from the stream until enough bytes are read,
+        # or until an EOF occurs or until read() would block.
+        chunks = [buf[pos:]]
+        wanted = max(self.buffer_size, n)
+        while avail < n:
+            try:
+                chunk = self.raw.read(wanted)
+            except IOError as e:
+                if e.errno != EINTR:
+                    raise
+                continue
+            if chunk in empty_values:
+                nodata_val = chunk
+                break
+            avail += len(chunk)
+            chunks.append(chunk)
+        # n is more then avail only when an EOF occurred or when
+        # read() would have blocked.
+        n = min(n, avail)
+        out = b"".join(chunks)
+        self._read_buf = out[n:]  # Save the extra data in the buffer.
+        self._read_pos = 0
+        return out[:n] if out else nodata_val
+
+    def peek(self, n=0):
+        """Returns buffered bytes without advancing the position.
+
+        The argument indicates a desired minimal number of bytes; we
+        do at most one raw read to satisfy it.  We never return more
+        than self.buffer_size.
+        """
+        with self._read_lock:
+            return self._peek_unlocked(n)
+
+    def _peek_unlocked(self, n=0):
+        want = min(n, self.buffer_size)
+        have = len(self._read_buf) - self._read_pos
+        if have < want or have <= 0:
+            to_read = self.buffer_size - have
+            while True:
+                try:
+                    current = self.raw.read(to_read)
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
+                break
+            if current:
+                self._read_buf = self._read_buf[self._read_pos:] + current
+                self._read_pos = 0
+        return self._read_buf[self._read_pos:]
+
+    def read1(self, n):
+        """Reads up to n bytes, with at most one read() system call."""
+        # Returns up to n bytes.  If at least one byte is buffered, we
+        # only return buffered bytes.  Otherwise, we do one raw read.
+        if n < 0:
+            raise ValueError("number of bytes to read must be positive")
+        if n == 0:
+            return b""
+        with self._read_lock:
+            self._peek_unlocked(1)
+            return self._read_unlocked(
+                min(n, len(self._read_buf) - self._read_pos))
+
+    def tell(self):
+        return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence value")
+        with self._read_lock:
+            if whence == 1:
+                pos -= len(self._read_buf) - self._read_pos
+            pos = _BufferedIOMixin.seek(self, pos, whence)
+            self._reset_read_buf()
+            return pos
+
+class BufferedWriter(_BufferedIOMixin):
+
+    """A buffer for a writeable sequential RawIO object.
+
+    The constructor creates a BufferedWriter for the given writeable raw
+    stream. If the buffer_size is not given, it defaults to
+    DEFAULT_BUFFER_SIZE.
+    """
+
+    _warning_stack_offset = 2
+
+    def __init__(self, raw,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        if not raw.writable():
+            raise IOError('"raw" argument must be writable.')
+
+        _BufferedIOMixin.__init__(self, raw)
+        if buffer_size <= 0:
+            raise ValueError("invalid buffer size")
+        if max_buffer_size is not None:
+            warnings.warn("max_buffer_size is deprecated", DeprecationWarning,
+                          self._warning_stack_offset)
+        self.buffer_size = buffer_size
+        self._write_buf = bytearray()
+        self._write_lock = Lock()
+
+    def write(self, b):
+        if self.closed:
+            raise ValueError("write to closed file")
+        if isinstance(b, str):
+            raise TypeError("can't write str to binary stream")
+        with self._write_lock:
+            # XXX we can implement some more tricks to try and avoid
+            # partial writes
+            if len(self._write_buf) > self.buffer_size:
+                # We're full, so let's pre-flush the buffer.  (This may
+                # raise BlockingIOError with characters_written == 0.)
+                self._flush_unlocked()
+            before = len(self._write_buf)
+            self._write_buf.extend(b)
+            written = len(self._write_buf) - before
+            if len(self._write_buf) > self.buffer_size:
+                try:
+                    self._flush_unlocked()
+                except BlockingIOError as e:
+                    if len(self._write_buf) > self.buffer_size:
+                        # We've hit the buffer_size. We have to accept a partial
+                        # write and cut back our buffer.
+                        overage = len(self._write_buf) - self.buffer_size
+                        written -= overage
+                        self._write_buf = self._write_buf[:self.buffer_size]
+                        raise BlockingIOError(e.errno, e.strerror, written)
+            return written
+
+    def truncate(self, pos=None):
+        with self._write_lock:
+            self._flush_unlocked()
+            if pos is None:
+                pos = self.raw.tell()
+            return self.raw.truncate(pos)
+
+    def flush(self):
+        with self._write_lock:
+            self._flush_unlocked()
+
+    def _flush_unlocked(self):
+        if self.closed:
+            raise ValueError("flush of closed file")
+        while self._write_buf:
+            try:
+                n = self.raw.write(self._write_buf)
+            except BlockingIOError:
+                raise RuntimeError("self.raw should implement RawIOBase: it "
+                                   "should not raise BlockingIOError")
+            except IOError as e:
+                if e.errno != EINTR:
+                    raise
+                continue
+            if n is None:
+                raise BlockingIOError(
+                    errno.EAGAIN,
+                    "write could not complete without blocking", 0)
+            if n > len(self._write_buf) or n < 0:
+                raise IOError("write() returned incorrect number of bytes")
+            del self._write_buf[:n]
+
+    def tell(self):
+        return _BufferedIOMixin.tell(self) + len(self._write_buf)
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence")
+        with self._write_lock:
+            self._flush_unlocked()
+            return _BufferedIOMixin.seek(self, pos, whence)
+
+
+class BufferedRWPair(BufferedIOBase):
+
+    """A buffered reader and writer object together.
+
+    A buffered reader object and buffered writer object put together to
+    form a sequential IO object that can read and write. This is typically
+    used with a socket or two-way pipe.
+
+    reader and writer are RawIOBase objects that are readable and
+    writeable respectively. If the buffer_size is omitted it defaults to
+    DEFAULT_BUFFER_SIZE.
+    """
+
+    # XXX The usefulness of this (compared to having two separate IO
+    # objects) is questionable.
+
+    def __init__(self, reader, writer,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        """Constructor.
+
+        The arguments are two RawIO instances.
+        """
+        if max_buffer_size is not None:
+            warnings.warn("max_buffer_size is deprecated", DeprecationWarning, 2)
+
+        if not reader.readable():
+            raise IOError('"reader" argument must be readable.')
+
+        if not writer.writable():
+            raise IOError('"writer" argument must be writable.')
+
+        self.reader = BufferedReader(reader, buffer_size)
+        self.writer = BufferedWriter(writer, buffer_size)
+
+    def read(self, n=None):
+        if n is None:
+            n = -1
+        return self.reader.read(n)
+
+    def readinto(self, b):
+        return self.reader.readinto(b)
+
+    def write(self, b):
+        return self.writer.write(b)
+
+    def peek(self, n=0):
+        return self.reader.peek(n)
+
+    def read1(self, n):
+        return self.reader.read1(n)
+
+    def readable(self):
+        return self.reader.readable()
+
+    def writable(self):
+        return self.writer.writable()
+
+    def flush(self):
+        return self.writer.flush()
+
+    def close(self):
+        self.writer.close()
+        self.reader.close()
+
+    def isatty(self):
+        return self.reader.isatty() or self.writer.isatty()
+
+    @property
+    def closed(self):
+        return self.writer.closed
+
+
+class BufferedRandom(BufferedWriter, BufferedReader):
+
+    """A buffered interface to random access streams.
+
+    The constructor creates a reader and writer for a seekable stream,
+    raw, given in the first argument. If the buffer_size is omitted it
+    defaults to DEFAULT_BUFFER_SIZE.
+    """
+
+    _warning_stack_offset = 3
+
+    def __init__(self, raw,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        raw._checkSeekable()
+        BufferedReader.__init__(self, raw, buffer_size)
+        BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size)
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence")
+        self.flush()
+        if self._read_buf:
+            # Undo read ahead.
+            with self._read_lock:
+                self.raw.seek(self._read_pos - len(self._read_buf), 1)
+        # First do the raw seek, then empty the read buffer, so that
+        # if the raw seek fails, we don't lose buffered data forever.
+        pos = self.raw.seek(pos, whence)
+        with self._read_lock:
+            self._reset_read_buf()
+        if pos < 0:
+            raise IOError("seek() returned invalid position")
+        return pos
+
+    def tell(self):
+        if self._write_buf:
+            return BufferedWriter.tell(self)
+        else:
+            return BufferedReader.tell(self)
+
+    def truncate(self, pos=None):
+        if pos is None:
+            pos = self.tell()
+        # Use seek to flush the read buffer.
+        return BufferedWriter.truncate(self, pos)
+
+    def read(self, n=None):
+        if n is None:
+            n = -1
+        self.flush()
+        return BufferedReader.read(self, n)
+
+    def readinto(self, b):
+        self.flush()
+        return BufferedReader.readinto(self, b)
+
+    def peek(self, n=0):
+        self.flush()
+        return BufferedReader.peek(self, n)
+
+    def read1(self, n):
+        self.flush()
+        return BufferedReader.read1(self, n)
+
+    def write(self, b):
+        if self._read_buf:
+            # Undo readahead
+            with self._read_lock:
+                self.raw.seek(self._read_pos - len(self._read_buf), 1)
+                self._reset_read_buf()
+        return BufferedWriter.write(self, b)
+
+
+class TextIOBase(IOBase):
+
+    """Base class for text I/O.
+
+    This class provides a character and line based interface to stream
+    I/O. There is no readinto method because Python's character strings
+    are immutable. There is no public constructor.
+    """
+
+    def read(self, n=-1):
+        """Read at most n characters from stream, where n is an int.
+
+        Read from underlying buffer until we have n characters or we hit EOF.
+        If n is negative or omitted, read until EOF.
+
+        Returns a string.
+        """
+        self._unsupported("read")
+
+    def write(self, s):
+        """Write string s to stream and returning an int."""
+        self._unsupported("write")
+
+    def truncate(self, pos=None):
+        """Truncate size to pos, where pos is an int."""
+        self._unsupported("truncate")
+
+    def readline(self):
+        """Read until newline or EOF.
+
+        Returns an empty string if EOF is hit immediately.
+        """
+        self._unsupported("readline")
+
+    def detach(self):
+        """
+        Separate the underlying buffer from the TextIOBase and return it.
+
+        After the underlying buffer has been detached, the TextIO is in an
+        unusable state.
+        """
+        self._unsupported("detach")
+
+    @property
+    def encoding(self):
+        """Subclasses should override."""
+        return None
+
+    @property
+    def newlines(self):
+        """Line endings translated so far.
+
+        Only line endings translated during reading are considered.
+
+        Subclasses should override.
+        """
+        return None
+
+    @property
+    def errors(self):
+        """Error setting of the decoder or encoder.
+
+        Subclasses should override."""
+        return None
+
+io.TextIOBase.register(TextIOBase)
+
+
+class IncrementalNewlineDecoder(codecs.IncrementalDecoder):
+    r"""Codec used when reading a file in universal newlines mode.  It wraps
+    another incremental decoder, translating \r\n and \r into \n.  It also
+    records the types of newlines encountered.  When used with
+    translate=False, it ensures that the newline sequence is returned in
+    one piece.
+    """
+    def __init__(self, decoder, translate, errors='strict'):
+        codecs.IncrementalDecoder.__init__(self, errors=errors)
+        self.translate = translate
+        self.decoder = decoder
+        self.seennl = 0
+        self.pendingcr = False
+
+    def decode(self, input, final=False):
+        # decode input (with the eventual \r from a previous pass)
+        if self.decoder is None:
+            output = input
+        else:
+            output = self.decoder.decode(input, final=final)
+        if self.pendingcr and (output or final):
+            output = "\r" + output
+            self.pendingcr = False
+
+        # retain last \r even when not translating data:
+        # then readline() is sure to get \r\n in one pass
+        if output.endswith("\r") and not final:
+            output = output[:-1]
+            self.pendingcr = True
+
+        # Record which newlines are read
+        crlf = output.count('\r\n')
+        cr = output.count('\r') - crlf
+        lf = output.count('\n') - crlf
+        self.seennl |= (lf and self._LF) | (cr and self._CR) \
+                    | (crlf and self._CRLF)
+
+        if self.translate:
+            if crlf:
+                output = output.replace("\r\n", "\n")
+            if cr:
+                output = output.replace("\r", "\n")
+
+        return output
+
+    def getstate(self):
+        if self.decoder is None:
+            buf = b""
+            flag = 0
+        else:
+            buf, flag = self.decoder.getstate()
+        flag <<= 1
+        if self.pendingcr:
+            flag |= 1
+        return buf, flag
+
+    def setstate(self, state):
+        buf, flag = state
+        self.pendingcr = bool(flag & 1)
+        if self.decoder is not None:
+            self.decoder.setstate((buf, flag >> 1))
+
+    def reset(self):
+        self.seennl = 0
+        self.pendingcr = False
+        if self.decoder is not None:
+            self.decoder.reset()
+
+    _LF = 1
+    _CR = 2
+    _CRLF = 4
+
+    @property
+    def newlines(self):
+        return (None,
+                "\n",
+                "\r",
+                ("\r", "\n"),
+                "\r\n",
+                ("\n", "\r\n"),
+                ("\r", "\r\n"),
+                ("\r", "\n", "\r\n")
+               )[self.seennl]
+
+
+class TextIOWrapper(TextIOBase):
+
+    r"""Character and line based layer over a BufferedIOBase object, buffer.
+
+    encoding gives the name of the encoding that the stream will be
+    decoded or encoded with. It defaults to locale.getpreferredencoding.
+
+    errors determines the strictness of encoding and decoding (see the
+    codecs.register) and defaults to "strict".
+
+    newline can be None, '', '\n', '\r', or '\r\n'.  It controls the
+    handling of line endings. If it is None, universal newlines is
+    enabled.  With this enabled, on input, the lines endings '\n', '\r',
+    or '\r\n' are translated to '\n' before being returned to the
+    caller. Conversely, on output, '\n' is translated to the system
+    default line separator, os.linesep. If newline is any other of its
+    legal values, that newline becomes the newline when the file is read
+    and it is returned untranslated. On output, '\n' is converted to the
+    newline.
+
+    If line_buffering is True, a call to flush is implied when a call to
+    write contains a newline character.
+    """
+
+    _CHUNK_SIZE = 2048
+
+    def __init__(self, buffer, encoding=None, errors=None, newline=None,
+                 line_buffering=False, write_through=False):
+        if newline is not None and not isinstance(newline, str):
+            raise TypeError("illegal newline type: %r" % (type(newline),))
+        if newline not in (None, "", "\n", "\r", "\r\n"):
+            raise ValueError("illegal newline value: %r" % (newline,))
+        if encoding is None:
+            try:
+                encoding = os.device_encoding(buffer.fileno())
+            except (AttributeError, UnsupportedOperation):
+                pass
+            if encoding is None:
+                try:
+                    import locale
+                except ImportError:
+                    # Importing locale may fail if Python is being built
+                    encoding = "ascii"
+                else:
+                    encoding = locale.getpreferredencoding()
+
+        if not isinstance(encoding, str):
+            raise ValueError("invalid encoding: %r" % encoding)
+
+        if errors is None:
+            errors = "strict"
+        else:
+            if not isinstance(errors, str):
+                raise ValueError("invalid errors: %r" % errors)
+
+        self._buffer = buffer
+        self._line_buffering = line_buffering
+        self._encoding = encoding
+        self._errors = errors
+        self._readuniversal = not newline
+        self._readtranslate = newline is None
+        self._readnl = newline
+        self._writetranslate = newline != ''
+        self._writenl = newline or os.linesep
+        self._encoder = None
+        self._decoder = None
+        self._decoded_chars = ''  # buffer for text returned from decoder
+        self._decoded_chars_used = 0  # offset into _decoded_chars for read()
+        self._snapshot = None  # info for reconstructing decoder state
+        self._seekable = self._telling = self.buffer.seekable()
+        self._has_read1 = hasattr(self.buffer, 'read1')
+
+        if self._seekable and self.writable():
+            position = self.buffer.tell()
+            if position != 0:
+                try:
+                    self._get_encoder().setstate(0)
+                except LookupError:
+                    # Sometimes the encoder doesn't exist
+                    pass
+
+    # self._snapshot is either None, or a tuple (dec_flags, next_input)
+    # where dec_flags is the second (integer) item of the decoder state
+    # and next_input is the chunk of input bytes that comes next after the
+    # snapshot point.  We use this to reconstruct decoder states in tell().
+
+    # Naming convention:
+    #   - "bytes_..." for integer variables that count input bytes
+    #   - "chars_..." for integer variables that count decoded characters
+
+    def __repr__(self):
+        result = "<_pyio.TextIOWrapper"
+        try:
+            name = self.name
+        except AttributeError:
+            pass
+        else:
+            result += " name={0!r}".format(name)
+        try:
+            mode = self.mode
+        except AttributeError:
+            pass
+        else:
+            result += " mode={0!r}".format(mode)
+        return result + " encoding={0!r}>".format(self.encoding)
+
+    @property
+    def encoding(self):
+        return self._encoding
+
+    @property
+    def errors(self):
+        return self._errors
+
+    @property
+    def line_buffering(self):
+        return self._line_buffering
+
+    @property
+    def buffer(self):
+        return self._buffer
+
+    def seekable(self):
+        return self._seekable
+
+    def readable(self):
+        return self.buffer.readable()
+
+    def writable(self):
+        return self.buffer.writable()
+
+    def flush(self):
+        self.buffer.flush()
+        self._telling = self._seekable
+
+    def close(self):
+        if self.buffer is not None and not self.closed:
+            self.flush()
+            self.buffer.close()
+
+    @property
+    def closed(self):
+        return self.buffer.closed
+
+    @property
+    def name(self):
+        return self.buffer.name
+
+    def fileno(self):
+        return self.buffer.fileno()
+
+    def isatty(self):
+        return self.buffer.isatty()
+
+    def write(self, s):
+        'Write data, where s is a str'
+        if self.closed:
+            raise ValueError("write to closed file")
+        if not isinstance(s, str):
+            raise TypeError("can't write %s to text stream" %
+                            s.__class__.__name__)
+        length = len(s)
+        haslf = (self._writetranslate or self._line_buffering) and "\n" in s
+        if haslf and self._writetranslate and self._writenl != "\n":
+            s = s.replace("\n", self._writenl)
+        encoder = self._encoder or self._get_encoder()
+        # XXX What if we were just reading?
+        b = encoder.encode(s)
+        self.buffer.write(b)
+        if self._line_buffering and (haslf or "\r" in s):
+            self.flush()
+        self._snapshot = None
+        if self._decoder:
+            self._decoder.reset()
+        return length
+
+    def _get_encoder(self):
+        make_encoder = codecs.getincrementalencoder(self._encoding)
+        self._encoder = make_encoder(self._errors)
+        return self._encoder
+
+    def _get_decoder(self):
+        make_decoder = codecs.getincrementaldecoder(self._encoding)
+        decoder = make_decoder(self._errors)
+        if self._readuniversal:
+            decoder = IncrementalNewlineDecoder(decoder, self._readtranslate)
+        self._decoder = decoder
+        return decoder
+
+    # The following three methods implement an ADT for _decoded_chars.
+    # Text returned from the decoder is buffered here until the client
+    # requests it by calling our read() or readline() method.
+    def _set_decoded_chars(self, chars):
+        """Set the _decoded_chars buffer."""
+        self._decoded_chars = chars
+        self._decoded_chars_used = 0
+
+    def _get_decoded_chars(self, n=None):
+        """Advance into the _decoded_chars buffer."""
+        offset = self._decoded_chars_used
+        if n is None:
+            chars = self._decoded_chars[offset:]
+        else:
+            chars = self._decoded_chars[offset:offset + n]
+        self._decoded_chars_used += len(chars)
+        return chars
+
+    def _rewind_decoded_chars(self, n):
+        """Rewind the _decoded_chars buffer."""
+        if self._decoded_chars_used < n:
+            raise AssertionError("rewind decoded_chars out of bounds")
+        self._decoded_chars_used -= n
+
+    def _read_chunk(self):
+        """
+        Read and decode the next chunk of data from the BufferedReader.
+        """
+
+        # The return value is True unless EOF was reached.  The decoded
+        # string is placed in self._decoded_chars (replacing its previous
+        # value).  The entire input chunk is sent to the decoder, though
+        # some of it may remain buffered in the decoder, yet to be
+        # converted.
+
+        if self._decoder is None:
+            raise ValueError("no decoder")
+
+        if self._telling:
+            # To prepare for tell(), we need to snapshot a point in the
+            # file where the decoder's input buffer is empty.
+
+            dec_buffer, dec_flags = self._decoder.getstate()
+            # Given this, we know there was a valid snapshot point
+            # len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
+
+        # Read a chunk, decode it, and put the result in self._decoded_chars.
+        if self._has_read1:
+            input_chunk = self.buffer.read1(self._CHUNK_SIZE)
+        else:
+            input_chunk = self.buffer.read(self._CHUNK_SIZE)
+        eof = not input_chunk
+        self._set_decoded_chars(self._decoder.decode(input_chunk, eof))
+
+        if self._telling:
+            # At the snapshot point, len(dec_buffer) bytes before the read,
+            # the next input to be decoded is dec_buffer + input_chunk.
+            self._snapshot = (dec_flags, dec_buffer + input_chunk)
+
+        return not eof
+
+    def _pack_cookie(self, position, dec_flags=0,
+                           bytes_to_feed=0, need_eof=0, chars_to_skip=0):
+        # The meaning of a tell() cookie is: seek to position, set the
+        # decoder flags to dec_flags, read bytes_to_feed bytes, feed them
+        # into the decoder with need_eof as the EOF flag, then skip
+        # chars_to_skip characters of the decoded result.  For most simple
+        # decoders, tell() will often just give a byte offset in the file.
+        return (position | (dec_flags<<64) | (bytes_to_feed<<128) |
+               (chars_to_skip<<192) | bool(need_eof)<<256)
+
+    def _unpack_cookie(self, bigint):
+        rest, position = divmod(bigint, 1<<64)
+        rest, dec_flags = divmod(rest, 1<<64)
+        rest, bytes_to_feed = divmod(rest, 1<<64)
+        need_eof, chars_to_skip = divmod(rest, 1<<64)
+        return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip
+
+    def tell(self):
+        if not self._seekable:
+            raise UnsupportedOperation("underlying stream is not seekable")
+        if not self._telling:
+            raise IOError("telling position disabled by next() call")
+        self.flush()
+        position = self.buffer.tell()
+        decoder = self._decoder
+        if decoder is None or self._snapshot is None:
+            if self._decoded_chars:
+                # This should never happen.
+                raise AssertionError("pending decoded text")
+            return position
+
+        # Skip backward to the snapshot point (see _read_chunk).
+        dec_flags, next_input = self._snapshot
+        position -= len(next_input)
+
+        # How many decoded characters have been used up since the snapshot?
+        chars_to_skip = self._decoded_chars_used
+        if chars_to_skip == 0:
+            # We haven't moved from the snapshot point.
+            return self._pack_cookie(position, dec_flags)
+
+        # Starting from the snapshot position, we will walk the decoder
+        # forward until it gives us enough decoded characters.
+        saved_state = decoder.getstate()
+        try:
+            # Note our initial start point.
+            decoder.setstate((b'', dec_flags))
+            start_pos = position
+            start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
+            need_eof = 0
+
+            # Feed the decoder one byte at a time.  As we go, note the
+            # nearest "safe start point" before the current location
+            # (a point where the decoder has nothing buffered, so seek()
+            # can safely start from there and advance to this location).
+            next_byte = bytearray(1)
+            for next_byte[0] in next_input:
+                bytes_fed += 1
+                chars_decoded += len(decoder.decode(next_byte))
+                dec_buffer, dec_flags = decoder.getstate()
+                if not dec_buffer and chars_decoded <= chars_to_skip:
+                    # Decoder buffer is empty, so this is a safe start point.
+                    start_pos += bytes_fed
+                    chars_to_skip -= chars_decoded
+                    start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
+                if chars_decoded >= chars_to_skip:
+                    break
+            else:
+                # We didn't get enough decoded data; signal EOF to get more.
+                chars_decoded += len(decoder.decode(b'', final=True))
+                need_eof = 1
+                if chars_decoded < chars_to_skip:
+                    raise IOError("can't reconstruct logical file position")
+
+            # The returned cookie corresponds to the last safe start point.
+            return self._pack_cookie(
+                start_pos, start_flags, bytes_fed, need_eof, chars_to_skip)
+        finally:
+            decoder.setstate(saved_state)
+
+    def truncate(self, pos=None):
+        self.flush()
+        if pos is None:
+            pos = self.tell()
+        return self.buffer.truncate(pos)
+
+    def detach(self):
+        if self.buffer is None:
+            raise ValueError("buffer is already detached")
+        self.flush()
+        buffer = self._buffer
+        self._buffer = None
+        return buffer
+
+    def seek(self, cookie, whence=0):
+        if self.closed:
+            raise ValueError("tell on closed file")
+        if not self._seekable:
+            raise UnsupportedOperation("underlying stream is not seekable")
+        if whence == 1: # seek relative to current position
+            if cookie != 0:
+                raise UnsupportedOperation("can't do nonzero cur-relative seeks")
+            # Seeking to the current position should attempt to
+            # sync the underlying buffer with the current position.
+            whence = 0
+            cookie = self.tell()
+        if whence == 2: # seek relative to end of file
+            if cookie != 0:
+                raise UnsupportedOperation("can't do nonzero end-relative seeks")
+            self.flush()
+            position = self.buffer.seek(0, 2)
+            self._set_decoded_chars('')
+            self._snapshot = None
+            if self._decoder:
+                self._decoder.reset()
+            return position
+        if whence != 0:
+            raise ValueError("invalid whence (%r, should be 0, 1 or 2)" %
+                             (whence,))
+        if cookie < 0:
+            raise ValueError("negative seek position %r" % (cookie,))
+        self.flush()
+
+        # The strategy of seek() is to go back to the safe start point
+        # and replay the effect of read(chars_to_skip) from there.
+        start_pos, dec_flags, bytes_to_feed, need_eof, chars_to_skip = \
+            self._unpack_cookie(cookie)
+
+        # Seek back to the safe start point.
+        self.buffer.seek(start_pos)
+        self._set_decoded_chars('')
+        self._snapshot = None
+
+        # Restore the decoder to its state from the safe start point.
+        if cookie == 0 and self._decoder:
+            self._decoder.reset()
+        elif self._decoder or dec_flags or chars_to_skip:
+            self._decoder = self._decoder or self._get_decoder()
+            self._decoder.setstate((b'', dec_flags))
+            self._snapshot = (dec_flags, b'')
+
+        if chars_to_skip:
+            # Just like _read_chunk, feed the decoder and save a snapshot.
+            input_chunk = self.buffer.read(bytes_to_feed)
+            self._set_decoded_chars(
+                self._decoder.decode(input_chunk, need_eof))
+            self._snapshot = (dec_flags, input_chunk)
+
+            # Skip chars_to_skip of the decoded characters.
+            if len(self._decoded_chars) < chars_to_skip:
+                raise IOError("can't restore logical file position")
+            self._decoded_chars_used = chars_to_skip
+
+        # Finally, reset the encoder (merely useful for proper BOM handling)
+        try:
+            encoder = self._encoder or self._get_encoder()
+        except LookupError:
+            # Sometimes the encoder doesn't exist
+            pass
+        else:
+            if cookie != 0:
+                encoder.setstate(0)
+            else:
+                encoder.reset()
+        return cookie
+
+    def read(self, n=None):
+        self._checkReadable()
+        if n is None:
+            n = -1
+        decoder = self._decoder or self._get_decoder()
+        try:
+            n.__index__
+        except AttributeError as err:
+            raise TypeError("an integer is required") from err
+        if n < 0:
+            # Read everything.
+            result = (self._get_decoded_chars() +
+                      decoder.decode(self.buffer.read(), final=True))
+            self._set_decoded_chars('')
+            self._snapshot = None
+            return result
+        else:
+            # Keep reading chunks until we have n characters to return.
+            eof = False
+            result = self._get_decoded_chars(n)
+            while len(result) < n and not eof:
+                eof = not self._read_chunk()
+                result += self._get_decoded_chars(n - len(result))
+            return result
+
+    def __next__(self):
+        self._telling = False
+        line = self.readline()
+        if not line:
+            self._snapshot = None
+            self._telling = self._seekable
+            raise StopIteration
+        return line
+
+    def readline(self, limit=None):
+        if self.closed:
+            raise ValueError("read from closed file")
+        if limit is None:
+            limit = -1
+        elif not isinstance(limit, int):
+            raise TypeError("limit must be an integer")
+
+        # Grab all the decoded text (we will rewind any extra bits later).
+        line = self._get_decoded_chars()
+
+        start = 0
+        # Make the decoder if it doesn't already exist.
+        if not self._decoder:
+            self._get_decoder()
+
+        pos = endpos = None
+        while True:
+            if self._readtranslate:
+                # Newlines are already translated, only search for \n
+                pos = line.find('\n', start)
+                if pos >= 0:
+                    endpos = pos + 1
+                    break
+                else:
+                    start = len(line)
+
+            elif self._readuniversal:
+                # Universal newline search. Find any of \r, \r\n, \n
+                # The decoder ensures that \r\n are not split in two pieces
+
+                # In C we'd look for these in parallel of course.
+                nlpos = line.find("\n", start)
+                crpos = line.find("\r", start)
+                if crpos == -1:
+                    if nlpos == -1:
+                        # Nothing found
+                        start = len(line)
+                    else:
+                        # Found \n
+                        endpos = nlpos + 1
+                        break
+                elif nlpos == -1:
+                    # Found lone \r
+                    endpos = crpos + 1
+                    break
+                elif nlpos < crpos:
+                    # Found \n
+                    endpos = nlpos + 1
+                    break
+                elif nlpos == crpos + 1:
+                    # Found \r\n
+                    endpos = crpos + 2
+                    break
+                else:
+                    # Found \r
+                    endpos = crpos + 1
+                    break
+            else:
+                # non-universal
+                pos = line.find(self._readnl)
+                if pos >= 0:
+                    endpos = pos + len(self._readnl)
+                    break
+
+            if limit >= 0 and len(line) >= limit:
+                endpos = limit  # reached length limit
+                break
+
+            # No line ending seen yet - get more data'
+            while self._read_chunk():
+                if self._decoded_chars:
+                    break
+            if self._decoded_chars:
+                line += self._get_decoded_chars()
+            else:
+                # end of file
+                self._set_decoded_chars('')
+                self._snapshot = None
+                return line
+
+        if limit >= 0 and endpos > limit:
+            endpos = limit  # don't exceed limit
+
+        # Rewind _decoded_chars to just after the line ending we found.
+        self._rewind_decoded_chars(len(line) - endpos)
+        return line[:endpos]
+
+    @property
+    def newlines(self):
+        return self._decoder.newlines if self._decoder else None
+
+
+class StringIO(TextIOWrapper):
+    """Text I/O implementation using an in-memory buffer.
+
+    The initial_value argument sets the value of object.  The newline
+    argument is like the one of TextIOWrapper's constructor.
+    """
+
+    def __init__(self, initial_value="", newline="\n"):
+        super(StringIO, self).__init__(BytesIO(),
+                                       encoding="utf-8",
+                                       errors="strict",
+                                       newline=newline)
+        # Issue #5645: make universal newlines semantics the same as in the
+        # C version, even under Windows.
+        if newline is None:
+            self._writetranslate = False
+        if initial_value is not None:
+            if not isinstance(initial_value, str):
+                raise TypeError("initial_value must be str or None, not {0}"
+                                .format(type(initial_value).__name__))
+                initial_value = str(initial_value)
+            self.write(initial_value)
+            self.seek(0)
+
+    def getvalue(self):
+        self.flush()
+        return self.buffer.getvalue().decode(self._encoding, self._errors)
+
+    def __repr__(self):
+        # TextIOWrapper tells the encoding in its repr. In StringIO,
+        # that's a implementation detail.
+        return object.__repr__(self)
+
+    @property
+    def errors(self):
+        return None
+
+    @property
+    def encoding(self):
+        return None
+
+    def detach(self):
+        # This doesn't make sense on StringIO.
+        self._unsupported("detach")
diff --git a/lib-python/3.2/_strptime.py b/lib-python/3.2/_strptime.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_strptime.py
@@ -0,0 +1,499 @@
+"""Strptime-related classes and functions.
+
+CLASSES:
+    LocaleTime -- Discovers and stores locale-specific time information
+    TimeRE -- Creates regexes for pattern matching a string of text containing
+                time information
+
+FUNCTIONS:
+    _getlang -- Figure out what language is being used for the locale
+    strptime -- Calculates the time struct represented by the passed-in string
+
+"""
+import time
+import locale
+import calendar
+from re import compile as re_compile
+from re import IGNORECASE, ASCII
+from re import escape as re_escape
+from datetime import (date as datetime_date,
+                      timedelta as datetime_timedelta,
+                      timezone as datetime_timezone)
+try:
+    from _thread import allocate_lock as _thread_allocate_lock
+except:
+    from _dummy_thread import allocate_lock as _thread_allocate_lock
+
+__all__ = []
+
+def _getlang():
+    # Figure out what the current language is set to.
+    return locale.getlocale(locale.LC_TIME)
+
+class LocaleTime(object):
+    """Stores and handles locale-specific information related to time.
+
+    ATTRIBUTES:
+        f_weekday -- full weekday names (7-item list)
+        a_weekday -- abbreviated weekday names (7-item list)
+        f_month -- full month names (13-item list; dummy value in [0], which
+                    is added by code)
+        a_month -- abbreviated month names (13-item list, dummy value in
+                    [0], which is added by code)
+        am_pm -- AM/PM representation (2-item list)
+        LC_date_time -- format string for date/time representation (string)
+        LC_date -- format string for date representation (string)
+        LC_time -- format string for time representation (string)
+        timezone -- daylight- and non-daylight-savings timezone representation
+                    (2-item list of sets)
+        lang -- Language used by instance (2-item tuple)
+    """
+
+    def __init__(self):
+        """Set all attributes.
+
+        Order of methods called matters for dependency reasons.
+
+        The locale language is set at the offset and then checked again before
+        exiting.  This is to make sure that the attributes were not set with a
+        mix of information from more than one locale.  This would most likely
+        happen when using threads where one thread calls a locale-dependent
+        function while another thread changes the locale while the function in
+        the other thread is still running.  Proper coding would call for
+        locks to prevent changing the locale while locale-dependent code is
+        running.  The check here is done in case someone does not think about
+        doing this.
+
+        Only other possible issue is if someone changed the timezone and did
+        not call tz.tzset .  That is an issue for the programmer, though,
+        since changing the timezone is worthless without that call.
+
+        """
+        self.lang = _getlang()
+        self.__calc_weekday()
+        self.__calc_month()
+        self.__calc_am_pm()
+        self.__calc_timezone()
+        self.__calc_date_time()
+        if _getlang() != self.lang:
+            raise ValueError("locale changed during initialization")
+
+    def __pad(self, seq, front):
+        # Add '' to seq to either the front (is True), else the back.
+        seq = list(seq)
+        if front:
+            seq.insert(0, '')
+        else:
+            seq.append('')
+        return seq
+
+    def __calc_weekday(self):
+        # Set self.a_weekday and self.f_weekday using the calendar
+        # module.
+        a_weekday = [calendar.day_abbr[i].lower() for i in range(7)]
+        f_weekday = [calendar.day_name[i].lower() for i in range(7)]
+        self.a_weekday = a_weekday
+        self.f_weekday = f_weekday
+
+    def __calc_month(self):
+        # Set self.f_month and self.a_month using the calendar module.
+        a_month = [calendar.month_abbr[i].lower() for i in range(13)]
+        f_month = [calendar.month_name[i].lower() for i in range(13)]
+        self.a_month = a_month
+        self.f_month = f_month
+
+    def __calc_am_pm(self):
+        # Set self.am_pm by using time.strftime().
+
+        # The magic date (1999,3,17,hour,44,55,2,76,0) is not really that
+        # magical; just happened to have used it everywhere else where a
+        # static date was needed.
+        am_pm = []
+        for hour in (1, 22):
+            time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0))
+            am_pm.append(time.strftime("%p", time_tuple).lower())
+        self.am_pm = am_pm
+
+    def __calc_date_time(self):
+        # Set self.date_time, self.date, & self.time by using
+        # time.strftime().
+
+        # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of
+        # overloaded numbers is minimized.  The order in which searches for
+        # values within the format string is very important; it eliminates
+        # possible ambiguity for what something represents.
+        time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
+        date_time = [None, None, None]
+        date_time[0] = time.strftime("%c", time_tuple).lower()
+        date_time[1] = time.strftime("%x", time_tuple).lower()
+        date_time[2] = time.strftime("%X", time_tuple).lower()
+        replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'),
+                    (self.f_month[3], '%B'), (self.a_weekday[2], '%a'),
+                    (self.a_month[3], '%b'), (self.am_pm[1], '%p'),
+                    ('1999', '%Y'), ('99', '%y'), ('22', '%H'),
+                    ('44', '%M'), ('55', '%S'), ('76', '%j'),
+                    ('17', '%d'), ('03', '%m'), ('3', '%m'),
+                    # '3' needed for when no leading zero.
+                    ('2', '%w'), ('10', '%I')]
+        replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone
+                                                for tz in tz_values])
+        for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')):
+            current_format = date_time[offset]
+            for old, new in replacement_pairs:
+                # Must deal with possible lack of locale info
+                # manifesting itself as the empty string (e.g., Swedish's
+                # lack of AM/PM info) or a platform returning a tuple of empty
+                # strings (e.g., MacOS 9 having timezone as ('','')).
+                if old:
+                    current_format = current_format.replace(old, new)
+            # If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
+            # 2005-01-03 occurs before the first Monday of the year.  Otherwise
+            # %U is used.
+            time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0))
+            if '00' in time.strftime(directive, time_tuple):
+                U_W = '%W'
+            else:
+                U_W = '%U'
+            date_time[offset] = current_format.replace('11', U_W)
+        self.LC_date_time = date_time[0]
+        self.LC_date = date_time[1]
+        self.LC_time = date_time[2]
+
+    def __calc_timezone(self):
+        # Set self.timezone by using time.tzname.
+        # Do not worry about possibility of time.tzname[0] == timetzname[1]
+        # and time.daylight; handle that in strptime .
+        try:
+            time.tzset()
+        except AttributeError:
+            pass
+        no_saving = frozenset(["utc", "gmt", time.tzname[0].lower()])
+        if time.daylight:
+            has_saving = frozenset([time.tzname[1].lower()])
+        else:
+            has_saving = frozenset()
+        self.timezone = (no_saving, has_saving)
+
+
+class TimeRE(dict):
+    """Handle conversion from format directives to regexes."""
+
+    def __init__(self, locale_time=None):
+        """Create keys/values.
+
+        Order of execution is important for dependency reasons.
+
+        """
+        if locale_time:
+            self.locale_time = locale_time
+        else:
+            self.locale_time = LocaleTime()
+        base = super()
+        base.__init__({
+            # The " \d" part of the regex is to make %c from ANSI C work
+            'd': r"(?P<d>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
+            'f': r"(?P<f>[0-9]{1,6})",
+            'H': r"(?P<H>2[0-3]|[0-1]\d|\d)",
+            'I': r"(?P<I>1[0-2]|0[1-9]|[1-9])",
+            'j': r"(?P<j>36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
+            'm': r"(?P<m>1[0-2]|0[1-9]|[1-9])",
+            'M': r"(?P<M>[0-5]\d|\d)",
+            'S': r"(?P<S>6[0-1]|[0-5]\d|\d)",
+            'U': r"(?P<U>5[0-3]|[0-4]\d|\d)",
+            'w': r"(?P<w>[0-6])",
+            # W is set below by using 'U'
+            'y': r"(?P<y>\d\d)",
+            #XXX: Does 'Y' need to worry about having less or more than
+            #     4 digits?
+            'Y': r"(?P<Y>\d\d\d\d)",
+            'z': r"(?P<z>[+-]\d\d[0-5]\d)",
+            'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
+            'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
+            'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
+            'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
+            'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
+            'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
+                                        for tz in tz_names),
+                                'Z'),
+            '%': '%'})
+        base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
+        base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
+        base.__setitem__('x', self.pattern(self.locale_time.LC_date))
+        base.__setitem__('X', self.pattern(self.locale_time.LC_time))
+
+    def __seqToRE(self, to_convert, directive):
+        """Convert a list to a regex string for matching a directive.
+
+        Want possible matching values to be from longest to shortest.  This
+        prevents the possibility of a match occuring for a value that also
+        a substring of a larger value that should have matched (e.g., 'abc'
+        matching when 'abcdef' should have been the match).
+
+        """
+        to_convert = sorted(to_convert, key=len, reverse=True)
+        for value in to_convert:
+            if value != '':
+                break
+        else:
+            return ''
+        regex = '|'.join(re_escape(stuff) for stuff in to_convert)
+        regex = '(?P<%s>%s' % (directive, regex)
+        return '%s)' % regex
+
+    def pattern(self, format):
+        """Return regex pattern for the format string.
+
+        Need to make sure that any characters that might be interpreted as
+        regex syntax are escaped.
+
+        """
+        processed_format = ''
+        # The sub() call escapes all characters that might be misconstrued
+        # as regex syntax.  Cannot use re.escape since we have to deal with
+        # format directives (%m, etc.).
+        regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
+        format = regex_chars.sub(r"\\\1", format)
+        whitespace_replacement = re_compile('\s+')
+        format = whitespace_replacement.sub('\s+', format)
+        while '%' in format:
+            directive_index = format.index('%')+1
+            processed_format = "%s%s%s" % (processed_format,
+                                           format[:directive_index-1],
+                                           self[format[directive_index]])
+            format = format[directive_index+1:]
+        return "%s%s" % (processed_format, format)
+
+    def compile(self, format):
+        """Return a compiled re object for the format string."""
+        return re_compile(self.pattern(format), IGNORECASE)
+
+_cache_lock = _thread_allocate_lock()
+# DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock
+# first!
+_TimeRE_cache = TimeRE()
+_CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache
+_regex_cache = {}
+
+def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon):
+    """Calculate the Julian day based on the year, week of the year, and day of
+    the week, with week_start_day representing whether the week of the year
+    assumes the week starts on Sunday or Monday (6 or 0)."""
+    first_weekday = datetime_date(year, 1, 1).weekday()
+    # If we are dealing with the %U directive (week starts on Sunday), it's
+    # easier to just shift the view to Sunday being the first day of the
+    # week.
+    if not week_starts_Mon:
+        first_weekday = (first_weekday + 1) % 7
+        day_of_week = (day_of_week + 1) % 7
+    # Need to watch out for a week 0 (when the first day of the year is not
+    # the same as that specified by %U or %W).
+    week_0_length = (7 - first_weekday) % 7
+    if week_of_year == 0:
+        return 1 + day_of_week - first_weekday
+    else:
+        days_to_week = week_0_length + (7 * (week_of_year - 1))
+        return 1 + days_to_week + day_of_week
+
+
+def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a 2-tuple consisting of a time struct and an int containing
+    the number of microseconds based on the input string and the
+    format string."""
+
+    for index, arg in enumerate([data_string, format]):
+        if not isinstance(arg, str):
+            msg = "strptime() argument {} must be str, not {}"
+            raise TypeError(msg.format(index, type(arg)))
+
+    global _TimeRE_cache, _regex_cache
+    with _cache_lock:
+
+        if _getlang() != _TimeRE_cache.locale_time.lang:
+            _TimeRE_cache = TimeRE()
+            _regex_cache.clear()
+        if len(_regex_cache) > _CACHE_MAX_SIZE:
+            _regex_cache.clear()
+        locale_time = _TimeRE_cache.locale_time
+        format_regex = _regex_cache.get(format)
+        if not format_regex:
+            try:
+                format_regex = _TimeRE_cache.compile(format)
+            # KeyError raised when a bad format is found; can be specified as
+            # \\, in which case it was a stray % but with a space after it
+            except KeyError as err:
+                bad_directive = err.args[0]
+                if bad_directive == "\\":
+                    bad_directive = "%"
+                del err
+                raise ValueError("'%s' is a bad directive in format '%s'" %
+                                    (bad_directive, format))
+            # IndexError only occurs when the format string is "%"
+            except IndexError:
+                raise ValueError("stray %% in format '%s'" % format)
+            _regex_cache[format] = format_regex
+    found = format_regex.match(data_string)
+    if not found:
+        raise ValueError("time data %r does not match format %r" %
+                         (data_string, format))
+    if len(data_string) != found.end():
+        raise ValueError("unconverted data remains: %s" %
+                          data_string[found.end():])
+
+    year = 1900
+    month = day = 1
+    hour = minute = second = fraction = 0
+    tz = -1
+    tzoffset = None
+    # Default to -1 to signify that values not known; not critical to have,
+    # though
+    week_of_year = -1
+    week_of_year_start = -1
+    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # values
+    weekday = julian = -1
+    found_dict = found.groupdict()
+    for group_key in found_dict.keys():
+        # Directives not explicitly handled below:
+        #   c, x, X
+        #      handled by making out of other directives
+        #   U, W
+        #      worthless without day of the week
+        if group_key == 'y':
+            year = int(found_dict['y'])
+            # Open Group specification for strptime() states that a %y
+            #value in the range of [00, 68] is in the century 2000, while
+            #[69,99] is in the century 1900
+            if year <= 68:
+                year += 2000
+            else:
+                year += 1900
+        elif group_key == 'Y':
+            year = int(found_dict['Y'])
+        elif group_key == 'm':
+            month = int(found_dict['m'])
+        elif group_key == 'B':
+            month = locale_time.f_month.index(found_dict['B'].lower())
+        elif group_key == 'b':
+            month = locale_time.a_month.index(found_dict['b'].lower())
+        elif group_key == 'd':
+            day = int(found_dict['d'])
+        elif group_key == 'H':
+            hour = int(found_dict['H'])
+        elif group_key == 'I':
+            hour = int(found_dict['I'])
+            ampm = found_dict.get('p', '').lower()
+            # If there was no AM/PM indicator, we'll treat this like AM
+            if ampm in ('', locale_time.am_pm[0]):
+                # We're in AM so the hour is correct unless we're
+                # looking at 12 midnight.
+                # 12 midnight == 12 AM == hour 0
+                if hour == 12:
+                    hour = 0
+            elif ampm == locale_time.am_pm[1]:
+                # We're in PM so we need to add 12 to the hour unless
+                # we're looking at 12 noon.
+                # 12 noon == 12 PM == hour 12
+                if hour != 12:
+                    hour += 12
+        elif group_key == 'M':
+            minute = int(found_dict['M'])
+        elif group_key == 'S':
+            second = int(found_dict['S'])
+        elif group_key == 'f':
+            s = found_dict['f']
+            # Pad to always return microseconds.
+            s += "0" * (6 - len(s))
+            fraction = int(s)
+        elif group_key == 'A':
+            weekday = locale_time.f_weekday.index(found_dict['A'].lower())
+        elif group_key == 'a':
+            weekday = locale_time.a_weekday.index(found_dict['a'].lower())
+        elif group_key == 'w':
+            weekday = int(found_dict['w'])
+            if weekday == 0:
+                weekday = 6
+            else:
+                weekday -= 1
+        elif group_key == 'j':
+            julian = int(found_dict['j'])
+        elif group_key in ('U', 'W'):
+            week_of_year = int(found_dict[group_key])
+            if group_key == 'U':
+                # U starts week on Sunday.
+                week_of_year_start = 6
+            else:
+                # W starts week on Monday.
+                week_of_year_start = 0
+        elif group_key == 'z':
+            z = found_dict['z']
+            tzoffset = int(z[1:3]) * 60 + int(z[3:5])
+            if z.startswith("-"):
+                tzoffset = -tzoffset
+        elif group_key == 'Z':
+            # Since -1 is default value only need to worry about setting tz if
+            # it can be something other than -1.
+            found_zone = found_dict['Z'].lower()
+            for value, tz_values in enumerate(locale_time.timezone):
+                if found_zone in tz_values:
+                    # Deal with bad locale setup where timezone names are the
+                    # same and yet time.daylight is true; too ambiguous to
+                    # be able to tell what timezone has daylight savings
+                    if (time.tzname[0] == time.tzname[1] and
+                       time.daylight and found_zone not in ("utc", "gmt")):
+                        break
+                    else:
+                        tz = value
+                        break
+    # If we know the week of the year and what day of that week, we can figure
+    # out the Julian day of the year.
+    if julian == -1 and week_of_year != -1 and weekday != -1:
+        week_starts_Mon = True if week_of_year_start == 0 else False
+        julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
+                                            week_starts_Mon)
+    # Cannot pre-calculate datetime_date() since can change in Julian
+    # calculation and thus could have different value for the day of the week
+    # calculation.
+    if julian == -1:
+        # Need to add 1 to result since first day of the year is 1, not 0.
+        julian = datetime_date(year, month, day).toordinal() - \
+                  datetime_date(year, 1, 1).toordinal() + 1
+    else:  # Assume that if they bothered to include Julian day it will
+           # be accurate.
+        datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal())
+        year = datetime_result.year
+        month = datetime_result.month
+        day = datetime_result.day
+    if weekday == -1:
+        weekday = datetime_date(year, month, day).weekday()
+    # Add timezone info
+    tzname = found_dict.get("Z")
+    if tzoffset is not None:
+        gmtoff = tzoffset * 60
+    else:
+        gmtoff = None
+
+    return (year, month, day,
+            hour, minute, second,
+            weekday, julian, tz, gmtoff, tzname), fraction
+
+def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a time struct based on the input string and the
+    format string."""
+    tt = _strptime(data_string, format)[0]
+    return time.struct_time(tt[:9])
+
+def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a class cls instance based on the input string and the
+    format string."""
+    tt, fraction = _strptime(data_string, format)
+    gmtoff, tzname = tt[-2:]
+    args = tt[:6] + (fraction,)
+    if gmtoff is not None:
+        tzdelta = datetime_timedelta(seconds=gmtoff)
+        if tzname:
+            tz = datetime_timezone(tzdelta, tzname)
+        else:
+            tz = datetime_timezone(tzdelta)
+        args += (tz,)
+
+    return cls(*args)
diff --git a/lib-python/3.2/_threading_local.py b/lib-python/3.2/_threading_local.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_threading_local.py
@@ -0,0 +1,246 @@
+"""Thread-local objects.
+
+(Note that this module provides a Python version of the threading.local
+ class.  Depending on the version of Python you're using, there may be a
+ faster one available.  You should always import the `local` class from
+ `threading`.)
+
+Thread-local objects support the management of thread-local data.
+If you have data that you want to be local to a thread, simply create
+a thread-local object and use its attributes:
+
+  >>> mydata = local()
+  >>> mydata.number = 42
+  >>> mydata.number
+  42
+
+You can also access the local-object's dictionary:
+
+  >>> mydata.__dict__
+  {'number': 42}
+  >>> mydata.__dict__.setdefault('widgets', [])
+  []
+  >>> mydata.widgets
+  []
+
+What's important about thread-local objects is that their data are
+local to a thread. If we access the data in a different thread:
+
+  >>> log = []
+  >>> def f():
+  ...     items = sorted(mydata.__dict__.items())
+  ...     log.append(items)
+  ...     mydata.number = 11
+  ...     log.append(mydata.number)
+
+  >>> import threading
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+  >>> log
+  [[], 11]
+
+we get different data.  Furthermore, changes made in the other thread
+don't affect data seen in this thread:
+
+  >>> mydata.number
+  42
+
+Of course, values you get from a local object, including a __dict__
+attribute, are for whatever thread was current at the time the
+attribute was read.  For that reason, you generally don't want to save
+these values across threads, as they apply only to the thread they
+came from.
+
+You can create custom local objects by subclassing the local class:
+
+  >>> class MyLocal(local):
+  ...     number = 2
+  ...     initialized = False
+  ...     def __init__(self, **kw):
+  ...         if self.initialized:
+  ...             raise SystemError('__init__ called too many times')
+  ...         self.initialized = True
+  ...         self.__dict__.update(kw)
+  ...     def squared(self):
+  ...         return self.number ** 2
+
+This can be useful to support default values, methods and
+initialization.  Note that if you define an __init__ method, it will be
+called each time the local object is used in a separate thread.  This
+is necessary to initialize each thread's dictionary.
+
+Now if we create a local object:
+
+  >>> mydata = MyLocal(color='red')
+
+Now we have a default number:
+
+  >>> mydata.number
+  2
+
+an initial color:
+
+  >>> mydata.color
+  'red'
+  >>> del mydata.color
+
+And a method that operates on the data:
+
+  >>> mydata.squared()
+  4
+
+As before, we can access the data in a separate thread:
+
+  >>> log = []
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+  >>> log
+  [[('color', 'red'), ('initialized', True)], 11]
+
+without affecting this thread's data:
+
+  >>> mydata.number
+  2
+  >>> mydata.color
+  Traceback (most recent call last):
+  ...
+  AttributeError: 'MyLocal' object has no attribute 'color'
+
+Note that subclasses can define slots, but they are not thread
+local. They are shared across threads:
+
+  >>> class MyLocal(local):
+  ...     __slots__ = 'number'
+
+  >>> mydata = MyLocal()
+  >>> mydata.number = 42
+  >>> mydata.color = 'red'
+
+So, the separate thread:
+
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+
+affects what we see:
+
+  >>> mydata.number
+  11
+
+>>> del mydata
+"""
+
+from weakref import ref
+from contextlib import contextmanager
+
+__all__ = ["local"]
+
+# We need to use objects from the threading module, but the threading
+# module may also want to use our `local` class, if support for locals
+# isn't compiled in to the `thread` module.  This creates potential problems
+# with circular imports.  For that reason, we don't import `threading`
+# until the bottom of this file (a hack sufficient to worm around the
+# potential problems).  Note that all platforms on CPython do have support
+# for locals in the `thread` module, and there is no circular import problem
+# then, so problems introduced by fiddling the order of imports here won't
+# manifest.
+
+class _localimpl:
+    """A class managing thread-local dicts"""
+    __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__'
+
+    def __init__(self):
+        # The key used in the Thread objects' attribute dicts.
+        # We keep it a string for speed but make it unlikely to clash with
+        # a "real" attribute.
+        self.key = '_threading_local._localimpl.' + str(id(self))
+        # { id(Thread) -> (ref(Thread), thread-local dict) }
+        self.dicts = {}
+
+    def get_dict(self):
+        """Return the dict for the current thread. Raises KeyError if none
+        defined."""
+        thread = current_thread()
+        return self.dicts[id(thread)][1]
+
+    def create_dict(self):
+        """Create a new dict for the current thread, and return it."""
+        localdict = {}
+        key = self.key
+        thread = current_thread()
+        idt = id(thread)
+        def local_deleted(_, key=key):
+            # When the localimpl is deleted, remove the thread attribute.
+            thread = wrthread()
+            if thread is not None:
+                del thread.__dict__[key]
+        def thread_deleted(_, idt=idt):
+            # When the thread is deleted, remove the local dict.
+            # Note that this is suboptimal if the thread object gets
+            # caught in a reference loop. We would like to be called
+            # as soon as the OS-level thread ends instead.
+            local = wrlocal()
+            if local is not None:
+                dct = local.dicts.pop(idt)
+        wrlocal = ref(self, local_deleted)
+        wrthread = ref(thread, thread_deleted)
+        thread.__dict__[key] = wrlocal
+        self.dicts[idt] = wrthread, localdict
+        return localdict
+
+
+ at contextmanager
+def _patch(self):
+    impl = object.__getattribute__(self, '_local__impl')
+    try:
+        dct = impl.get_dict()
+    except KeyError:
+        dct = impl.create_dict()
+        args, kw = impl.localargs
+        self.__init__(*args, **kw)
+    with impl.locallock:
+        object.__setattr__(self, '__dict__', dct)
+        yield
+
+
+class local:
+    __slots__ = '_local__impl', '__dict__'
+
+    def __new__(cls, *args, **kw):
+        if (args or kw) and (cls.__init__ is object.__init__):
+            raise TypeError("Initialization arguments are not supported")
+        self = object.__new__(cls)
+        impl = _localimpl()
+        impl.localargs = (args, kw)
+        impl.locallock = RLock()
+        object.__setattr__(self, '_local__impl', impl)
+        # We need to create the thread dict in anticipation of
+        # __init__ being called, to make sure we don't call it
+        # again ourselves.
+        impl.create_dict()
+        return self
+
+    def __getattribute__(self, name):
+        with _patch(self):
+            return object.__getattribute__(self, name)
+
+    def __setattr__(self, name, value):
+        if name == '__dict__':
+            raise AttributeError(
+                "%r object attribute '__dict__' is read-only"
+                % self.__class__.__name__)
+        with _patch(self):
+            return object.__setattr__(self, name, value)
+
+    def __delattr__(self, name):
+        if name == '__dict__':
+            raise AttributeError(
+                "%r object attribute '__dict__' is read-only"
+                % self.__class__.__name__)
+        with _patch(self):
+            return object.__delattr__(self, name)
+
+
+from threading import current_thread, RLock
diff --git a/lib-python/3.2/_weakrefset.py b/lib-python/3.2/_weakrefset.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_weakrefset.py
@@ -0,0 +1,214 @@
+# Access WeakSet through the weakref module.
+# This code is separated-out because it is needed
+# by abc.py to load everything else at startup.
+
+from _weakref import ref
+
+__all__ = ['WeakSet']
+
+
+class _IterationGuard:
+    # This context manager registers itself in the current iterators of the
+    # weak container, such as to delay all removals until the context manager
+    # exits.
+    # This technique should be relatively thread-safe (since sets are).
+
+    def __init__(self, weakcontainer):
+        # Don't create cycles
+        self.weakcontainer = ref(weakcontainer)
+
+    def __enter__(self):
+        w = self.weakcontainer()
+        if w is not None:
+            w._iterating.add(self)
+        return self
+
+    def __exit__(self, e, t, b):
+        w = self.weakcontainer()
+        if w is not None:
+            s = w._iterating
+            s.remove(self)
+            if not s:
+                w._commit_removals()
+
+
+class WeakSet:
+    def __init__(self, data=None):
+        self.data = set()
+        def _remove(item, selfref=ref(self)):
+            self = selfref()
+            if self is not None:
+                if self._iterating:
+                    self._pending_removals.append(item)
+                else:
+                    self.data.discard(item)
+        self._remove = _remove
+        # A list of keys to be removed
+        self._pending_removals = []
+        self._iterating = set()
+        if data is not None:
+            self.update(data)
+
+    def _commit_removals(self):
+        l = self._pending_removals
+        discard = self.data.discard
+        while l:
+            discard(l.pop())
+
+    def __iter__(self):
+        with _IterationGuard(self):
+            for itemref in self.data:
+                item = itemref()
+                if item is not None:
+                    yield item
+
+    def __len__(self):
+        return sum(x() is not None for x in self.data)
+
+    def __contains__(self, item):
+        try:
+            wr = ref(item)
+        except TypeError:
+            return False
+        return wr in self.data
+
+    def __reduce__(self):
+        return (self.__class__, (list(self),),
+                getattr(self, '__dict__', None))
+
+    def add(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.add(ref(item, self._remove))
+
+    def clear(self):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.clear()
+
+    def copy(self):
+        return self.__class__(self)
+
+    def pop(self):
+        if self._pending_removals:
+            self._commit_removals()
+        while True:
+            try:
+                itemref = self.data.pop()
+            except KeyError:
+                raise KeyError('pop from empty WeakSet')
+            item = itemref()
+            if item is not None:
+                return item
+
+    def remove(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.remove(ref(item))
+
+    def discard(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.discard(ref(item))
+
+    def update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if isinstance(other, self.__class__):
+            self.data.update(other.data)
+        else:
+            for element in other:
+                self.add(element)
+
+    def __ior__(self, other):
+        self.update(other)
+        return self
+
+    # Helper functions for simple delegating methods.
+    def _apply(self, other, method):
+        if not isinstance(other, self.__class__):
+            other = self.__class__(other)
+        newdata = method(other.data)
+        newset = self.__class__()
+        newset.data = newdata
+        return newset
+
+    def difference(self, other):
+        return self._apply(other, self.data.difference)
+    __sub__ = difference
+
+    def difference_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.difference_update(ref(item) for item in other)
+    def __isub__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.difference_update(ref(item) for item in other)
+        return self
+
+    def intersection(self, other):
+        return self._apply(other, self.data.intersection)
+    __and__ = intersection
+
+    def intersection_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.intersection_update(ref(item) for item in other)
+    def __iand__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.intersection_update(ref(item) for item in other)
+        return self
+
+    def issubset(self, other):
+        return self.data.issubset(ref(item) for item in other)
+    __lt__ = issubset
+
+    def __le__(self, other):
+        return self.data <= set(ref(item) for item in other)
+
+    def issuperset(self, other):
+        return self.data.issuperset(ref(item) for item in other)
+    __gt__ = issuperset
+
+    def __ge__(self, other):
+        return self.data >= set(ref(item) for item in other)
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return self.data == set(ref(item) for item in other)
+
+    def symmetric_difference(self, other):
+        return self._apply(other, self.data.symmetric_difference)
+    __xor__ = symmetric_difference
+
+    def symmetric_difference_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.symmetric_difference_update(ref(item) for item in other)
+    def __ixor__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.symmetric_difference_update(ref(item) for item in other)
+        return self
+
+    def union(self, other):
+        return self._apply(other, self.data.union)
+    __or__ = union
+
+    def isdisjoint(self, other):
+        return len(self.intersection(other)) == 0
diff --git a/lib-python/3.2/abc.py b/lib-python/3.2/abc.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/abc.py
@@ -0,0 +1,211 @@
+# Copyright 2007 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Abstract Base Classes (ABCs) according to PEP 3119."""
+
+from _weakrefset import WeakSet
+
+def abstractmethod(funcobj):
+    """A decorator indicating abstract methods.
+
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract methods are overridden.
+    The abstract methods can be called using any of the normal
+    'super' call mechanisms.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractmethod
+            def my_abstract_method(self, ...):
+                ...
+    """
+    funcobj.__isabstractmethod__ = True
+    return funcobj
+
+
+class abstractclassmethod(classmethod):
+    """A decorator indicating abstract classmethods.
+
+    Similar to abstractmethod.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractclassmethod
+            def my_abstract_classmethod(cls, ...):
+                ...
+    """
+
+    __isabstractmethod__ = True
+
+    def __init__(self, callable):
+        callable.__isabstractmethod__ = True
+        super().__init__(callable)
+
+
+class abstractstaticmethod(staticmethod):
+    """A decorator indicating abstract staticmethods.
+
+    Similar to abstractmethod.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractstaticmethod
+            def my_abstract_staticmethod(...):
+                ...
+    """
+
+    __isabstractmethod__ = True
+
+    def __init__(self, callable):
+        callable.__isabstractmethod__ = True
+        super().__init__(callable)
+
+
+class abstractproperty(property):
+    """A decorator indicating abstract properties.
+
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract properties are overridden.
+    The abstract properties can be called using any of the normal
+    'super' call mechanisms.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractproperty
+            def my_abstract_property(self):
+                ...
+
+    This defines a read-only property; you can also define a read-write
+    abstract property using the 'long' form of property declaration:
+
+        class C(metaclass=ABCMeta):
+            def getx(self): ...
+            def setx(self, value): ...
+            x = abstractproperty(getx, setx)
+    """
+    __isabstractmethod__ = True
+
+
+class ABCMeta(type):
+
+    """Metaclass for defining Abstract Base Classes (ABCs).
+
+    Use this metaclass to create an ABC.  An ABC can be subclassed
+    directly, and then acts as a mix-in class.  You can also register
+    unrelated concrete classes (even built-in classes) and unrelated
+    ABCs as 'virtual subclasses' -- these and their descendants will
+    be considered subclasses of the registering ABC by the built-in
+    issubclass() function, but the registering ABC won't show up in
+    their MRO (Method Resolution Order) nor will method
+    implementations defined by the registering ABC be callable (not
+    even via super()).
+
+    """
+
+    # A global counter that is incremented each time a class is
+    # registered as a virtual subclass of anything.  It forces the
+    # negative cache to be cleared before its next use.
+    _abc_invalidation_counter = 0
+
+    def __new__(mcls, name, bases, namespace):
+        cls = super().__new__(mcls, name, bases, namespace)
+        # Compute set of abstract method names
+        abstracts = {name
+                     for name, value in namespace.items()
+                     if getattr(value, "__isabstractmethod__", False)}
+        for base in bases:
+            for name in getattr(base, "__abstractmethods__", set()):
+                value = getattr(cls, name, None)
+                if getattr(value, "__isabstractmethod__", False):
+                    abstracts.add(name)
+        cls.__abstractmethods__ = frozenset(abstracts)
+        # Set up inheritance registry
+        cls._abc_registry = WeakSet()
+        cls._abc_cache = WeakSet()
+        cls._abc_negative_cache = WeakSet()
+        cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+        return cls
+
+    def register(cls, subclass):
+        """Register a virtual subclass of an ABC."""
+        if not isinstance(subclass, type):
+            raise TypeError("Can only register classes")
+        if issubclass(subclass, cls):
+            return  # Already a subclass
+        # Subtle: test for cycles *after* testing for "already a subclass";
+        # this means we allow X.register(X) and interpret it as a no-op.
+        if issubclass(cls, subclass):
+            # This would create a cycle, which is bad for the algorithm below
+            raise RuntimeError("Refusing to create an inheritance cycle")
+        cls._abc_registry.add(subclass)
+        ABCMeta._abc_invalidation_counter += 1  # Invalidate negative cache
+
+    def _dump_registry(cls, file=None):
+        """Debug helper to print the ABC registry."""
+        print("Class: %s.%s" % (cls.__module__, cls.__name__), file=file)
+        print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file)
+        for name in sorted(cls.__dict__.keys()):
+            if name.startswith("_abc_"):
+                value = getattr(cls, name)
+                print("%s: %r" % (name, value), file=file)
+
+    def __instancecheck__(cls, instance):
+        """Override for isinstance(instance, cls)."""
+        # Inline the cache checking
+        subclass = instance.__class__
+        if subclass in cls._abc_cache:
+            return True
+        subtype = type(instance)
+        if subtype is subclass:
+            if (cls._abc_negative_cache_version ==
+                ABCMeta._abc_invalidation_counter and
+                subclass in cls._abc_negative_cache):
+                return False
+            # Fall back to the subclass check.
+            return cls.__subclasscheck__(subclass)
+        return any(cls.__subclasscheck__(c) for c in {subclass, subtype})
+
+    def __subclasscheck__(cls, subclass):
+        """Override for issubclass(subclass, cls)."""
+        # Check cache
+        if subclass in cls._abc_cache:
+            return True
+        # Check negative cache; may have to invalidate
+        if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter:
+            # Invalidate the negative cache
+            cls._abc_negative_cache = WeakSet()
+            cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+        elif subclass in cls._abc_negative_cache:
+            return False
+        # Check the subclass hook
+        ok = cls.__subclasshook__(subclass)
+        if ok is not NotImplemented:
+            assert isinstance(ok, bool)
+            if ok:
+                cls._abc_cache.add(subclass)
+            else:
+                cls._abc_negative_cache.add(subclass)
+            return ok
+        # Check if it's a direct subclass
+        if cls in getattr(subclass, '__mro__', ()):
+            cls._abc_cache.add(subclass)
+            return True
+        # Check if it's a subclass of a registered class (recursive)
+        for rcls in cls._abc_registry:
+            if issubclass(subclass, rcls):
+                cls._abc_cache.add(subclass)
+                return True
+        # Check if it's a subclass of a subclass (recursive)
+        for scls in cls.__subclasses__():
+            if issubclass(subclass, scls):
+                cls._abc_cache.add(subclass)
+                return True
+        # No dice; update negative cache
+        cls._abc_negative_cache.add(subclass)
+        return False
diff --git a/lib-python/3.2/aifc.py b/lib-python/3.2/aifc.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/aifc.py
@@ -0,0 +1,891 @@
+"""Stuff to parse AIFF-C and AIFF files.
+
+Unless explicitly stated otherwise, the description below is true
+both for AIFF-C files and AIFF files.
+
+An AIFF-C file has the following structure.
+
+  +-----------------+
+  | FORM            |
+  +-----------------+
+  | <size>          |
+  +----+------------+
+  |    | AIFC       |
+  |    +------------+
+  |    | <chunks>   |
+  |    |    .       |
+  |    |    .       |
+  |    |    .       |
+  +----+------------+
+
+An AIFF file has the string "AIFF" instead of "AIFC".
+
+A chunk consists of an identifier (4 bytes) followed by a size (4 bytes,
+big endian order), followed by the data.  The size field does not include
+the size of the 8 byte header.
+
+The following chunk types are recognized.
+
+  FVER
+      <version number of AIFF-C defining document> (AIFF-C only).
+  MARK
+      <# of markers> (2 bytes)
+      list of markers:
+          <marker ID> (2 bytes, must be > 0)
+          <position> (4 bytes)
+          <marker name> ("pstring")
+  COMM
+      <# of channels> (2 bytes)
+      <# of sound frames> (4 bytes)
+      <size of the samples> (2 bytes)
+      <sampling frequency> (10 bytes, IEEE 80-bit extended
+          floating point)
+      in AIFF-C files only:
+      <compression type> (4 bytes)
+      <human-readable version of compression type> ("pstring")
+  SSND
+      <offset> (4 bytes, not used by this program)
+      <blocksize> (4 bytes, not used by this program)
+      <sound data>
+
+A pstring consists of 1 byte length, a string of characters, and 0 or 1
+byte pad to make the total length even.
+
+Usage.
+
+Reading AIFF files:
+  f = aifc.open(file, 'r')
+where file is either the name of a file or an open file pointer.
+The open file pointer must have methods read(), seek(), and close().
+In some types of audio files, if the setpos() method is not used,
+the seek() method is not necessary.
+
+This returns an instance of a class with the following public methods:
+  getnchannels()  -- returns number of audio channels (1 for
+             mono, 2 for stereo)
+  getsampwidth()  -- returns sample width in bytes
+  getframerate()  -- returns sampling frequency
+  getnframes()    -- returns number of audio frames
+  getcomptype()   -- returns compression type ('NONE' for AIFF files)
+  getcompname()   -- returns human-readable version of
+             compression type ('not compressed' for AIFF files)
+  getparams() -- returns a tuple consisting of all of the
+             above in the above order
+  getmarkers()    -- get the list of marks in the audio file or None
+             if there are no marks
+  getmark(id) -- get mark with the specified id (raises an error
+             if the mark does not exist)
+  readframes(n)   -- returns at most n frames of audio
+  rewind()    -- rewind to the beginning of the audio stream
+  setpos(pos) -- seek to the specified position
+  tell()      -- return the current position
+  close()     -- close the instance (make it unusable)
+The position returned by tell(), the position given to setpos() and
+the position of marks are all compatible and have nothing to do with
+the actual position in the file.
+The close() method is called automatically when the class instance
+is destroyed.
+
+Writing AIFF files:
+  f = aifc.open(file, 'w')
+where file is either the name of a file or an open file pointer.
+The open file pointer must have methods write(), tell(), seek(), and
+close().
+
+This returns an instance of a class with the following public methods:
+  aiff()      -- create an AIFF file (AIFF-C default)
+  aifc()      -- create an AIFF-C file
+  setnchannels(n) -- set the number of channels
+  setsampwidth(n) -- set the sample width
+  setframerate(n) -- set the frame rate
+  setnframes(n)   -- set the number of frames
+  setcomptype(type, name)
+          -- set the compression type and the
+             human-readable compression type
+  setparams(tuple)
+          -- set all parameters at once
+  setmark(id, pos, name)
+          -- add specified mark to the list of marks
+  tell()      -- return current position in output file (useful
+             in combination with setmark())
+  writeframesraw(data)
+          -- write audio frames without pathing up the
+             file header
+  writeframes(data)
+          -- write audio frames and patch up the file header
+  close()     -- patch up the file header and close the
+             output file
+You should set the parameters before the first writeframesraw or
+writeframes.  The total number of frames does not need to be set,
+but when it is set to the correct value, the header does not have to
+be patched up.
+It is best to first set all parameters, perhaps possibly the
+compression type, and then write audio frames using writeframesraw.
+When all frames have been written, either call writeframes('') or
+close() to patch up the sizes in the header.
+Marks can be added anytime.  If there are any marks, ypu must call
+close() after all frames have been written.
+The close() method is called automatically when the class instance
+is destroyed.
+
+When a file is opened with the extension '.aiff', an AIFF file is
+written, otherwise an AIFF-C file is written.  This default can be
+changed by calling aiff() or aifc() before the first writeframes or
+writeframesraw.
+"""
+
+import struct
+import builtins
+
+__all__ = ["Error", "open", "openfp"]
+
+class Error(Exception):
+    pass
+
+_AIFC_version = 0xA2805140     # Version 1 of AIFF-C
+
+def _read_long(file):
+    try:
+        return struct.unpack('>l', file.read(4))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_ulong(file):
+    try:
+        return struct.unpack('>L', file.read(4))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_short(file):
+    try:
+        return struct.unpack('>h', file.read(2))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_ushort(file):
+    try:
+        return struct.unpack('>H', file.read(2))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_string(file):
+    length = ord(file.read(1))
+    if length == 0:
+        data = b''
+    else:
+        data = file.read(length)
+    if length & 1 == 0:
+        dummy = file.read(1)
+    return data
+
+_HUGE_VAL = 1.79769313486231e+308 # See <limits.h>
+
+def _read_float(f): # 10 bytes
+    expon = _read_short(f) # 2 bytes
+    sign = 1
+    if expon < 0:
+        sign = -1
+        expon = expon + 0x8000
+    himant = _read_ulong(f) # 4 bytes
+    lomant = _read_ulong(f) # 4 bytes
+    if expon == himant == lomant == 0:
+        f = 0.0
+    elif expon == 0x7FFF:
+        f = _HUGE_VAL
+    else:
+        expon = expon - 16383
+        f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63)
+    return sign * f
+
+def _write_short(f, x):
+    f.write(struct.pack('>h', x))
+
+def _write_ushort(f, x):
+    f.write(struct.pack('>H', x))
+
+def _write_long(f, x):
+    f.write(struct.pack('>l', x))
+
+def _write_ulong(f, x):
+    f.write(struct.pack('>L', x))
+
+def _write_string(f, s):
+    if len(s) > 255:
+        raise ValueError("string exceeds maximum pstring length")
+    f.write(struct.pack('B', len(s)))
+    f.write(s)
+    if len(s) & 1 == 0:
+        f.write(b'\x00')
+
+def _write_float(f, x):
+    import math
+    if x < 0:
+        sign = 0x8000
+        x = x * -1
+    else:
+        sign = 0
+    if x == 0:
+        expon = 0
+        himant = 0
+        lomant = 0
+    else:
+        fmant, expon = math.frexp(x)
+        if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN
+            expon = sign|0x7FFF
+            himant = 0
+            lomant = 0
+        else:                   # Finite
+            expon = expon + 16382
+            if expon < 0:           # denormalized
+                fmant = math.ldexp(fmant, expon)
+                expon = 0
+            expon = expon | sign
+            fmant = math.ldexp(fmant, 32)
+            fsmant = math.floor(fmant)
+            himant = int(fsmant)
+            fmant = math.ldexp(fmant - fsmant, 32)
+            fsmant = math.floor(fmant)
+            lomant = int(fsmant)
+    _write_ushort(f, expon)
+    _write_ulong(f, himant)
+    _write_ulong(f, lomant)
+
+from chunk import Chunk
+
+class Aifc_read:
+    # Variables used in this class:
+    #
+    # These variables are available to the user though appropriate
+    # methods of this class:
+    # _file -- the open file with methods read(), close(), and seek()
+    #       set through the __init__() method
+    # _nchannels -- the number of audio channels
+    #       available through the getnchannels() method
+    # _nframes -- the number of audio frames
+    #       available through the getnframes() method
+    # _sampwidth -- the number of bytes per audio sample
+    #       available through the getsampwidth() method
+    # _framerate -- the sampling frequency
+    #       available through the getframerate() method
+    # _comptype -- the AIFF-C compression type ('NONE' if AIFF)
+    #       available through the getcomptype() method
+    # _compname -- the human-readable AIFF-C compression type
+    #       available through the getcomptype() method
+    # _markers -- the marks in the audio file
+    #       available through the getmarkers() and getmark()
+    #       methods
+    # _soundpos -- the position in the audio stream
+    #       available through the tell() method, set through the
+    #       setpos() method
+    #
+    # These variables are used internally only:
+    # _version -- the AIFF-C version number
+    # _decomp -- the decompressor from builtin module cl
+    # _comm_chunk_read -- 1 iff the COMM chunk has been read
+    # _aifc -- 1 iff reading an AIFF-C file
+    # _ssnd_seek_needed -- 1 iff positioned correctly in audio
+    #       file for readframes()
+    # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
+    # _framesize -- size of one frame in the file
+
+    def initfp(self, file):
+        self._version = 0
+        self._convert = None
+        self._markers = []
+        self._soundpos = 0
+        self._file = file
+        chunk = Chunk(file)
+        if chunk.getname() != b'FORM':
+            raise Error('file does not start with FORM id')
+        formdata = chunk.read(4)
+        if formdata == b'AIFF':
+            self._aifc = 0
+        elif formdata == b'AIFC':
+            self._aifc = 1
+        else:
+            raise Error('not an AIFF or AIFF-C file')
+        self._comm_chunk_read = 0
+        while 1:
+            self._ssnd_seek_needed = 1
+            try:
+                chunk = Chunk(self._file)
+            except EOFError:
+                break
+            chunkname = chunk.getname()
+            if chunkname == b'COMM':
+                self._read_comm_chunk(chunk)
+                self._comm_chunk_read = 1
+            elif chunkname == b'SSND':
+                self._ssnd_chunk = chunk
+                dummy = chunk.read(8)
+                self._ssnd_seek_needed = 0
+            elif chunkname == b'FVER':
+                self._version = _read_ulong(chunk)
+            elif chunkname == b'MARK':
+                self._readmark(chunk)
+            chunk.skip()
+        if not self._comm_chunk_read or not self._ssnd_chunk:
+            raise Error('COMM chunk and/or SSND chunk missing')
+
+    def __init__(self, f):
+        if isinstance(f, str):
+            f = builtins.open(f, 'rb')
+        # else, assume it is an open file object already
+        self.initfp(f)
+
+    #
+    # User visible methods.
+    #
+    def getfp(self):
+        return self._file
+
+    def rewind(self):
+        self._ssnd_seek_needed = 1
+        self._soundpos = 0
+
+    def close(self):
+        self._file.close()
+
+    def tell(self):
+        return self._soundpos
+
+    def getnchannels(self):
+        return self._nchannels
+
+    def getnframes(self):
+        return self._nframes
+
+    def getsampwidth(self):
+        return self._sampwidth
+
+    def getframerate(self):
+        return self._framerate
+
+    def getcomptype(self):
+        return self._comptype
+
+    def getcompname(self):
+        return self._compname
+
+##  def getversion(self):
+##      return self._version
+
+    def getparams(self):
+        return self.getnchannels(), self.getsampwidth(), \
+              self.getframerate(), self.getnframes(), \
+              self.getcomptype(), self.getcompname()
+
+    def getmarkers(self):
+        if len(self._markers) == 0:
+            return None
+        return self._markers
+
+    def getmark(self, id):
+        for marker in self._markers:
+            if id == marker[0]:
+                return marker
+        raise Error('marker {0!r} does not exist'.format(id))
+
+    def setpos(self, pos):
+        if pos < 0 or pos > self._nframes:
+            raise Error('position not in range')
+        self._soundpos = pos
+        self._ssnd_seek_needed = 1
+
+    def readframes(self, nframes):
+        if self._ssnd_seek_needed:
+            self._ssnd_chunk.seek(0)
+            dummy = self._ssnd_chunk.read(8)
+            pos = self._soundpos * self._framesize
+            if pos:
+                self._ssnd_chunk.seek(pos + 8)
+            self._ssnd_seek_needed = 0
+        if nframes == 0:
+            return b''
+        data = self._ssnd_chunk.read(nframes * self._framesize)
+        if self._convert and data:
+            data = self._convert(data)
+        self._soundpos = self._soundpos + len(data) // (self._nchannels
+                                                        * self._sampwidth)
+        return data
+
+    #
+    # Internal methods.
+    #
+
+    def _alaw2lin(self, data):
+        import audioop
+        return audioop.alaw2lin(data, 2)
+
+    def _ulaw2lin(self, data):
+        import audioop
+        return audioop.ulaw2lin(data, 2)
+
+    def _adpcm2lin(self, data):
+        import audioop
+        if not hasattr(self, '_adpcmstate'):
+            # first time
+            self._adpcmstate = None
+        data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
+        return data
+
+    def _read_comm_chunk(self, chunk):
+        self._nchannels = _read_short(chunk)
+        self._nframes = _read_long(chunk)
+        self._sampwidth = (_read_short(chunk) + 7) // 8
+        self._framerate = int(_read_float(chunk))
+        self._framesize = self._nchannels * self._sampwidth
+        if self._aifc:
+            #DEBUG: SGI's soundeditor produces a bad size :-(
+            kludge = 0
+            if chunk.chunksize == 18:
+                kludge = 1
+                print('Warning: bad COMM chunk size')
+                chunk.chunksize = 23
+            #DEBUG end
+            self._comptype = chunk.read(4)
+            #DEBUG start
+            if kludge:
+                length = ord(chunk.file.read(1))
+                if length & 1 == 0:
+                    length = length + 1
+                chunk.chunksize = chunk.chunksize + length
+                chunk.file.seek(-1, 1)
+            #DEBUG end
+            self._compname = _read_string(chunk)
+            if self._comptype != b'NONE':
+                if self._comptype == b'G722':
+                    self._convert = self._adpcm2lin
+                    self._framesize = self._framesize // 4
+                elif self._comptype in (b'ulaw', b'ULAW'):
+                    self._convert = self._ulaw2lin
+                    self._framesize = self._framesize // 2
+                elif self._comptype in (b'alaw', b'ALAW'):
+                    self._convert = self._alaw2lin
+                    self._framesize = self._framesize // 2
+                else:
+                    raise Error('unsupported compression type')
+        else:
+            self._comptype = b'NONE'
+            self._compname = b'not compressed'
+
+    def _readmark(self, chunk):
+        nmarkers = _read_short(chunk)
+        # Some files appear to contain invalid counts.
+        # Cope with this by testing for EOF.
+        try:
+            for i in range(nmarkers):
+                id = _read_short(chunk)
+                pos = _read_long(chunk)
+                name = _read_string(chunk)
+                if pos or name:
+                    # some files appear to have
+                    # dummy markers consisting of
+                    # a position 0 and name ''
+                    self._markers.append((id, pos, name))
+        except EOFError:
+            print('Warning: MARK chunk contains only', end=' ')
+            print(len(self._markers), end=' ')
+            if len(self._markers) == 1: print('marker', end=' ')
+            else: print('markers', end=' ')
+            print('instead of', nmarkers)
+
+class Aifc_write:
+    # Variables used in this class:
+    #
+    # These variables are user settable through appropriate methods
+    # of this class:
+    # _file -- the open file with methods write(), close(), tell(), seek()
+    #       set through the __init__() method
+    # _comptype -- the AIFF-C compression type ('NONE' in AIFF)
+    #       set through the setcomptype() or setparams() method
+    # _compname -- the human-readable AIFF-C compression type
+    #       set through the setcomptype() or setparams() method
+    # _nchannels -- the number of audio channels
+    #       set through the setnchannels() or setparams() method
+    # _sampwidth -- the number of bytes per audio sample
+    #       set through the setsampwidth() or setparams() method
+    # _framerate -- the sampling frequency
+    #       set through the setframerate() or setparams() method
+    # _nframes -- the number of audio frames written to the header
+    #       set through the setnframes() or setparams() method
+    # _aifc -- whether we're writing an AIFF-C file or an AIFF file
+    #       set through the aifc() method, reset through the
+    #       aiff() method
+    #
+    # These variables are used internally only:
+    # _version -- the AIFF-C version number
+    # _comp -- the compressor from builtin module cl
+    # _nframeswritten -- the number of audio frames actually written
+    # _datalength -- the size of the audio samples written to the header
+    # _datawritten -- the size of the audio samples actually written
+
+    def __init__(self, f):
+        if isinstance(f, str):
+            filename = f
+            f = builtins.open(f, 'wb')
+        else:
+            # else, assume it is an open file object already
+            filename = '???'
+        self.initfp(f)
+        if filename[-5:] == '.aiff':
+            self._aifc = 0
+        else:
+            self._aifc = 1
+
+    def initfp(self, file):
+        self._file = file
+        self._version = _AIFC_version
+        self._comptype = b'NONE'
+        self._compname = b'not compressed'
+        self._convert = None
+        self._nchannels = 0
+        self._sampwidth = 0
+        self._framerate = 0
+        self._nframes = 0
+        self._nframeswritten = 0
+        self._datawritten = 0
+        self._datalength = 0
+        self._markers = []
+        self._marklength = 0
+        self._aifc = 1      # AIFF-C is default
+
+    def __del__(self):
+        self.close()
+
+    #
+    # User visible methods.
+    #
+    def aiff(self):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._aifc = 0
+
+    def aifc(self):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._aifc = 1
+
+    def setnchannels(self, nchannels):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if nchannels < 1:
+            raise Error('bad # of channels')
+        self._nchannels = nchannels
+
+    def getnchannels(self):
+        if not self._nchannels:
+            raise Error('number of channels not set')
+        return self._nchannels
+
+    def setsampwidth(self, sampwidth):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if sampwidth < 1 or sampwidth > 4:
+            raise Error('bad sample width')
+        self._sampwidth = sampwidth
+
+    def getsampwidth(self):
+        if not self._sampwidth:
+            raise Error('sample width not set')
+        return self._sampwidth
+
+    def setframerate(self, framerate):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if framerate <= 0:
+            raise Error('bad frame rate')
+        self._framerate = framerate
+
+    def getframerate(self):
+        if not self._framerate:
+            raise Error('frame rate not set')
+        return self._framerate
+
+    def setnframes(self, nframes):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._nframes = nframes
+
+    def getnframes(self):
+        return self._nframeswritten
+
+    def setcomptype(self, comptype, compname):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if comptype not in (b'NONE', b'ulaw', b'ULAW',
+                            b'alaw', b'ALAW', b'G722'):
+            raise Error('unsupported compression type')
+        self._comptype = comptype
+        self._compname = compname
+
+    def getcomptype(self):
+        return self._comptype
+
+    def getcompname(self):
+        return self._compname
+
+##  def setversion(self, version):
+##      if self._nframeswritten:
+##          raise Error, 'cannot change parameters after starting to write'
+##      self._version = version
+
+    def setparams(self, params):
+        nchannels, sampwidth, framerate, nframes, comptype, compname = params
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if comptype not in (b'NONE', b'ulaw', b'ULAW',
+                            b'alaw', b'ALAW', b'G722'):
+            raise Error('unsupported compression type')
+        self.setnchannels(nchannels)
+        self.setsampwidth(sampwidth)
+        self.setframerate(framerate)
+        self.setnframes(nframes)
+        self.setcomptype(comptype, compname)
+
+    def getparams(self):
+        if not self._nchannels or not self._sampwidth or not self._framerate:
+            raise Error('not all parameters set')
+        return self._nchannels, self._sampwidth, self._framerate, \
+              self._nframes, self._comptype, self._compname
+
+    def setmark(self, id, pos, name):
+        if id <= 0:
+            raise Error('marker ID must be > 0')
+        if pos < 0:
+            raise Error('marker position must be >= 0')
+        if not isinstance(name, bytes):
+            raise Error('marker name must be bytes')
+        for i in range(len(self._markers)):
+            if id == self._markers[i][0]:
+                self._markers[i] = id, pos, name
+                return
+        self._markers.append((id, pos, name))
+
+    def getmark(self, id):
+        for marker in self._markers:
+            if id == marker[0]:
+                return marker
+        raise Error('marker {0!r} does not exist'.format(id))
+
+    def getmarkers(self):
+        if len(self._markers) == 0:
+            return None
+        return self._markers
+
+    def tell(self):
+        return self._nframeswritten
+
+    def writeframesraw(self, data):
+        self._ensure_header_written(len(data))
+        nframes = len(data) // (self._sampwidth * self._nchannels)
+        if self._convert:
+            data = self._convert(data)
+        self._file.write(data)
+        self._nframeswritten = self._nframeswritten + nframes
+        self._datawritten = self._datawritten + len(data)
+
+    def writeframes(self, data):
+        self.writeframesraw(data)
+        if self._nframeswritten != self._nframes or \
+              self._datalength != self._datawritten:
+            self._patchheader()
+
+    def close(self):
+        if self._file:
+            self._ensure_header_written(0)
+            if self._datawritten & 1:
+                # quick pad to even size
+                self._file.write(b'\x00')
+                self._datawritten = self._datawritten + 1
+            self._writemarkers()
+            if self._nframeswritten != self._nframes or \
+                  self._datalength != self._datawritten or \
+                  self._marklength:
+                self._patchheader()
+            # Prevent ref cycles
+            self._convert = None
+            self._file.close()
+            self._file = None
+
+    #
+    # Internal methods.
+    #
+
+    def _lin2alaw(self, data):
+        import audioop
+        return audioop.lin2alaw(data, 2)
+
+    def _lin2ulaw(self, data):
+        import audioop
+        return audioop.lin2ulaw(data, 2)
+
+    def _lin2adpcm(self, data):
+        import audioop
+        if not hasattr(self, '_adpcmstate'):
+            self._adpcmstate = None
+        data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
+        return data
+
+    def _ensure_header_written(self, datasize):
+        if not self._nframeswritten:
+            if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
+                if not self._sampwidth:
+                    self._sampwidth = 2
+                if self._sampwidth != 2:
+                    raise Error('sample width must be 2 when compressing '
+                                'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
+            if not self._nchannels:
+                raise Error('# channels not specified')
+            if not self._sampwidth:
+                raise Error('sample width not specified')
+            if not self._framerate:
+                raise Error('sampling rate not specified')
+            self._write_header(datasize)
+
+    def _init_compression(self):
+        if self._comptype == b'G722':
+            self._convert = self._lin2adpcm
+        elif self._comptype in (b'ulaw', b'ULAW'):
+            self._convert = self._lin2ulaw
+        elif self._comptype in (b'alaw', b'ALAW'):
+            self._convert = self._lin2alaw
+
+    def _write_header(self, initlength):
+        if self._aifc and self._comptype != b'NONE':
+            self._init_compression()
+        self._file.write(b'FORM')
+        if not self._nframes:
+            self._nframes = initlength // (self._nchannels * self._sampwidth)
+        self._datalength = self._nframes * self._nchannels * self._sampwidth
+        if self._datalength & 1:
+            self._datalength = self._datalength + 1
+        if self._aifc:
+            if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'):
+                self._datalength = self._datalength // 2
+                if self._datalength & 1:
+                    self._datalength = self._datalength + 1
+            elif self._comptype == b'G722':
+                self._datalength = (self._datalength + 3) // 4
+                if self._datalength & 1:
+                    self._datalength = self._datalength + 1
+        self._form_length_pos = self._file.tell()
+        commlength = self._write_form_length(self._datalength)
+        if self._aifc:
+            self._file.write(b'AIFC')
+            self._file.write(b'FVER')
+            _write_ulong(self._file, 4)
+            _write_ulong(self._file, self._version)
+        else:
+            self._file.write(b'AIFF')
+        self._file.write(b'COMM')
+        _write_ulong(self._file, commlength)
+        _write_short(self._file, self._nchannels)
+        self._nframes_pos = self._file.tell()
+        _write_ulong(self._file, self._nframes)
+        _write_short(self._file, self._sampwidth * 8)
+        _write_float(self._file, self._framerate)
+        if self._aifc:
+            self._file.write(self._comptype)
+            _write_string(self._file, self._compname)
+        self._file.write(b'SSND')
+        self._ssnd_length_pos = self._file.tell()
+        _write_ulong(self._file, self._datalength + 8)
+        _write_ulong(self._file, 0)
+        _write_ulong(self._file, 0)
+
+    def _write_form_length(self, datalength):
+        if self._aifc:
+            commlength = 18 + 5 + len(self._compname)
+            if commlength & 1:
+                commlength = commlength + 1
+            verslength = 12
+        else:
+            commlength = 18
+            verslength = 0
+        _write_ulong(self._file, 4 + verslength + self._marklength + \
+                     8 + commlength + 16 + datalength)
+        return commlength
+
+    def _patchheader(self):
+        curpos = self._file.tell()
+        if self._datawritten & 1:
+            datalength = self._datawritten + 1
+            self._file.write(b'\x00')
+        else:
+            datalength = self._datawritten
+        if datalength == self._datalength and \
+              self._nframes == self._nframeswritten and \
+              self._marklength == 0:
+            self._file.seek(curpos, 0)
+            return
+        self._file.seek(self._form_length_pos, 0)
+        dummy = self._write_form_length(datalength)
+        self._file.seek(self._nframes_pos, 0)
+        _write_ulong(self._file, self._nframeswritten)
+        self._file.seek(self._ssnd_length_pos, 0)
+        _write_ulong(self._file, datalength + 8)
+        self._file.seek(curpos, 0)
+        self._nframes = self._nframeswritten
+        self._datalength = datalength
+
+    def _writemarkers(self):
+        if len(self._markers) == 0:
+            return
+        self._file.write(b'MARK')
+        length = 2
+        for marker in self._markers:
+            id, pos, name = marker
+            length = length + len(name) + 1 + 6
+            if len(name) & 1 == 0:
+                length = length + 1
+        _write_ulong(self._file, length)
+        self._marklength = length + 8
+        _write_short(self._file, len(self._markers))
+        for marker in self._markers:
+            id, pos, name = marker
+            _write_short(self._file, id)
+            _write_ulong(self._file, pos)
+            _write_string(self._file, name)
+
+def open(f, mode=None):
+    if mode is None:
+        if hasattr(f, 'mode'):
+            mode = f.mode
+        else:
+            mode = 'rb'
+    if mode in ('r', 'rb'):
+        return Aifc_read(f)
+    elif mode in ('w', 'wb'):
+        return Aifc_write(f)
+    else:
+        raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
+
+openfp = open # B/W compatibility
+
+if __name__ == '__main__':
+    import sys
+    if not sys.argv[1:]:
+        sys.argv.append('/usr/demos/data/audio/bach.aiff')
+    fn = sys.argv[1]
+    f = open(fn, 'r')
+    print("Reading", fn)
+    print("nchannels =", f.getnchannels())
+    print("nframes   =", f.getnframes())
+    print("sampwidth =", f.getsampwidth())
+    print("framerate =", f.getframerate())
+    print("comptype  =", f.getcomptype())
+    print("compname  =", f.getcompname())
+    if sys.argv[2:]:
+        gn = sys.argv[2]
+        print("Writing", gn)
+        g = open(gn, 'w')
+        g.setparams(f.getparams())
+        while 1:
+            data = f.readframes(1024)
+            if not data:
+                break
+            g.writeframes(data)
+        g.close()
+        f.close()
+        print("Done.")
diff --git a/lib-python/3.2/antigravity.py b/lib-python/3.2/antigravity.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/antigravity.py
@@ -0,0 +1,17 @@
+
+import webbrowser
+import hashlib
+
+webbrowser.open("http://xkcd.com/353/")
+
+def geohash(latitude, longitude, datedow):
+    '''Compute geohash() using the Munroe algorithm.
+
+    >>> geohash(37.421542, -122.085589, b'2005-05-26-10458.68')
+    37.857713 -122.544543
+
+    '''
+    # http://xkcd.com/426/
+    h = hashlib.md5(datedow).hexdigest()
+    p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
+    print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:]))
diff --git a/lib-python/3.2/argparse.py b/lib-python/3.2/argparse.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/argparse.py
@@ -0,0 +1,2362 @@
+# Author: Steven J. Bethard <steven.bethard at gmail.com>.
+
+"""Command-line parsing library
+
+This module is an optparse-inspired command-line parsing library that:
+
+    - handles both optional and positional arguments
+    - produces highly informative usage messages
+    - supports parsers that dispatch to sub-parsers
+
+The following is a simple usage example that sums integers from the
+command-line and writes the result to a file::
+
+    parser = argparse.ArgumentParser(
+        description='sum the integers at the command line')
+    parser.add_argument(
+        'integers', metavar='int', nargs='+', type=int,
+        help='an integer to be summed')
+    parser.add_argument(
+        '--log', default=sys.stdout, type=argparse.FileType('w'),
+        help='the file where the sum should be written')
+    args = parser.parse_args()
+    args.log.write('%s' % sum(args.integers))
+    args.log.close()
+
+The module contains the following public classes:
+
+    - ArgumentParser -- The main entry point for command-line parsing. As the
+        example above shows, the add_argument() method is used to populate
+        the parser with actions for optional and positional arguments. Then
+        the parse_args() method is invoked to convert the args at the
+        command-line into an object with attributes.
+
+    - ArgumentError -- The exception raised by ArgumentParser objects when
+        there are errors with the parser's actions. Errors raised while
+        parsing the command-line are caught by ArgumentParser and emitted
+        as command-line messages.
+
+    - FileType -- A factory for defining types of files to be created. As the
+        example above shows, instances of FileType are typically passed as
+        the type= argument of add_argument() calls.
+
+    - Action -- The base class for parser actions. Typically actions are
+        selected by passing strings like 'store_true' or 'append_const' to
+        the action= argument of add_argument(). However, for greater
+        customization of ArgumentParser actions, subclasses of Action may
+        be defined and passed as the action= argument.
+
+    - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter,
+        ArgumentDefaultsHelpFormatter -- Formatter classes which
+        may be passed as the formatter_class= argument to the
+        ArgumentParser constructor. HelpFormatter is the default,
+        RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser
+        not to change the formatting for help text, and
+        ArgumentDefaultsHelpFormatter adds information about argument defaults
+        to the help.
+
+All other classes in this module are considered implementation details.
+(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
+considered public as object names -- the API of the formatter objects is
+still considered an implementation detail.)
+"""
+
+__version__ = '1.1'
+__all__ = [
+    'ArgumentParser',
+    'ArgumentError',
+    'ArgumentTypeError',
+    'FileType',
+    'HelpFormatter',
+    'ArgumentDefaultsHelpFormatter',
+    'RawDescriptionHelpFormatter',
+    'RawTextHelpFormatter',
+    'Namespace',
+    'Action',
+    'ONE_OR_MORE',
+    'OPTIONAL',
+    'PARSER',
+    'REMAINDER',
+    'SUPPRESS',
+    'ZERO_OR_MORE',
+]
+
+
+import collections as _collections
+import copy as _copy
+import os as _os
+import re as _re
+import sys as _sys
+import textwrap as _textwrap
+
+from gettext import gettext as _, ngettext
+
+
+SUPPRESS = '==SUPPRESS=='
+
+OPTIONAL = '?'
+ZERO_OR_MORE = '*'
+ONE_OR_MORE = '+'
+PARSER = 'A...'
+REMAINDER = '...'
+_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args'
+
+# =============================
+# Utility functions and classes
+# =============================
+
+class _AttributeHolder(object):
+    """Abstract base class that provides __repr__.
+
+    The __repr__ method returns a string in the format::
+        ClassName(attr=name, attr=name, ...)
+    The attributes are determined either by a class-level attribute,
+    '_kwarg_names', or by inspecting the instance __dict__.
+    """
+
+    def __repr__(self):
+        type_name = type(self).__name__
+        arg_strings = []
+        for arg in self._get_args():
+            arg_strings.append(repr(arg))
+        for name, value in self._get_kwargs():
+            arg_strings.append('%s=%r' % (name, value))
+        return '%s(%s)' % (type_name, ', '.join(arg_strings))
+
+    def _get_kwargs(self):
+        return sorted(self.__dict__.items())
+
+    def _get_args(self):
+        return []
+
+
+def _ensure_value(namespace, name, value):
+    if getattr(namespace, name, None) is None:
+        setattr(namespace, name, value)
+    return getattr(namespace, name)
+
+
+# ===============
+# Formatting Help
+# ===============
+
+class HelpFormatter(object):
+    """Formatter for generating usage messages and argument help strings.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def __init__(self,
+                 prog,
+                 indent_increment=2,
+                 max_help_position=24,
+                 width=None):
+
+        # default setting for width
+        if width is None:
+            try:
+                width = int(_os.environ['COLUMNS'])
+            except (KeyError, ValueError):
+                width = 80
+            width -= 2
+
+        self._prog = prog
+        self._indent_increment = indent_increment
+        self._max_help_position = max_help_position
+        self._width = width
+
+        self._current_indent = 0
+        self._level = 0
+        self._action_max_length = 0
+
+        self._root_section = self._Section(self, None)
+        self._current_section = self._root_section
+
+        self._whitespace_matcher = _re.compile(r'\s+')
+        self._long_break_matcher = _re.compile(r'\n\n\n+')
+
+    # ===============================
+    # Section and indentation methods
+    # ===============================
+    def _indent(self):
+        self._current_indent += self._indent_increment
+        self._level += 1
+
+    def _dedent(self):
+        self._current_indent -= self._indent_increment
+        assert self._current_indent >= 0, 'Indent decreased below 0.'
+        self._level -= 1
+
+    class _Section(object):
+
+        def __init__(self, formatter, parent, heading=None):
+            self.formatter = formatter
+            self.parent = parent
+            self.heading = heading
+            self.items = []
+
+        def format_help(self):
+            # format the indented section
+            if self.parent is not None:
+                self.formatter._indent()
+            join = self.formatter._join_parts
+            for func, args in self.items:
+                func(*args)
+            item_help = join([func(*args) for func, args in self.items])
+            if self.parent is not None:
+                self.formatter._dedent()
+
+            # return nothing if the section was empty
+            if not item_help:
+                return ''
+
+            # add the heading if the section was non-empty
+            if self.heading is not SUPPRESS and self.heading is not None:
+                current_indent = self.formatter._current_indent
+                heading = '%*s%s:\n' % (current_indent, '', self.heading)
+            else:
+                heading = ''
+
+            # join the section-initial newline, the heading and the help
+            return join(['\n', heading, item_help, '\n'])
+
+    def _add_item(self, func, args):
+        self._current_section.items.append((func, args))
+
+    # ========================
+    # Message building methods
+    # ========================
+    def start_section(self, heading):
+        self._indent()
+        section = self._Section(self, self._current_section, heading)
+        self._add_item(section.format_help, [])
+        self._current_section = section
+
+    def end_section(self):
+        self._current_section = self._current_section.parent
+        self._dedent()
+
+    def add_text(self, text):
+        if text is not SUPPRESS and text is not None:
+            self._add_item(self._format_text, [text])
+
+    def add_usage(self, usage, actions, groups, prefix=None):
+        if usage is not SUPPRESS:
+            args = usage, actions, groups, prefix
+            self._add_item(self._format_usage, args)
+
+    def add_argument(self, action):
+        if action.help is not SUPPRESS:
+
+            # find all invocations
+            get_invocation = self._format_action_invocation
+            invocations = [get_invocation(action)]
+            for subaction in self._iter_indented_subactions(action):
+                invocations.append(get_invocation(subaction))
+
+            # update the maximum item length
+            invocation_length = max([len(s) for s in invocations])
+            action_length = invocation_length + self._current_indent
+            self._action_max_length = max(self._action_max_length,
+                                          action_length)
+
+            # add the item to the list
+            self._add_item(self._format_action, [action])
+
+    def add_arguments(self, actions):
+        for action in actions:
+            self.add_argument(action)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+    def format_help(self):
+        help = self._root_section.format_help()
+        if help:
+            help = self._long_break_matcher.sub('\n\n', help)
+            help = help.strip('\n') + '\n'
+        return help
+
+    def _join_parts(self, part_strings):
+        return ''.join([part
+                        for part in part_strings
+                        if part and part is not SUPPRESS])
+
+    def _format_usage(self, usage, actions, groups, prefix):
+        if prefix is None:
+            prefix = _('usage: ')
+
+        # if usage is specified, use that
+        if usage is not None:
+            usage = usage % dict(prog=self._prog)
+
+        # if no optionals or positionals are available, usage is just prog
+        elif usage is None and not actions:
+            usage = '%(prog)s' % dict(prog=self._prog)
+
+        # if optionals and positionals are available, calculate usage
+        elif usage is None:
+            prog = '%(prog)s' % dict(prog=self._prog)
+
+            # split optionals from positionals
+            optionals = []
+            positionals = []
+            for action in actions:
+                if action.option_strings:
+                    optionals.append(action)
+                else:
+                    positionals.append(action)
+
+            # build full usage string
+            format = self._format_actions_usage
+            action_usage = format(optionals + positionals, groups)
+            usage = ' '.join([s for s in [prog, action_usage] if s])
+
+            # wrap the usage parts if it's too long
+            text_width = self._width - self._current_indent
+            if len(prefix) + len(usage) > text_width:
+
+                # break usage into wrappable parts
+                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
+                opt_usage = format(optionals, groups)
+                pos_usage = format(positionals, groups)
+                opt_parts = _re.findall(part_regexp, opt_usage)
+                pos_parts = _re.findall(part_regexp, pos_usage)
+                assert ' '.join(opt_parts) == opt_usage
+                assert ' '.join(pos_parts) == pos_usage
+
+                # helper for wrapping lines
+                def get_lines(parts, indent, prefix=None):
+                    lines = []
+                    line = []
+                    if prefix is not None:
+                        line_len = len(prefix) - 1
+                    else:
+                        line_len = len(indent) - 1
+                    for part in parts:
+                        if line_len + 1 + len(part) > text_width:
+                            lines.append(indent + ' '.join(line))
+                            line = []
+                            line_len = len(indent) - 1
+                        line.append(part)
+                        line_len += len(part) + 1
+                    if line:
+                        lines.append(indent + ' '.join(line))
+                    if prefix is not None:
+                        lines[0] = lines[0][len(indent):]
+                    return lines
+
+                # if prog is short, follow it with optionals or positionals
+                if len(prefix) + len(prog) <= 0.75 * text_width:
+                    indent = ' ' * (len(prefix) + len(prog) + 1)
+                    if opt_parts:
+                        lines = get_lines([prog] + opt_parts, indent, prefix)
+                        lines.extend(get_lines(pos_parts, indent))
+                    elif pos_parts:
+                        lines = get_lines([prog] + pos_parts, indent, prefix)
+                    else:
+                        lines = [prog]
+
+                # if prog is long, put it on its own line
+                else:
+                    indent = ' ' * len(prefix)
+                    parts = opt_parts + pos_parts
+                    lines = get_lines(parts, indent)
+                    if len(lines) > 1:
+                        lines = []
+                        lines.extend(get_lines(opt_parts, indent))
+                        lines.extend(get_lines(pos_parts, indent))
+                    lines = [prog] + lines
+
+                # join lines into usage
+                usage = '\n'.join(lines)
+
+        # prefix with 'usage:'
+        return '%s%s\n\n' % (prefix, usage)
+
+    def _format_actions_usage(self, actions, groups):
+        # find group indices and identify actions in groups
+        group_actions = set()
+        inserts = {}
+        for group in groups:
+            try:
+                start = actions.index(group._group_actions[0])
+            except ValueError:
+                continue
+            else:
+                end = start + len(group._group_actions)
+                if actions[start:end] == group._group_actions:
+                    for action in group._group_actions:
+                        group_actions.add(action)
+                    if not group.required:
+                        if start in inserts:
+                            inserts[start] += ' ['
+                        else:
+                            inserts[start] = '['
+                        inserts[end] = ']'
+                    else:
+                        if start in inserts:
+                            inserts[start] += ' ('
+                        else:
+                            inserts[start] = '('
+                        inserts[end] = ')'
+                    for i in range(start + 1, end):
+                        inserts[i] = '|'
+
+        # collect all actions format strings
+        parts = []
+        for i, action in enumerate(actions):
+
+            # suppressed arguments are marked with None
+            # remove | separators for suppressed arguments
+            if action.help is SUPPRESS:
+                parts.append(None)
+                if inserts.get(i) == '|':
+                    inserts.pop(i)
+                elif inserts.get(i + 1) == '|':
+                    inserts.pop(i + 1)
+
+            # produce all arg strings
+            elif not action.option_strings:
+                part = self._format_args(action, action.dest)
+
+                # if it's in a group, strip the outer []
+                if action in group_actions:
+                    if part[0] == '[' and part[-1] == ']':
+                        part = part[1:-1]
+
+                # add the action string to the list
+                parts.append(part)
+
+            # produce the first way to invoke the option in brackets
+            else:
+                option_string = action.option_strings[0]
+
+                # if the Optional doesn't take a value, format is:
+                #    -s or --long
+                if action.nargs == 0:
+                    part = '%s' % option_string
+
+                # if the Optional takes a value, format is:
+                #    -s ARGS or --long ARGS
+                else:
+                    default = action.dest.upper()
+                    args_string = self._format_args(action, default)
+                    part = '%s %s' % (option_string, args_string)
+
+                # make it look optional if it's not required or in a group
+                if not action.required and action not in group_actions:
+                    part = '[%s]' % part
+
+                # add the action string to the list
+                parts.append(part)
+
+        # insert things at the necessary indices
+        for i in sorted(inserts, reverse=True):
+            parts[i:i] = [inserts[i]]
+
+        # join all the action items with spaces
+        text = ' '.join([item for item in parts if item is not None])
+
+        # clean up separators for mutually exclusive groups
+        open = r'[\[(]'
+        close = r'[\])]'
+        text = _re.sub(r'(%s) ' % open, r'\1', text)
+        text = _re.sub(r' (%s)' % close, r'\1', text)
+        text = _re.sub(r'%s *%s' % (open, close), r'', text)
+        text = _re.sub(r'\(([^|]*)\)', r'\1', text)
+        text = text.strip()
+
+        # return the text
+        return text
+
+    def _format_text(self, text):
+        if '%(prog)' in text:
+            text = text % dict(prog=self._prog)
+        text_width = self._width - self._current_indent
+        indent = ' ' * self._current_indent
+        return self._fill_text(text, text_width, indent) + '\n\n'
+
+    def _format_action(self, action):
+        # determine the required width and the entry label
+        help_position = min(self._action_max_length + 2,
+                            self._max_help_position)
+        help_width = self._width - help_position
+        action_width = help_position - self._current_indent - 2
+        action_header = self._format_action_invocation(action)
+
+        # ho nelp; start on same line and add a final newline
+        if not action.help:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+
+        # short action name; start on the same line and pad two spaces
+        elif len(action_header) <= action_width:
+            tup = self._current_indent, '', action_width, action_header
+            action_header = '%*s%-*s  ' % tup
+            indent_first = 0
+
+        # long action name; start on the next line
+        else:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+            indent_first = help_position
+
+        # collect the pieces of the action help
+        parts = [action_header]
+
+        # if there was help for the action, add lines of help text
+        if action.help:
+            help_text = self._expand_help(action)
+            help_lines = self._split_lines(help_text, help_width)
+            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
+            for line in help_lines[1:]:
+                parts.append('%*s%s\n' % (help_position, '', line))
+
+        # or add a newline if the description doesn't end with one
+        elif not action_header.endswith('\n'):
+            parts.append('\n')
+
+        # if there are any sub-actions, add their help as well
+        for subaction in self._iter_indented_subactions(action):
+            parts.append(self._format_action(subaction))
+
+        # return a single string
+        return self._join_parts(parts)
+
+    def _format_action_invocation(self, action):
+        if not action.option_strings:
+            metavar, = self._metavar_formatter(action, action.dest)(1)
+            return metavar
+
+        else:
+            parts = []
+
+            # if the Optional doesn't take a value, format is:
+            #    -s, --long
+            if action.nargs == 0:
+                parts.extend(action.option_strings)
+
+            # if the Optional takes a value, format is:
+            #    -s ARGS, --long ARGS
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    parts.append('%s %s' % (option_string, args_string))
+
+            return ', '.join(parts)
+
+    def _metavar_formatter(self, action, default_metavar):
+        if action.metavar is not None:
+            result = action.metavar
+        elif action.choices is not None:
+            choice_strs = [str(choice) for choice in action.choices]
+            result = '{%s}' % ','.join(choice_strs)
+        else:
+            result = default_metavar
+
+        def format(tuple_size):
+            if isinstance(result, tuple):
+                return result
+            else:
+                return (result, ) * tuple_size
+        return format
+
+    def _format_args(self, action, default_metavar):
+        get_metavar = self._metavar_formatter(action, default_metavar)
+        if action.nargs is None:
+            result = '%s' % get_metavar(1)
+        elif action.nargs == OPTIONAL:
+            result = '[%s]' % get_metavar(1)
+        elif action.nargs == ZERO_OR_MORE:
+            result = '[%s [%s ...]]' % get_metavar(2)
+        elif action.nargs == ONE_OR_MORE:
+            result = '%s [%s ...]' % get_metavar(2)
+        elif action.nargs == REMAINDER:
+            result = '...'
+        elif action.nargs == PARSER:
+            result = '%s ...' % get_metavar(1)
+        else:
+            formats = ['%s' for _ in range(action.nargs)]
+            result = ' '.join(formats) % get_metavar(action.nargs)
+        return result
+
+    def _expand_help(self, action):
+        params = dict(vars(action), prog=self._prog)
+        for name in list(params):
+            if params[name] is SUPPRESS:
+                del params[name]
+        for name in list(params):
+            if hasattr(params[name], '__name__'):
+                params[name] = params[name].__name__
+        if params.get('choices') is not None:
+            choices_str = ', '.join([str(c) for c in params['choices']])
+            params['choices'] = choices_str
+        return self._get_help_string(action) % params
+
+    def _iter_indented_subactions(self, action):
+        try:
+            get_subactions = action._get_subactions
+        except AttributeError:
+            pass
+        else:
+            self._indent()
+            for subaction in get_subactions():
+                yield subaction
+            self._dedent()
+
+    def _split_lines(self, text, width):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.wrap(text, width)
+
+    def _fill_text(self, text, width, indent):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.fill(text, width, initial_indent=indent,
+                                           subsequent_indent=indent)
+
+    def _get_help_string(self, action):
+        return action.help
+
+
+class RawDescriptionHelpFormatter(HelpFormatter):
+    """Help message formatter which retains any formatting in descriptions.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _fill_text(self, text, width, indent):
+        return ''.join([indent + line for line in text.splitlines(True)])
+
+
+class RawTextHelpFormatter(RawDescriptionHelpFormatter):
+    """Help message formatter which retains formatting of all help text.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _split_lines(self, text, width):
+        return text.splitlines()
+
+
+class ArgumentDefaultsHelpFormatter(HelpFormatter):
+    """Help message formatter which adds default values to argument help.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _get_help_string(self, action):
+        help = action.help
+        if '%(default)' not in action.help:
+            if action.default is not SUPPRESS:
+                defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
+                if action.option_strings or action.nargs in defaulting_nargs:
+                    help += ' (default: %(default)s)'
+        return help
+
+
+# =====================
+# Options and Arguments
+# =====================
+
+def _get_action_name(argument):
+    if argument is None:
+        return None
+    elif argument.option_strings:
+        return  '/'.join(argument.option_strings)
+    elif argument.metavar not in (None, SUPPRESS):
+        return argument.metavar
+    elif argument.dest not in (None, SUPPRESS):
+        return argument.dest
+    else:
+        return None
+
+
+class ArgumentError(Exception):
+    """An error from creating or using an argument (optional or positional).
+
+    The string value of this exception is the message, augmented with
+    information about the argument that caused it.
+    """
+
+    def __init__(self, argument, message):
+        self.argument_name = _get_action_name(argument)
+        self.message = message
+
+    def __str__(self):
+        if self.argument_name is None:
+            format = '%(message)s'
+        else:
+            format = 'argument %(argument_name)s: %(message)s'
+        return format % dict(message=self.message,
+                             argument_name=self.argument_name)
+
+
+class ArgumentTypeError(Exception):
+    """An error from trying to convert a command line string to a type."""
+    pass
+
+
+# ==============
+# Action classes
+# ==============
+
+class Action(_AttributeHolder):
+    """Information about how to convert command line strings to Python objects.
+
+    Action objects are used by an ArgumentParser to represent the information
+    needed to parse a single argument from one or more strings from the
+    command line. The keyword arguments to the Action constructor are also
+    all attributes of Action instances.
+
+    Keyword Arguments:
+
+        - option_strings -- A list of command-line option strings which
+            should be associated with this action.
+
+        - dest -- The name of the attribute to hold the created object(s)
+
+        - nargs -- The number of command-line arguments that should be
+            consumed. By default, one argument will be consumed and a single
+            value will be produced.  Other values include:
+                - N (an integer) consumes N arguments (and produces a list)
+                - '?' consumes zero or one arguments
+                - '*' consumes zero or more arguments (and produces a list)
+                - '+' consumes one or more arguments (and produces a list)
+            Note that the difference between the default and nargs=1 is that
+            with the default, a single value will be produced, while with
+            nargs=1, a list containing a single value will be produced.
+
+        - const -- The value to be produced if the option is specified and the
+            option uses an action that takes no values.
+
+        - default -- The value to be produced if the option is not specified.
+
+        - type -- The type which the command-line arguments should be converted
+            to, should be one of 'string', 'int', 'float', 'complex' or a
+            callable object that accepts a single string argument. If None,
+            'string' is assumed.
+
+        - choices -- A container of values that should be allowed. If not None,
+            after a command-line argument has been converted to the appropriate
+            type, an exception will be raised if it is not a member of this
+            collection.
+
+        - required -- True if the action must always be specified at the
+            command line. This is only meaningful for optional command-line
+            arguments.
+
+        - help -- The help string describing the argument.
+
+        - metavar -- The name to be used for the option's argument with the
+            help string. If None, the 'dest' value will be used as the name.
+    """
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        self.option_strings = option_strings
+        self.dest = dest
+        self.nargs = nargs
+        self.const = const
+        self.default = default
+        self.type = type
+        self.choices = choices
+        self.required = required
+        self.help = help
+        self.metavar = metavar
+
+    def _get_kwargs(self):
+        names = [
+            'option_strings',
+            'dest',
+            'nargs',
+            'const',
+            'default',
+            'type',
+            'choices',
+            'help',
+            'metavar',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        raise NotImplementedError(_('.__call__() not defined'))
+
+
+class _StoreAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs for store actions must be > 0; if you '
+                             'have nothing to store, actions such as store '
+                             'true or store const may be more appropriate')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_StoreAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+
+class _StoreConstAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_StoreConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, self.const)
+
+
+class _StoreTrueAction(_StoreConstAction):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=False,
+                 required=False,
+                 help=None):
+        super(_StoreTrueAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=True,
+            default=default,
+            required=required,
+            help=help)
+
+
+class _StoreFalseAction(_StoreConstAction):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=True,
+                 required=False,
+                 help=None):
+        super(_StoreFalseAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=False,
+            default=default,
+            required=required,
+            help=help)
+
+
+class _AppendAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs for append actions must be > 0; if arg '
+                             'strings are not supplying the value to append, '
+                             'the append const action may be more appropriate')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_AppendAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = _copy.copy(_ensure_value(namespace, self.dest, []))
+        items.append(values)
+        setattr(namespace, self.dest, items)
+
+
+class _AppendConstAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_AppendConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = _copy.copy(_ensure_value(namespace, self.dest, []))
+        items.append(self.const)
+        setattr(namespace, self.dest, items)
+
+
+class _CountAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=None,
+                 required=False,
+                 help=None):
+        super(_CountAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        new_count = _ensure_value(namespace, self.dest, 0) + 1
+        setattr(namespace, self.dest, new_count)
+
+
+class _HelpAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help=None):
+        super(_HelpAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_help()
+        parser.exit()
+
+
+class _VersionAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 version=None,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help="show program's version number and exit"):
+        super(_VersionAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+        self.version = version
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        version = self.version
+        if version is None:
+            version = parser.version
+        formatter = parser._get_formatter()
+        formatter.add_text(version)
+        parser.exit(message=formatter.format_help())
+
+
+class _SubParsersAction(Action):
+
+    class _ChoicesPseudoAction(Action):
+
+        def __init__(self, name, aliases, help):
+            metavar = dest = name
+            if aliases:
+                metavar += ' (%s)' % ', '.join(aliases)
+            sup = super(_SubParsersAction._ChoicesPseudoAction, self)
+            sup.__init__(option_strings=[], dest=dest, help=help,
+                         metavar=metavar)
+
+    def __init__(self,
+                 option_strings,
+                 prog,
+                 parser_class,
+                 dest=SUPPRESS,
+                 help=None,
+                 metavar=None):
+
+        self._prog_prefix = prog
+        self._parser_class = parser_class
+        self._name_parser_map = _collections.OrderedDict()
+        self._choices_actions = []
+
+        super(_SubParsersAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=PARSER,
+            choices=self._name_parser_map,
+            help=help,
+            metavar=metavar)
+
+    def add_parser(self, name, **kwargs):
+        # set prog from the existing prefix
+        if kwargs.get('prog') is None:
+            kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
+
+        aliases = kwargs.pop('aliases', ())
+
+        # create a pseudo-action to hold the choice help
+        if 'help' in kwargs:
+            help = kwargs.pop('help')
+            choice_action = self._ChoicesPseudoAction(name, aliases, help)
+            self._choices_actions.append(choice_action)
+
+        # create the parser and add it to the map
+        parser = self._parser_class(**kwargs)
+        self._name_parser_map[name] = parser
+
+        # make parser available under aliases also
+        for alias in aliases:
+            self._name_parser_map[alias] = parser
+
+        return parser
+
+    def _get_subactions(self):
+        return self._choices_actions
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser_name = values[0]
+        arg_strings = values[1:]
+
+        # set the parser name if requested
+        if self.dest is not SUPPRESS:
+            setattr(namespace, self.dest, parser_name)
+
+        # select the parser
+        try:
+            parser = self._name_parser_map[parser_name]
+        except KeyError:
+            args = {'parser_name': parser_name,
+                    'choices': ', '.join(self._name_parser_map)}
+            msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
+            raise ArgumentError(self, msg)
+
+        # parse all the remaining options into the namespace
+        # store any unrecognized options on the object, so that the top
+        # level parser can decide what to do with them
+        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+        if arg_strings:
+            vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
+            getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
+
+
+# ==============
+# Type classes
+# ==============
+
+class FileType(object):
+    """Factory for creating file object types
+
+    Instances of FileType are typically passed as type= arguments to the
+    ArgumentParser add_argument() method.
+
+    Keyword Arguments:
+        - mode -- A string indicating how the file is to be opened. Accepts the
+            same values as the builtin open() function.
+        - bufsize -- The file's desired buffer size. Accepts the same values as
+            the builtin open() function.
+    """
+
+    def __init__(self, mode='r', bufsize=-1):
+        self._mode = mode
+        self._bufsize = bufsize
+
+    def __call__(self, string):
+        # the special argument "-" means sys.std{in,out}
+        if string == '-':
+            if 'r' in self._mode:
+                return _sys.stdin
+            elif 'w' in self._mode:
+                return _sys.stdout
+            else:
+                msg = _('argument "-" with mode %r') % self._mode
+                raise ValueError(msg)
+
+        # all other arguments are used as file names
+        try:
+            return open(string, self._mode, self._bufsize)
+        except IOError as e:
+            message = _("can't open '%s': %s")
+            raise ArgumentTypeError(message % (string, e))
+
+    def __repr__(self):
+        args = self._mode, self._bufsize
+        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
+        return '%s(%s)' % (type(self).__name__, args_str)
+
+# ===========================
+# Optional and Positional Parsing
+# ===========================
+
+class Namespace(_AttributeHolder):
+    """Simple object for storing attributes.
+
+    Implements equality by attribute names and values, and provides a simple
+    string representation.
+    """
+
+    def __init__(self, **kwargs):
+        for name in kwargs:
+            setattr(self, name, kwargs[name])
+
+    def __eq__(self, other):
+        return vars(self) == vars(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+
+class _ActionsContainer(object):
+
+    def __init__(self,
+                 description,
+                 prefix_chars,
+                 argument_default,
+                 conflict_handler):
+        super(_ActionsContainer, self).__init__()
+
+        self.description = description
+        self.argument_default = argument_default
+        self.prefix_chars = prefix_chars
+        self.conflict_handler = conflict_handler
+
+        # set up registries
+        self._registries = {}
+
+        # register actions
+        self.register('action', None, _StoreAction)
+        self.register('action', 'store', _StoreAction)
+        self.register('action', 'store_const', _StoreConstAction)
+        self.register('action', 'store_true', _StoreTrueAction)
+        self.register('action', 'store_false', _StoreFalseAction)
+        self.register('action', 'append', _AppendAction)
+        self.register('action', 'append_const', _AppendConstAction)
+        self.register('action', 'count', _CountAction)
+        self.register('action', 'help', _HelpAction)
+        self.register('action', 'version', _VersionAction)
+        self.register('action', 'parsers', _SubParsersAction)
+
+        # raise an exception if the conflict handler is invalid
+        self._get_handler()
+
+        # action storage
+        self._actions = []
+        self._option_string_actions = {}
+
+        # groups
+        self._action_groups = []
+        self._mutually_exclusive_groups = []
+
+        # defaults storage
+        self._defaults = {}
+
+        # determines whether an "option" looks like a negative number
+        self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')
+
+        # whether or not there are any optionals that look like negative
+        # numbers -- uses a list so it can be shared and edited
+        self._has_negative_number_optionals = []
+
+    # ====================
+    # Registration methods
+    # ====================
+    def register(self, registry_name, value, object):
+        registry = self._registries.setdefault(registry_name, {})
+        registry[value] = object
+
+    def _registry_get(self, registry_name, value, default=None):
+        return self._registries[registry_name].get(value, default)
+
+    # ==================================
+    # Namespace default accessor methods
+    # ==================================
+    def set_defaults(self, **kwargs):
+        self._defaults.update(kwargs)
+
+        # if these defaults match any existing arguments, replace
+        # the previous default on the object with the new one
+        for action in self._actions:
+            if action.dest in kwargs:
+                action.default = kwargs[action.dest]
+
+    def get_default(self, dest):
+        for action in self._actions:
+            if action.dest == dest and action.default is not None:
+                return action.default
+        return self._defaults.get(dest, None)
+
+
+    # =======================
+    # Adding argument actions
+    # =======================
+    def add_argument(self, *args, **kwargs):
+        """
+        add_argument(dest, ..., name=value, ...)
+        add_argument(option_string, option_string, ..., name=value, ...)
+        """
+
+        # if no positional args are supplied or only one is supplied and
+        # it doesn't look like an option string, parse a positional
+        # argument
+        chars = self.prefix_chars
+        if not args or len(args) == 1 and args[0][0] not in chars:
+            if args and 'dest' in kwargs:
+                raise ValueError('dest supplied twice for positional argument')
+            kwargs = self._get_positional_kwargs(*args, **kwargs)
+
+        # otherwise, we're adding an optional argument
+        else:
+            kwargs = self._get_optional_kwargs(*args, **kwargs)
+
+        # if no default was supplied, use the parser-level default
+        if 'default' not in kwargs:
+            dest = kwargs['dest']
+            if dest in self._defaults:
+                kwargs['default'] = self._defaults[dest]
+            elif self.argument_default is not None:
+                kwargs['default'] = self.argument_default
+
+        # create the action object, and add it to the parser
+        action_class = self._pop_action_class(kwargs)
+        if not callable(action_class):
+            raise ValueError('unknown action "%s"' % (action_class,))
+        action = action_class(**kwargs)
+
+        # raise an error if the action type is not callable
+        type_func = self._registry_get('type', action.type, action.type)
+        if not callable(type_func):
+            raise ValueError('%r is not callable' % (type_func,))
+
+        # raise an error if the metavar does not match the type
+        if hasattr(self, "_get_formatter"):
+            try:
+                self._get_formatter()._format_args(action, None)
+            except TypeError:
+                raise ValueError("length of metavar tuple does not match nargs")
+
+        return self._add_action(action)
+
+    def add_argument_group(self, *args, **kwargs):
+        group = _ArgumentGroup(self, *args, **kwargs)
+        self._action_groups.append(group)
+        return group
+
+    def add_mutually_exclusive_group(self, **kwargs):
+        group = _MutuallyExclusiveGroup(self, **kwargs)
+        self._mutually_exclusive_groups.append(group)
+        return group
+
+    def _add_action(self, action):
+        # resolve any conflicts
+        self._check_conflict(action)
+
+        # add to actions list
+        self._actions.append(action)
+        action.container = self
+
+        # index the action by any option strings it has
+        for option_string in action.option_strings:
+            self._option_string_actions[option_string] = action
+
+        # set the flag if any option strings look like negative numbers
+        for option_string in action.option_strings:
+            if self._negative_number_matcher.match(option_string):
+                if not self._has_negative_number_optionals:
+                    self._has_negative_number_optionals.append(True)
+
+        # return the created action
+        return action
+
+    def _remove_action(self, action):
+        self._actions.remove(action)
+
+    def _add_container_actions(self, container):
+        # collect groups by titles
+        title_group_map = {}
+        for group in self._action_groups:
+            if group.title in title_group_map:
+                msg = _('cannot merge actions - two groups are named %r')
+                raise ValueError(msg % (group.title))
+            title_group_map[group.title] = group
+
+        # map each action to its group
+        group_map = {}
+        for group in container._action_groups:
+
+            # if a group with the title exists, use that, otherwise
+            # create a new group matching the container's group
+            if group.title not in title_group_map:
+                title_group_map[group.title] = self.add_argument_group(
+                    title=group.title,
+                    description=group.description,
+                    conflict_handler=group.conflict_handler)
+
+            # map the actions to their new group
+            for action in group._group_actions:
+                group_map[action] = title_group_map[group.title]
+
+        # add container's mutually exclusive groups
+        # NOTE: if add_mutually_exclusive_group ever gains title= and
+        # description= then this code will need to be expanded as above
+        for group in container._mutually_exclusive_groups:
+            mutex_group = self.add_mutually_exclusive_group(
+                required=group.required)
+
+            # map the actions to their new mutex group
+            for action in group._group_actions:
+                group_map[action] = mutex_group
+
+        # add all actions to this container or their group
+        for action in container._actions:
+            group_map.get(action, self)._add_action(action)
+
+    def _get_positional_kwargs(self, dest, **kwargs):
+        # make sure required is not specified
+        if 'required' in kwargs:
+            msg = _("'required' is an invalid argument for positionals")
+            raise TypeError(msg)
+
+        # mark positional arguments as required if at least one is
+        # always required
+        if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]:
+            kwargs['required'] = True
+        if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs:
+            kwargs['required'] = True
+
+        # return the keyword arguments with no option strings
+        return dict(kwargs, dest=dest, option_strings=[])
+
+    def _get_optional_kwargs(self, *args, **kwargs):
+        # determine short and long option strings
+        option_strings = []
+        long_option_strings = []
+        for option_string in args:
+            # error on strings that don't start with an appropriate prefix
+            if not option_string[0] in self.prefix_chars:
+                args = {'option': option_string,
+                        'prefix_chars': self.prefix_chars}
+                msg = _('invalid option string %(option)r: '
+                        'must start with a character %(prefix_chars)r')
+                raise ValueError(msg % args)
+
+            # strings starting with two prefix characters are long options
+            option_strings.append(option_string)
+            if option_string[0] in self.prefix_chars:
+                if len(option_string) > 1:
+                    if option_string[1] in self.prefix_chars:
+                        long_option_strings.append(option_string)
+
+        # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
+        dest = kwargs.pop('dest', None)
+        if dest is None:
+            if long_option_strings:
+                dest_option_string = long_option_strings[0]
+            else:
+                dest_option_string = option_strings[0]
+            dest = dest_option_string.lstrip(self.prefix_chars)
+            if not dest:
+                msg = _('dest= is required for options like %r')
+                raise ValueError(msg % option_string)
+            dest = dest.replace('-', '_')
+
+        # return the updated keyword arguments
+        return dict(kwargs, dest=dest, option_strings=option_strings)
+
+    def _pop_action_class(self, kwargs, default=None):
+        action = kwargs.pop('action', default)
+        return self._registry_get('action', action, action)
+
+    def _get_handler(self):
+        # determine function from conflict handler string
+        handler_func_name = '_handle_conflict_%s' % self.conflict_handler
+        try:
+            return getattr(self, handler_func_name)
+        except AttributeError:
+            msg = _('invalid conflict_resolution value: %r')
+            raise ValueError(msg % self.conflict_handler)
+
+    def _check_conflict(self, action):
+
+        # find all options that conflict with this option
+        confl_optionals = []
+        for option_string in action.option_strings:
+            if option_string in self._option_string_actions:
+                confl_optional = self._option_string_actions[option_string]
+                confl_optionals.append((option_string, confl_optional))
+
+        # resolve any conflicts
+        if confl_optionals:
+            conflict_handler = self._get_handler()
+            conflict_handler(action, confl_optionals)
+
+    def _handle_conflict_error(self, action, conflicting_actions):
+        message = ngettext('conflicting option string: %s',
+                           'conflicting option strings: %s',
+                           len(conflicting_actions))
+        conflict_string = ', '.join([option_string
+                                     for option_string, action
+                                     in conflicting_actions])
+        raise ArgumentError(action, message % conflict_string)
+
+    def _handle_conflict_resolve(self, action, conflicting_actions):
+
+        # remove all conflicting options
+        for option_string, action in conflicting_actions:
+
+            # remove the conflicting option
+            action.option_strings.remove(option_string)
+            self._option_string_actions.pop(option_string, None)
+
+            # if the option now has no option string, remove it from the
+            # container holding it
+            if not action.option_strings:
+                action.container._remove_action(action)
+
+
+class _ArgumentGroup(_ActionsContainer):
+
+    def __init__(self, container, title=None, description=None, **kwargs):
+        # add any missing keyword arguments by checking the container
+        update = kwargs.setdefault
+        update('conflict_handler', container.conflict_handler)
+        update('prefix_chars', container.prefix_chars)
+        update('argument_default', container.argument_default)
+        super_init = super(_ArgumentGroup, self).__init__
+        super_init(description=description, **kwargs)
+
+        # group attributes
+        self.title = title
+        self._group_actions = []
+
+        # share most attributes with the container
+        self._registries = container._registries
+        self._actions = container._actions
+        self._option_string_actions = container._option_string_actions
+        self._defaults = container._defaults
+        self._has_negative_number_optionals = \
+            container._has_negative_number_optionals
+        self._mutually_exclusive_groups = container._mutually_exclusive_groups
+
+    def _add_action(self, action):
+        action = super(_ArgumentGroup, self)._add_action(action)
+        self._group_actions.append(action)
+        return action
+
+    def _remove_action(self, action):
+        super(_ArgumentGroup, self)._remove_action(action)
+        self._group_actions.remove(action)
+
+
+class _MutuallyExclusiveGroup(_ArgumentGroup):
+
+    def __init__(self, container, required=False):
+        super(_MutuallyExclusiveGroup, self).__init__(container)
+        self.required = required
+        self._container = container
+
+    def _add_action(self, action):
+        if action.required:
+            msg = _('mutually exclusive arguments must be optional')
+            raise ValueError(msg)
+        action = self._container._add_action(action)
+        self._group_actions.append(action)
+        return action
+
+    def _remove_action(self, action):
+        self._container._remove_action(action)
+        self._group_actions.remove(action)
+
+
+class ArgumentParser(_AttributeHolder, _ActionsContainer):
+    """Object for parsing command line strings into Python objects.
+
+    Keyword Arguments:
+        - prog -- The name of the program (default: sys.argv[0])
+        - usage -- A usage message (default: auto-generated from arguments)
+        - description -- A description of what the program does
+        - epilog -- Text following the argument descriptions
+        - parents -- Parsers whose arguments should be copied into this one
+        - formatter_class -- HelpFormatter class for printing help messages
+        - prefix_chars -- Characters that prefix optional arguments
+        - fromfile_prefix_chars -- Characters that prefix files containing
+            additional arguments
+        - argument_default -- The default value for all arguments
+        - conflict_handler -- String indicating how to handle conflicts
+        - add_help -- Add a -h/-help option
+    """
+
+    def __init__(self,
+                 prog=None,
+                 usage=None,
+                 description=None,
+                 epilog=None,
+                 version=None,
+                 parents=[],
+                 formatter_class=HelpFormatter,
+                 prefix_chars='-',
+                 fromfile_prefix_chars=None,
+                 argument_default=None,
+                 conflict_handler='error',
+                 add_help=True):
+
+        if version is not None:
+            import warnings
+            warnings.warn(
+                """The "version" argument to ArgumentParser is deprecated. """
+                """Please use """
+                """"add_argument(..., action='version', version="N", ...)" """
+                """instead""", DeprecationWarning)
+
+        superinit = super(ArgumentParser, self).__init__
+        superinit(description=description,
+                  prefix_chars=prefix_chars,
+                  argument_default=argument_default,
+                  conflict_handler=conflict_handler)
+
+        # default setting for prog
+        if prog is None:
+            prog = _os.path.basename(_sys.argv[0])
+
+        self.prog = prog
+        self.usage = usage
+        self.epilog = epilog
+        self.version = version
+        self.formatter_class = formatter_class
+        self.fromfile_prefix_chars = fromfile_prefix_chars
+        self.add_help = add_help
+
+        add_group = self.add_argument_group
+        self._positionals = add_group(_('positional arguments'))
+        self._optionals = add_group(_('optional arguments'))
+        self._subparsers = None
+
+        # register types
+        def identity(string):
+            return string
+        self.register('type', None, identity)
+
+        # add help and version arguments if necessary
+        # (using explicit default to override global argument_default)
+        default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
+        if self.add_help:
+            self.add_argument(
+                default_prefix+'h', default_prefix*2+'help',
+                action='help', default=SUPPRESS,
+                help=_('show this help message and exit'))
+        if self.version:
+            self.add_argument(
+                default_prefix+'v', default_prefix*2+'version',
+                action='version', default=SUPPRESS,
+                version=self.version,
+                help=_("show program's version number and exit"))
+
+        # add parent arguments and defaults
+        for parent in parents:
+            self._add_container_actions(parent)
+            try:
+                defaults = parent._defaults
+            except AttributeError:
+                pass
+            else:
+                self._defaults.update(defaults)
+
+    # =======================
+    # Pretty __repr__ methods
+    # =======================
+    def _get_kwargs(self):
+        names = [
+            'prog',
+            'usage',
+            'description',
+            'version',
+            'formatter_class',
+            'conflict_handler',
+            'add_help',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    # ==================================
+    # Optional/Positional adding methods
+    # ==================================
+    def add_subparsers(self, **kwargs):
+        if self._subparsers is not None:
+            self.error(_('cannot have multiple subparser arguments'))
+
+        # add the parser class to the arguments if it's not present
+        kwargs.setdefault('parser_class', type(self))
+
+        if 'title' in kwargs or 'description' in kwargs:
+            title = _(kwargs.pop('title', 'subcommands'))
+            description = _(kwargs.pop('description', None))
+            self._subparsers = self.add_argument_group(title, description)
+        else:
+            self._subparsers = self._positionals
+
+        # prog defaults to the usage message of this parser, skipping
+        # optional arguments and with no "usage:" prefix
+        if kwargs.get('prog') is None:
+            formatter = self._get_formatter()
+            positionals = self._get_positional_actions()
+            groups = self._mutually_exclusive_groups
+            formatter.add_usage(self.usage, positionals, groups, '')
+            kwargs['prog'] = formatter.format_help().strip()
+
+        # create the parsers action and add it to the positionals list
+        parsers_class = self._pop_action_class(kwargs, 'parsers')
+        action = parsers_class(option_strings=[], **kwargs)
+        self._subparsers._add_action(action)
+
+        # return the created parsers action
+        return action
+
+    def _add_action(self, action):
+        if action.option_strings:
+            self._optionals._add_action(action)
+        else:
+            self._positionals._add_action(action)
+        return action
+
+    def _get_optional_actions(self):
+        return [action
+                for action in self._actions
+                if action.option_strings]
+
+    def _get_positional_actions(self):
+        return [action
+                for action in self._actions
+                if not action.option_strings]
+
+    # =====================================
+    # Command line argument parsing methods
+    # =====================================
+    def parse_args(self, args=None, namespace=None):
+        args, argv = self.parse_known_args(args, namespace)
+        if argv:
+            msg = _('unrecognized arguments: %s')
+            self.error(msg % ' '.join(argv))
+        return args
+
+    def parse_known_args(self, args=None, namespace=None):
+        # args default to the system args
+        if args is None:
+            args = _sys.argv[1:]
+
+        # default Namespace built from parser defaults
+        if namespace is None:
+            namespace = Namespace()
+
+        # add any action defaults that aren't present
+        for action in self._actions:
+            if action.dest is not SUPPRESS:
+                if not hasattr(namespace, action.dest):
+                    if action.default is not SUPPRESS:
+                        default = action.default
+                        if isinstance(action.default, str):
+                            default = self._get_value(action, default)
+                        setattr(namespace, action.dest, default)
+
+        # add any parser defaults that aren't present
+        for dest in self._defaults:
+            if not hasattr(namespace, dest):
+                setattr(namespace, dest, self._defaults[dest])
+
+        # parse the arguments and exit if there are any errors
+        try:
+            namespace, args = self._parse_known_args(args, namespace)
+            if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
+                args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
+                delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
+            return namespace, args
+        except ArgumentError:
+            err = _sys.exc_info()[1]
+            self.error(str(err))
+
+    def _parse_known_args(self, arg_strings, namespace):
+        # replace arg strings that are file references
+        if self.fromfile_prefix_chars is not None:
+            arg_strings = self._read_args_from_files(arg_strings)
+
+        # map all mutually exclusive arguments to the other arguments
+        # they can't occur with
+        action_conflicts = {}
+        for mutex_group in self._mutually_exclusive_groups:
+            group_actions = mutex_group._group_actions
+            for i, mutex_action in enumerate(mutex_group._group_actions):
+                conflicts = action_conflicts.setdefault(mutex_action, [])
+                conflicts.extend(group_actions[:i])
+                conflicts.extend(group_actions[i + 1:])
+
+        # find all option indices, and determine the arg_string_pattern
+        # which has an 'O' if there is an option at an index,
+        # an 'A' if there is an argument, or a '-' if there is a '--'
+        option_string_indices = {}
+        arg_string_pattern_parts = []
+        arg_strings_iter = iter(arg_strings)
+        for i, arg_string in enumerate(arg_strings_iter):
+
+            # all args after -- are non-options
+            if arg_string == '--':
+                arg_string_pattern_parts.append('-')
+                for arg_string in arg_strings_iter:
+                    arg_string_pattern_parts.append('A')
+
+            # otherwise, add the arg to the arg strings
+            # and note the index if it was an option
+            else:
+                option_tuple = self._parse_optional(arg_string)
+                if option_tuple is None:
+                    pattern = 'A'
+                else:
+                    option_string_indices[i] = option_tuple
+                    pattern = 'O'
+                arg_string_pattern_parts.append(pattern)
+
+        # join the pieces together to form the pattern
+        arg_strings_pattern = ''.join(arg_string_pattern_parts)
+
+        # converts arg strings to the appropriate and then takes the action
+        seen_actions = set()
+        seen_non_default_actions = set()
+
+        def take_action(action, argument_strings, option_string=None):
+            seen_actions.add(action)
+            argument_values = self._get_values(action, argument_strings)
+
+            # error if this argument is not allowed with other previously
+            # seen arguments, assuming that actions that use the default
+            # value don't really count as "present"
+            if argument_values is not action.default:
+                seen_non_default_actions.add(action)
+                for conflict_action in action_conflicts.get(action, []):
+                    if conflict_action in seen_non_default_actions:
+                        msg = _('not allowed with argument %s')
+                        action_name = _get_action_name(conflict_action)
+                        raise ArgumentError(action, msg % action_name)
+
+            # take the action if we didn't receive a SUPPRESS value
+            # (e.g. from a default)
+            if argument_values is not SUPPRESS:
+                action(self, namespace, argument_values, option_string)
+
+        # function to convert arg_strings into an optional action
+        def consume_optional(start_index):
+
+            # get the optional identified at this index
+            option_tuple = option_string_indices[start_index]
+            action, option_string, explicit_arg = option_tuple
+
+            # identify additional optionals in the same arg string
+            # (e.g. -xyz is the same as -x -y -z if no args are required)
+            match_argument = self._match_argument
+            action_tuples = []
+            while True:
+
+                # if we found no optional action, skip it
+                if action is None:
+                    extras.append(arg_strings[start_index])
+                    return start_index + 1
+
+                # if there is an explicit argument, try to match the
+                # optional's string arguments to only this
+                if explicit_arg is not None:
+                    arg_count = match_argument(action, 'A')
+
+                    # if the action is a single-dash option and takes no
+                    # arguments, try to parse more single-dash options out
+                    # of the tail of the option string
+                    chars = self.prefix_chars
+                    if arg_count == 0 and option_string[1] not in chars:
+                        action_tuples.append((action, [], option_string))
+                        char = option_string[0]
+                        option_string = char + explicit_arg[0]
+                        new_explicit_arg = explicit_arg[1:] or None
+                        optionals_map = self._option_string_actions
+                        if option_string in optionals_map:
+                            action = optionals_map[option_string]
+                            explicit_arg = new_explicit_arg
+                        else:
+                            msg = _('ignored explicit argument %r')
+                            raise ArgumentError(action, msg % explicit_arg)
+
+                    # if the action expect exactly one argument, we've
+                    # successfully matched the option; exit the loop
+                    elif arg_count == 1:
+                        stop = start_index + 1
+                        args = [explicit_arg]
+                        action_tuples.append((action, args, option_string))
+                        break
+
+                    # error if a double-dash option did not use the
+                    # explicit argument
+                    else:
+                        msg = _('ignored explicit argument %r')
+                        raise ArgumentError(action, msg % explicit_arg)
+
+                # if there is no explicit argument, try to match the
+                # optional's string arguments with the following strings
+                # if successful, exit the loop
+                else:
+                    start = start_index + 1
+                    selected_patterns = arg_strings_pattern[start:]
+                    arg_count = match_argument(action, selected_patterns)
+                    stop = start + arg_count
+                    args = arg_strings[start:stop]
+                    action_tuples.append((action, args, option_string))
+                    break
+
+            # add the Optional to the list and return the index at which
+            # the Optional's string args stopped
+            assert action_tuples
+            for action, args, option_string in action_tuples:
+                take_action(action, args, option_string)
+            return stop
+
+        # the list of Positionals left to be parsed; this is modified
+        # by consume_positionals()
+        positionals = self._get_positional_actions()
+
+        # function to convert arg_strings into positional actions
+        def consume_positionals(start_index):
+            # match as many Positionals as possible
+            match_partial = self._match_arguments_partial
+            selected_pattern = arg_strings_pattern[start_index:]
+            arg_counts = match_partial(positionals, selected_pattern)
+
+            # slice off the appropriate arg strings for each Positional
+            # and add the Positional and its args to the list
+            for action, arg_count in zip(positionals, arg_counts):
+                args = arg_strings[start_index: start_index + arg_count]
+                start_index += arg_count
+                take_action(action, args)
+
+            # slice off the Positionals that we just parsed and return the
+            # index at which the Positionals' string args stopped
+            positionals[:] = positionals[len(arg_counts):]
+            return start_index
+
+        # consume Positionals and Optionals alternately, until we have
+        # passed the last option string
+        extras = []
+        start_index = 0
+        if option_string_indices:
+            max_option_string_index = max(option_string_indices)
+        else:
+            max_option_string_index = -1
+        while start_index <= max_option_string_index:
+
+            # consume any Positionals preceding the next option
+            next_option_string_index = min([
+                index
+                for index in option_string_indices
+                if index >= start_index])
+            if start_index != next_option_string_index:
+                positionals_end_index = consume_positionals(start_index)
+
+                # only try to parse the next optional if we didn't consume
+                # the option string during the positionals parsing
+                if positionals_end_index > start_index:
+                    start_index = positionals_end_index
+                    continue
+                else:
+                    start_index = positionals_end_index
+
+            # if we consumed all the positionals we could and we're not
+            # at the index of an option string, there were extra arguments
+            if start_index not in option_string_indices:
+                strings = arg_strings[start_index:next_option_string_index]
+                extras.extend(strings)
+                start_index = next_option_string_index
+
+            # consume the next optional and any arguments for it
+            start_index = consume_optional(start_index)
+
+        # consume any positionals following the last Optional
+        stop_index = consume_positionals(start_index)
+
+        # if we didn't consume all the argument strings, there were extras
+        extras.extend(arg_strings[stop_index:])
+
+        # if we didn't use all the Positional objects, there were too few
+        # arg strings supplied.
+        if positionals:
+            self.error(_('too few arguments'))
+
+        # make sure all required actions were present
+        for action in self._actions:
+            if action.required:
+                if action not in seen_actions:
+                    name = _get_action_name(action)
+                    self.error(_('argument %s is required') % name)
+
+        # make sure all required groups had one option present
+        for group in self._mutually_exclusive_groups:
+            if group.required:
+                for action in group._group_actions:
+                    if action in seen_non_default_actions:
+                        break
+
+                # if no actions were used, report the error
+                else:
+                    names = [_get_action_name(action)
+                             for action in group._group_actions
+                             if action.help is not SUPPRESS]
+                    msg = _('one of the arguments %s is required')
+                    self.error(msg % ' '.join(names))
+
+        # return the updated namespace and the extra arguments
+        return namespace, extras
+
+    def _read_args_from_files(self, arg_strings):
+        # expand arguments referencing files
+        new_arg_strings = []
+        for arg_string in arg_strings:
+
+            # for regular arguments, just add them back into the list
+            if arg_string[0] not in self.fromfile_prefix_chars:
+                new_arg_strings.append(arg_string)
+
+            # replace arguments referencing files with the file content
+            else:
+                try:
+                    args_file = open(arg_string[1:])
+                    try:
+                        arg_strings = []
+                        for arg_line in args_file.read().splitlines():
+                            for arg in self.convert_arg_line_to_args(arg_line):
+                                arg_strings.append(arg)
+                        arg_strings = self._read_args_from_files(arg_strings)
+                        new_arg_strings.extend(arg_strings)
+                    finally:
+                        args_file.close()
+                except IOError:
+                    err = _sys.exc_info()[1]
+                    self.error(str(err))
+
+        # return the modified argument list
+        return new_arg_strings
+
+    def convert_arg_line_to_args(self, arg_line):
+        return [arg_line]
+
+    def _match_argument(self, action, arg_strings_pattern):
+        # match the pattern for this action to the arg strings
+        nargs_pattern = self._get_nargs_pattern(action)
+        match = _re.match(nargs_pattern, arg_strings_pattern)
+
+        # raise an exception if we weren't able to find a match
+        if match is None:
+            nargs_errors = {
+                None: _('expected one argument'),
+                OPTIONAL: _('expected at most one argument'),
+                ONE_OR_MORE: _('expected at least one argument'),
+            }
+            default = ngettext('expected %s argument',
+                               'expected %s arguments',
+                               action.nargs) % action.nargs
+            msg = nargs_errors.get(action.nargs, default)
+            raise ArgumentError(action, msg)
+
+        # return the number of arguments matched
+        return len(match.group(1))
+
+    def _match_arguments_partial(self, actions, arg_strings_pattern):
+        # progressively shorten the actions list by slicing off the
+        # final actions until we find a match
+        result = []
+        for i in range(len(actions), 0, -1):
+            actions_slice = actions[:i]
+            pattern = ''.join([self._get_nargs_pattern(action)
+                               for action in actions_slice])
+            match = _re.match(pattern, arg_strings_pattern)
+            if match is not None:
+                result.extend([len(string) for string in match.groups()])
+                break
+
+        # return the list of arg string counts
+        return result
+
+    def _parse_optional(self, arg_string):
+        # if it's an empty string, it was meant to be a positional
+        if not arg_string:
+            return None
+
+        # if it doesn't start with a prefix, it was meant to be positional
+        if not arg_string[0] in self.prefix_chars:
+            return None
+
+        # if the option string is present in the parser, return the action
+        if arg_string in self._option_string_actions:
+            action = self._option_string_actions[arg_string]
+            return action, arg_string, None
+
+        # if it's just a single character, it was meant to be positional
+        if len(arg_string) == 1:
+            return None
+
+        # if the option string before the "=" is present, return the action
+        if '=' in arg_string:
+            option_string, explicit_arg = arg_string.split('=', 1)
+            if option_string in self._option_string_actions:
+                action = self._option_string_actions[option_string]
+                return action, option_string, explicit_arg
+
+        # search through all possible prefixes of the option string
+        # and all actions in the parser for possible interpretations
+        option_tuples = self._get_option_tuples(arg_string)
+
+        # if multiple actions match, the option string was ambiguous
+        if len(option_tuples) > 1:
+            options = ', '.join([option_string
+                for action, option_string, explicit_arg in option_tuples])
+            args = {'option': arg_string, 'matches': options}
+            msg = _('ambiguous option: %(option)s could match %(matches)s')
+            self.error(msg % args)
+
+        # if exactly one action matched, this segmentation is good,
+        # so return the parsed action
+        elif len(option_tuples) == 1:
+            option_tuple, = option_tuples
+            return option_tuple
+
+        # if it was not found as an option, but it looks like a negative
+        # number, it was meant to be positional
+        # unless there are negative-number-like options
+        if self._negative_number_matcher.match(arg_string):
+            if not self._has_negative_number_optionals:
+                return None
+
+        # if it contains a space, it was meant to be a positional
+        if ' ' in arg_string:
+            return None
+
+        # it was meant to be an optional but there is no such option
+        # in this parser (though it might be a valid option in a subparser)
+        return None, arg_string, None
+
+    def _get_option_tuples(self, option_string):
+        result = []
+
+        # option strings starting with two prefix characters are only
+        # split at the '='
+        chars = self.prefix_chars
+        if option_string[0] in chars and option_string[1] in chars:
+            if '=' in option_string:
+                option_prefix, explicit_arg = option_string.split('=', 1)
+            else:
+                option_prefix = option_string
+                explicit_arg = None
+            for option_string in self._option_string_actions:
+                if option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
+
+        # single character options can be concatenated with their arguments
+        # but multiple character options always have to have their argument
+        # separate
+        elif option_string[0] in chars and option_string[1] not in chars:
+            option_prefix = option_string
+            explicit_arg = None
+            short_option_prefix = option_string[:2]
+            short_explicit_arg = option_string[2:]
+
+            for option_string in self._option_string_actions:
+                if option_string == short_option_prefix:
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, short_explicit_arg
+                    result.append(tup)
+                elif option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
+
+        # shouldn't ever get here
+        else:
+            self.error(_('unexpected option string: %s') % option_string)
+
+        # return the collected option tuples
+        return result
+
+    def _get_nargs_pattern(self, action):
+        # in all examples below, we have to allow for '--' args
+        # which are represented as '-' in the pattern
+        nargs = action.nargs
+
+        # the default (None) is assumed to be a single argument
+        if nargs is None:
+            nargs_pattern = '(-*A-*)'
+
+        # allow zero or one arguments
+        elif nargs == OPTIONAL:
+            nargs_pattern = '(-*A?-*)'
+
+        # allow zero or more arguments
+        elif nargs == ZERO_OR_MORE:
+            nargs_pattern = '(-*[A-]*)'
+
+        # allow one or more arguments
+        elif nargs == ONE_OR_MORE:
+            nargs_pattern = '(-*A[A-]*)'
+
+        # allow any number of options or arguments
+        elif nargs == REMAINDER:
+            nargs_pattern = '([-AO]*)'
+
+        # allow one argument followed by any number of options or arguments
+        elif nargs == PARSER:
+            nargs_pattern = '(-*A[-AO]*)'
+
+        # all others should be integers
+        else:
+            nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)
+
+        # if this is an optional action, -- is not allowed
+        if action.option_strings:
+            nargs_pattern = nargs_pattern.replace('-*', '')
+            nargs_pattern = nargs_pattern.replace('-', '')
+
+        # return the pattern
+        return nargs_pattern
+
+    # ========================
+    # Value conversion methods
+    # ========================
+    def _get_values(self, action, arg_strings):
+        # for everything but PARSER args, strip out '--'
+        if action.nargs not in [PARSER, REMAINDER]:
+            arg_strings = [s for s in arg_strings if s != '--']
+
+        # optional argument produces a default when not present
+        if not arg_strings and action.nargs == OPTIONAL:
+            if action.option_strings:
+                value = action.const
+            else:
+                value = action.default
+            if isinstance(value, str):
+                value = self._get_value(action, value)
+                self._check_value(action, value)
+
+        # when nargs='*' on a positional, if there were no command-line
+        # args, use the default if it is anything other than None
+        elif (not arg_strings and action.nargs == ZERO_OR_MORE and
+              not action.option_strings):
+            if action.default is not None:
+                value = action.default
+            else:
+                value = arg_strings
+            self._check_value(action, value)
+
+        # single argument or optional argument produces a single value
+        elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
+            arg_string, = arg_strings
+            value = self._get_value(action, arg_string)
+            self._check_value(action, value)
+
+        # REMAINDER arguments convert all values, checking none
+        elif action.nargs == REMAINDER:
+            value = [self._get_value(action, v) for v in arg_strings]
+
+        # PARSER arguments convert all values, but check only the first
+        elif action.nargs == PARSER:
+            value = [self._get_value(action, v) for v in arg_strings]
+            self._check_value(action, value[0])
+
+        # all other types of nargs produce a list
+        else:
+            value = [self._get_value(action, v) for v in arg_strings]
+            for v in value:
+                self._check_value(action, v)
+
+        # return the converted value
+        return value
+
+    def _get_value(self, action, arg_string):
+        type_func = self._registry_get('type', action.type, action.type)
+        if not callable(type_func):
+            msg = _('%r is not callable')
+            raise ArgumentError(action, msg % type_func)
+
+        # convert the value to the appropriate type
+        try:
+            result = type_func(arg_string)
+
+        # ArgumentTypeErrors indicate errors
+        except ArgumentTypeError:
+            name = getattr(action.type, '__name__', repr(action.type))
+            msg = str(_sys.exc_info()[1])
+            raise ArgumentError(action, msg)
+
+        # TypeErrors or ValueErrors also indicate errors
+        except (TypeError, ValueError):
+            name = getattr(action.type, '__name__', repr(action.type))
+            args = {'type': name, 'value': arg_string}
+            msg = _('invalid %(type)s value: %(value)r')
+            raise ArgumentError(action, msg % args)
+
+        # return the converted value
+        return result
+
+    def _check_value(self, action, value):
+        # converted value must be one of the choices (if specified)
+        if action.choices is not None and value not in action.choices:
+            args = {'value': value,
+                    'choices': ', '.join(map(repr, action.choices))}
+            msg = _('invalid choice: %(value)r (choose from %(choices)s)')
+            raise ArgumentError(action, msg % args)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+    def format_usage(self):
+        formatter = self._get_formatter()
+        formatter.add_usage(self.usage, self._actions,
+                            self._mutually_exclusive_groups)
+        return formatter.format_help()
+
+    def format_help(self):
+        formatter = self._get_formatter()
+
+        # usage
+        formatter.add_usage(self.usage, self._actions,
+                            self._mutually_exclusive_groups)
+
+        # description
+        formatter.add_text(self.description)
+
+        # positionals, optionals and user-defined groups
+        for action_group in self._action_groups:
+            formatter.start_section(action_group.title)
+            formatter.add_text(action_group.description)
+            formatter.add_arguments(action_group._group_actions)
+            formatter.end_section()
+
+        # epilog
+        formatter.add_text(self.epilog)
+
+        # determine help from format above
+        return formatter.format_help()
+
+    def format_version(self):
+        import warnings
+        warnings.warn(
+            'The format_version method is deprecated -- the "version" '
+            'argument to ArgumentParser is no longer supported.',
+            DeprecationWarning)
+        formatter = self._get_formatter()
+        formatter.add_text(self.version)
+        return formatter.format_help()
+
+    def _get_formatter(self):
+        return self.formatter_class(prog=self.prog)
+
+    # =====================
+    # Help-printing methods
+    # =====================
+    def print_usage(self, file=None):
+        if file is None:
+            file = _sys.stdout
+        self._print_message(self.format_usage(), file)
+
+    def print_help(self, file=None):
+        if file is None:
+            file = _sys.stdout
+        self._print_message(self.format_help(), file)
+
+    def print_version(self, file=None):
+        import warnings
+        warnings.warn(
+            'The print_version method is deprecated -- the "version" '
+            'argument to ArgumentParser is no longer supported.',
+            DeprecationWarning)
+        self._print_message(self.format_version(), file)
+
+    def _print_message(self, message, file=None):
+        if message:
+            if file is None:
+                file = _sys.stderr
+            file.write(message)
+
+    # ===============
+    # Exiting methods
+    # ===============
+    def exit(self, status=0, message=None):
+        if message:
+            self._print_message(message, _sys.stderr)
+        _sys.exit(status)
+
+    def error(self, message):
+        """error(message: string)
+
+        Prints a usage message incorporating the message to stderr and
+        exits.
+
+        If you override this in a subclass, it should not return -- it
+        should either exit or raise an exception.
+        """
+        self.print_usage(_sys.stderr)
+        args = {'prog': self.prog, 'message': message}
+        self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
diff --git a/lib-python/3.2/ast.py b/lib-python/3.2/ast.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/ast.py
@@ -0,0 +1,318 @@
+"""
+    ast
+    ~~~
+
+    The `ast` module helps Python applications to process trees of the Python
+    abstract syntax grammar.  The abstract syntax itself might change with
+    each Python release; this module helps to find out programmatically what
+    the current grammar looks like and allows modifications of it.
+
+    An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
+    a flag to the `compile()` builtin function or by using the `parse()`
+    function from this module.  The result will be a tree of objects whose
+    classes all inherit from `ast.AST`.
+
+    A modified abstract syntax tree can be compiled into a Python code object
+    using the built-in `compile()` function.
+
+    Additionally various helper functions are provided that make working with
+    the trees simpler.  The main intention of the helper functions and this
+    module in general is to provide an easy to use interface for libraries
+    that work tightly with the python syntax (template engines for example).
+
+
+    :copyright: Copyright 2008 by Armin Ronacher.
+    :license: Python License.
+"""
+from _ast import *
+from _ast import __version__
+
+
+def parse(source, filename='<unknown>', mode='exec'):
+    """
+    Parse the source into an AST node.
+    Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
+    """
+    return compile(source, filename, mode, PyCF_ONLY_AST)
+
+
+def literal_eval(node_or_string):
+    """
+    Safely evaluate an expression node or a string containing a Python
+    expression.  The string or node provided may only consist of the following
+    Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
+    sets, booleans, and None.
+    """
+    _safe_names = {'None': None, 'True': True, 'False': False}
+    if isinstance(node_or_string, str):
+        node_or_string = parse(node_or_string, mode='eval')
+    if isinstance(node_or_string, Expression):
+        node_or_string = node_or_string.body
+    def _convert(node):
+        if isinstance(node, (Str, Bytes)):
+            return node.s
+        elif isinstance(node, Num):
+            return node.n
+        elif isinstance(node, Tuple):
+            return tuple(map(_convert, node.elts))
+        elif isinstance(node, List):
+            return list(map(_convert, node.elts))
+        elif isinstance(node, Set):
+            return set(map(_convert, node.elts))
+        elif isinstance(node, Dict):
+            return dict((_convert(k), _convert(v)) for k, v
+                        in zip(node.keys, node.values))
+        elif isinstance(node, Name):
+            if node.id in _safe_names:
+                return _safe_names[node.id]
+        elif isinstance(node, UnaryOp) and \
+             isinstance(node.op, (UAdd, USub)) and \
+             isinstance(node.operand, (Num, UnaryOp, BinOp)):
+            operand = _convert(node.operand)
+            if isinstance(node.op, UAdd):
+                return + operand
+            else:
+                return - operand
+        elif isinstance(node, BinOp) and \
+             isinstance(node.op, (Add, Sub)) and \
+             isinstance(node.right, (Num, UnaryOp, BinOp)) and \
+             isinstance(node.left, (Num, UnaryOp, BinOp)):
+            left = _convert(node.left)
+            right = _convert(node.right)
+            if isinstance(node.op, Add):
+                return left + right
+            else:
+                return left - right
+        raise ValueError('malformed node or string: ' + repr(node))
+    return _convert(node_or_string)
+
+
+def dump(node, annotate_fields=True, include_attributes=False):
+    """
+    Return a formatted dump of the tree in *node*.  This is mainly useful for
+    debugging purposes.  The returned string will show the names and the values
+    for fields.  This makes the code impossible to evaluate, so if evaluation is
+    wanted *annotate_fields* must be set to False.  Attributes such as line
+    numbers and column offsets are not dumped by default.  If this is wanted,
+    *include_attributes* can be set to True.
+    """
+    def _format(node):
+        if isinstance(node, AST):
+            fields = [(a, _format(b)) for a, b in iter_fields(node)]
+            rv = '%s(%s' % (node.__class__.__name__, ', '.join(
+                ('%s=%s' % field for field in fields)
+                if annotate_fields else
+                (b for a, b in fields)
+            ))
+            if include_attributes and node._attributes:
+                rv += fields and ', ' or ' '
+                rv += ', '.join('%s=%s' % (a, _format(getattr(node, a)))
+                                for a in node._attributes)
+            return rv + ')'
+        elif isinstance(node, list):
+            return '[%s]' % ', '.join(_format(x) for x in node)
+        return repr(node)
+    if not isinstance(node, AST):
+        raise TypeError('expected AST, got %r' % node.__class__.__name__)
+    return _format(node)
+
+
+def copy_location(new_node, old_node):
+    """
+    Copy source location (`lineno` and `col_offset` attributes) from
+    *old_node* to *new_node* if possible, and return *new_node*.
+    """
+    for attr in 'lineno', 'col_offset':
+        if attr in old_node._attributes and attr in new_node._attributes \
+           and hasattr(old_node, attr):
+            setattr(new_node, attr, getattr(old_node, attr))
+    return new_node
+
+
+def fix_missing_locations(node):
+    """
+    When you compile a node tree with compile(), the compiler expects lineno and
+    col_offset attributes for every node that supports them.  This is rather
+    tedious to fill in for generated nodes, so this helper adds these attributes
+    recursively where not already set, by setting them to the values of the
+    parent node.  It works recursively starting at *node*.
+    """
+    def _fix(node, lineno, col_offset):
+        if 'lineno' in node._attributes:
+            if not hasattr(node, 'lineno'):
+                node.lineno = lineno
+            else:
+                lineno = node.lineno
+        if 'col_offset' in node._attributes:
+            if not hasattr(node, 'col_offset'):
+                node.col_offset = col_offset
+            else:
+                col_offset = node.col_offset
+        for child in iter_child_nodes(node):
+            _fix(child, lineno, col_offset)
+    _fix(node, 1, 0)
+    return node
+
+
+def increment_lineno(node, n=1):
+    """
+    Increment the line number of each node in the tree starting at *node* by *n*.
+    This is useful to "move code" to a different location in a file.
+    """
+    for child in walk(node):
+        if 'lineno' in child._attributes:
+            child.lineno = getattr(child, 'lineno', 0) + n
+    return node
+
+
+def iter_fields(node):
+    """
+    Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields``
+    that is present on *node*.
+    """
+    for field in node._fields:
+        try:
+            yield field, getattr(node, field)
+        except AttributeError:
+            pass
+
+
+def iter_child_nodes(node):
+    """
+    Yield all direct child nodes of *node*, that is, all fields that are nodes
+    and all items of fields that are lists of nodes.
+    """
+    for name, field in iter_fields(node):
+        if isinstance(field, AST):
+            yield field
+        elif isinstance(field, list):
+            for item in field:
+                if isinstance(item, AST):
+                    yield item
+
+
+def get_docstring(node, clean=True):
+    """
+    Return the docstring for the given node or None if no docstring can
+    be found.  If the node provided does not have docstrings a TypeError
+    will be raised.
+    """
+    if not isinstance(node, (FunctionDef, ClassDef, Module)):
+        raise TypeError("%r can't have docstrings" % node.__class__.__name__)
+    if node.body and isinstance(node.body[0], Expr) and \
+       isinstance(node.body[0].value, Str):
+        if clean:
+            import inspect
+            return inspect.cleandoc(node.body[0].value.s)
+        return node.body[0].value.s
+
+
+def walk(node):
+    """
+    Recursively yield all descendant nodes in the tree starting at *node*
+    (including *node* itself), in no specified order.  This is useful if you
+    only want to modify nodes in place and don't care about the context.
+    """
+    from collections import deque
+    todo = deque([node])
+    while todo:
+        node = todo.popleft()
+        todo.extend(iter_child_nodes(node))
+        yield node
+
+
+class NodeVisitor(object):
+    """
+    A node visitor base class that walks the abstract syntax tree and calls a
+    visitor function for every node found.  This function may return a value
+    which is forwarded by the `visit` method.
+
+    This class is meant to be subclassed, with the subclass adding visitor
+    methods.
+
+    Per default the visitor functions for the nodes are ``'visit_'`` +
+    class name of the node.  So a `TryFinally` node visit function would
+    be `visit_TryFinally`.  This behavior can be changed by overriding
+    the `visit` method.  If no visitor function exists for a node
+    (return value `None`) the `generic_visit` visitor is used instead.
+
+    Don't use the `NodeVisitor` if you want to apply changes to nodes during
+    traversing.  For this a special visitor exists (`NodeTransformer`) that
+    allows modifications.
+    """
+
+    def visit(self, node):
+        """Visit a node."""
+        method = 'visit_' + node.__class__.__name__
+        visitor = getattr(self, method, self.generic_visit)
+        return visitor(node)
+
+    def generic_visit(self, node):
+        """Called if no explicit visitor function exists for a node."""
+        for field, value in iter_fields(node):
+            if isinstance(value, list):
+                for item in value:
+                    if isinstance(item, AST):
+                        self.visit(item)
+            elif isinstance(value, AST):
+                self.visit(value)
+
+
+class NodeTransformer(NodeVisitor):
+    """
+    A :class:`NodeVisitor` subclass that walks the abstract syntax tree and
+    allows modification of nodes.
+
+    The `NodeTransformer` will walk the AST and use the return value of the
+    visitor methods to replace or remove the old node.  If the return value of
+    the visitor method is ``None``, the node will be removed from its location,
+    otherwise it is replaced with the return value.  The return value may be the
+    original node in which case no replacement takes place.
+
+    Here is an example transformer that rewrites all occurrences of name lookups
+    (``foo``) to ``data['foo']``::
+
+       class RewriteName(NodeTransformer):
+
+           def visit_Name(self, node):
+               return copy_location(Subscript(
+                   value=Name(id='data', ctx=Load()),
+                   slice=Index(value=Str(s=node.id)),
+                   ctx=node.ctx
+               ), node)
+
+    Keep in mind that if the node you're operating on has child nodes you must
+    either transform the child nodes yourself or call the :meth:`generic_visit`
+    method for the node first.
+
+    For nodes that were part of a collection of statements (that applies to all
+    statement nodes), the visitor may also return a list of nodes rather than
+    just a single node.
+
+    Usually you use the transformer like this::
+
+       node = YourTransformer().visit(node)
+    """
+
+    def generic_visit(self, node):
+        for field, old_value in iter_fields(node):
+            old_value = getattr(node, field, None)
+            if isinstance(old_value, list):
+                new_values = []
+                for value in old_value:
+                    if isinstance(value, AST):
+                        value = self.visit(value)
+                        if value is None:
+                            continue
+                        elif not isinstance(value, AST):
+                            new_values.extend(value)
+                            continue
+                    new_values.append(value)
+                old_value[:] = new_values
+            elif isinstance(old_value, AST):
+                new_node = self.visit(old_value)
+                if new_node is None:
+                    delattr(node, field)
+                else:
+                    setattr(node, field, new_node)
+        return node
diff --git a/lib-python/3.2/asynchat.py b/lib-python/3.2/asynchat.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/asynchat.py
@@ -0,0 +1,336 @@
+# -*- Mode: Python; tab-width: 4 -*-
+#       Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
+#       Author: Sam Rushing <rushing at nightmare.com>
+
+# ======================================================================
+# Copyright 1996 by Sam Rushing
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+r"""A class supporting chat-style (command/response) protocols.
+
+This class adds support for 'chat' style protocols - where one side
+sends a 'command', and the other sends a response (examples would be
+the common internet protocols - smtp, nntp, ftp, etc..).
+
+The handle_read() method looks at the input stream for the current
+'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
+for multi-line output), calling self.found_terminator() on its
+receipt.
+
+for example:
+Say you build an async nntp client using this class.  At the start
+of the connection, you'll have self.terminator set to '\r\n', in
+order to process the single-line greeting.  Just before issuing a
+'LIST' command you'll set it to '\r\n.\r\n'.  The output of the LIST
+command will be accumulated (using your own 'collect_incoming_data'
+method) up to the terminator, and then control will be returned to
+you - by calling your self.found_terminator() method.
+"""
+import socket
+import asyncore
+from collections import deque
+
+def buffer(obj, start=None, stop=None):
+    # if memoryview objects gain slicing semantics,
+    # this function will change for the better
+    # memoryview used for the TypeError
+    memoryview(obj)
+    if start == None:
+        start = 0
+    if stop == None:
+        stop = len(obj)
+    x = obj[start:stop]
+    ## print("buffer type is: %s"%(type(x),))
+    return x
+
+class async_chat (asyncore.dispatcher):
+    """This is an abstract class.  You must derive from this class, and add
+    the two methods collect_incoming_data() and found_terminator()"""
+
+    # these are overridable defaults
+
+    ac_in_buffer_size       = 4096
+    ac_out_buffer_size      = 4096
+
+    # we don't want to enable the use of encoding by default, because that is a
+    # sign of an application bug that we don't want to pass silently
+
+    use_encoding            = 0
+    encoding                = 'latin1'
+
+    def __init__ (self, sock=None, map=None):
+        # for string terminator matching
+        self.ac_in_buffer = b''
+
+        # we use a list here rather than cStringIO for a few reasons...
+        # del lst[:] is faster than sio.truncate(0)
+        # lst = [] is faster than sio.truncate(0)
+        # cStringIO will be gaining unicode support in py3k, which
+        # will negatively affect the performance of bytes compared to
+        # a ''.join() equivalent
+        self.incoming = []
+
+        # we toss the use of the "simple producer" and replace it with
+        # a pure deque, which the original fifo was a wrapping of
+        self.producer_fifo = deque()
+        asyncore.dispatcher.__init__ (self, sock, map)
+
+    def collect_incoming_data(self, data):
+        raise NotImplementedError("must be implemented in subclass")
+
+    def _collect_incoming_data(self, data):
+        self.incoming.append(data)
+
+    def _get_data(self):
+        d = b''.join(self.incoming)
+        del self.incoming[:]
+        return d
+
+    def found_terminator(self):
+        raise NotImplementedError("must be implemented in subclass")
+
+    def set_terminator (self, term):
+        "Set the input delimiter.  Can be a fixed string of any length, an integer, or None"
+        if isinstance(term, str) and self.use_encoding:
+            term = bytes(term, self.encoding)
+        self.terminator = term
+
+    def get_terminator (self):
+        return self.terminator
+
+    # grab some more data from the socket,
+    # throw it to the collector method,
+    # check for the terminator,
+    # if found, transition to the next state.
+
+    def handle_read (self):
+
+        try:
+            data = self.recv (self.ac_in_buffer_size)
+        except socket.error as why:
+            self.handle_error()
+            return
+
+        if isinstance(data, str) and self.use_encoding:
+            data = bytes(str, self.encoding)
+        self.ac_in_buffer = self.ac_in_buffer + data
+
+        # Continue to search for self.terminator in self.ac_in_buffer,
+        # while calling self.collect_incoming_data.  The while loop
+        # is necessary because we might read several data+terminator
+        # combos with a single recv(4096).
+
+        while self.ac_in_buffer:
+            lb = len(self.ac_in_buffer)
+            terminator = self.get_terminator()
+            if not terminator:
+                # no terminator, collect it all
+                self.collect_incoming_data (self.ac_in_buffer)
+                self.ac_in_buffer = b''
+            elif isinstance(terminator, int):
+                # numeric terminator
+                n = terminator
+                if lb < n:
+                    self.collect_incoming_data (self.ac_in_buffer)
+                    self.ac_in_buffer = b''
+                    self.terminator = self.terminator - lb
+                else:
+                    self.collect_incoming_data (self.ac_in_buffer[:n])
+                    self.ac_in_buffer = self.ac_in_buffer[n:]
+                    self.terminator = 0
+                    self.found_terminator()
+            else:
+                # 3 cases:
+                # 1) end of buffer matches terminator exactly:
+                #    collect data, transition
+                # 2) end of buffer matches some prefix:
+                #    collect data to the prefix
+                # 3) end of buffer does not match any prefix:
+                #    collect data
+                terminator_len = len(terminator)
+                index = self.ac_in_buffer.find(terminator)
+                if index != -1:
+                    # we found the terminator
+                    if index > 0:
+                        # don't bother reporting the empty string (source of subtle bugs)
+                        self.collect_incoming_data (self.ac_in_buffer[:index])
+                    self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
+                    # This does the Right Thing if the terminator is changed here.
+                    self.found_terminator()
+                else:
+                    # check for a prefix of the terminator
+                    index = find_prefix_at_end (self.ac_in_buffer, terminator)
+                    if index:
+                        if index != lb:
+                            # we found a prefix, collect up to the prefix
+                            self.collect_incoming_data (self.ac_in_buffer[:-index])
+                            self.ac_in_buffer = self.ac_in_buffer[-index:]
+                        break
+                    else:
+                        # no prefix, collect it all
+                        self.collect_incoming_data (self.ac_in_buffer)
+                        self.ac_in_buffer = b''
+
+    def handle_write (self):
+        self.initiate_send()
+
+    def handle_close (self):
+        self.close()
+
+    def push (self, data):
+        sabs = self.ac_out_buffer_size
+        if len(data) > sabs:
+            for i in range(0, len(data), sabs):
+                self.producer_fifo.append(data[i:i+sabs])
+        else:
+            self.producer_fifo.append(data)
+        self.initiate_send()
+
+    def push_with_producer (self, producer):
+        self.producer_fifo.append(producer)
+        self.initiate_send()
+
+    def readable (self):
+        "predicate for inclusion in the readable for select()"
+        # cannot use the old predicate, it violates the claim of the
+        # set_terminator method.
+
+        # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
+        return 1
+
+    def writable (self):
+        "predicate for inclusion in the writable for select()"
+        return self.producer_fifo or (not self.connected)
+
+    def close_when_done (self):
+        "automatically close this channel once the outgoing queue is empty"
+        self.producer_fifo.append(None)
+
+    def initiate_send(self):
+        while self.producer_fifo and self.connected:
+            first = self.producer_fifo[0]
+            # handle empty string/buffer or None entry
+            if not first:
+                del self.producer_fifo[0]
+                if first is None:
+                    ## print("first is None")
+                    self.handle_close()
+                    return
+                ## print("first is not None")
+
+            # handle classic producer behavior
+            obs = self.ac_out_buffer_size
+            try:
+                data = buffer(first, 0, obs)
+            except TypeError:
+                data = first.more()
+                if data:
+                    self.producer_fifo.appendleft(data)
+                else:
+                    del self.producer_fifo[0]
+                continue
+
+            if isinstance(data, str) and self.use_encoding:
+                data = bytes(data, self.encoding)
+
+            # send the data
+            try:
+                num_sent = self.send(data)
+            except socket.error:
+                self.handle_error()
+                return
+
+            if num_sent:
+                if num_sent < len(data) or obs < len(first):
+                    self.producer_fifo[0] = first[num_sent:]
+                else:
+                    del self.producer_fifo[0]
+            # we tried to send some actual data
+            return
+
+    def discard_buffers (self):
+        # Emergencies only!
+        self.ac_in_buffer = b''
+        del self.incoming[:]
+        self.producer_fifo.clear()
+
+class simple_producer:
+
+    def __init__ (self, data, buffer_size=512):
+        self.data = data
+        self.buffer_size = buffer_size
+
+    def more (self):
+        if len (self.data) > self.buffer_size:
+            result = self.data[:self.buffer_size]
+            self.data = self.data[self.buffer_size:]
+            return result
+        else:
+            result = self.data
+            self.data = b''
+            return result
+
+class fifo:
+    def __init__ (self, list=None):
+        if not list:
+            self.list = deque()
+        else:
+            self.list = deque(list)
+
+    def __len__ (self):
+        return len(self.list)
+
+    def is_empty (self):
+        return not self.list
+
+    def first (self):
+        return self.list[0]
+
+    def push (self, data):
+        self.list.append(data)
+
+    def pop (self):
+        if self.list:
+            return (1, self.list.popleft())
+        else:
+            return (0, None)
+
+# Given 'haystack', see if any prefix of 'needle' is at its end.  This
+# assumes an exact match has already been checked.  Return the number of
+# characters matched.
+# for example:
+# f_p_a_e ("qwerty\r", "\r\n") => 1
+# f_p_a_e ("qwertydkjf", "\r\n") => 0
+# f_p_a_e ("qwerty\r\n", "\r\n") => <undefined>
+
+# this could maybe be made faster with a computed regex?
+# [answer: no; circa Python-2.0, Jan 2001]
+# new python:   28961/s
+# old python:   18307/s
+# re:        12820/s
+# regex:     14035/s
+
+def find_prefix_at_end (haystack, needle):
+    l = len(needle) - 1
+    while l and not haystack.endswith(needle[:l]):
+        l -= 1
+    return l
diff --git a/lib-python/3.2/asyncore.py b/lib-python/3.2/asyncore.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/asyncore.py
@@ -0,0 +1,663 @@
+# -*- Mode: Python -*-
+#   Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
+#   Author: Sam Rushing <rushing at nightmare.com>
+
+# ======================================================================
+# Copyright 1996 by Sam Rushing
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+"""Basic infrastructure for asynchronous socket service clients and servers.
+
+There are only two ways to have a program on a single processor do "more
+than one thing at a time".  Multi-threaded programming is the simplest and
+most popular way to do it, but there is another very different technique,
+that lets you have nearly all the advantages of multi-threading, without
+actually using multiple threads. it's really only practical if your program
+is largely I/O bound. If your program is CPU bound, then pre-emptive
+scheduled threads are probably what you really need. Network servers are
+rarely CPU-bound, however.
+
+If your operating system supports the select() system call in its I/O
+library (and nearly all do), then you can use it to juggle multiple
+communication channels at once; doing other work while your I/O is taking
+place in the "background."  Although this strategy can seem strange and
+complex, especially at first, it is in many ways easier to understand and
+control than multi-threaded programming. The module documented here solves
+many of the difficult problems for you, making the task of building
+sophisticated high-performance network servers and clients a snap.
+"""
+
+import select
+import socket
+import sys
+import time
+import warnings
+
+import os
+from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
+     ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
+     errorcode
+
+_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
+                           EBADF))
+
+try:
+    socket_map
+except NameError:
+    socket_map = {}
+
+def _strerror(err):
+    try:
+        return os.strerror(err)
+    except (ValueError, OverflowError, NameError):
+        if err in errorcode:
+            return errorcode[err]
+        return "Unknown error %s" %err
+
+class ExitNow(Exception):
+    pass
+
+_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit)
+
+def read(obj):
+    try:
+        obj.handle_read_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def write(obj):
+    try:
+        obj.handle_write_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def _exception(obj):
+    try:
+        obj.handle_expt_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def readwrite(obj, flags):
+    try:
+        if flags & select.POLLIN:
+            obj.handle_read_event()
+        if flags & select.POLLOUT:
+            obj.handle_write_event()
+        if flags & select.POLLPRI:
+            obj.handle_expt_event()
+        if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
+            obj.handle_close()
+    except socket.error as e:
+        if e.args[0] not in _DISCONNECTED:
+            obj.handle_error()
+        else:
+            obj.handle_close()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def poll(timeout=0.0, map=None):
+    if map is None:
+        map = socket_map
+    if map:
+        r = []; w = []; e = []
+        for fd, obj in list(map.items()):
+            is_r = obj.readable()
+            is_w = obj.writable()
+            if is_r:
+                r.append(fd)
+            # accepting sockets should not be writable
+            if is_w and not obj.accepting:
+                w.append(fd)
+            if is_r or is_w:
+                e.append(fd)
+        if [] == r == w == e:
+            time.sleep(timeout)
+            return
+
+        try:
+            r, w, e = select.select(r, w, e, timeout)
+        except select.error as err:
+            if err.args[0] != EINTR:
+                raise
+            else:
+                return
+
+        for fd in r:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            read(obj)
+
+        for fd in w:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            write(obj)
+
+        for fd in e:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            _exception(obj)
+
+def poll2(timeout=0.0, map=None):
+    # Use the poll() support added to the select module in Python 2.0
+    if map is None:
+        map = socket_map
+    if timeout is not None:
+        # timeout is in milliseconds
+        timeout = int(timeout*1000)
+    pollster = select.poll()
+    if map:
+        for fd, obj in list(map.items()):
+            flags = 0
+            if obj.readable():
+                flags |= select.POLLIN | select.POLLPRI
+            # accepting sockets should not be writable
+            if obj.writable() and not obj.accepting:
+                flags |= select.POLLOUT
+            if flags:
+                # Only check for exceptions if object was either readable
+                # or writable.
+                flags |= select.POLLERR | select.POLLHUP | select.POLLNVAL
+                pollster.register(fd, flags)
+        try:
+            r = pollster.poll(timeout)
+        except select.error as err:
+            if err.args[0] != EINTR:
+                raise
+            r = []
+        for fd, flags in r:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            readwrite(obj, flags)
+
+poll3 = poll2                           # Alias for backward compatibility
+
+def loop(timeout=30.0, use_poll=False, map=None, count=None):
+    if map is None:
+        map = socket_map
+
+    if use_poll and hasattr(select, 'poll'):
+        poll_fun = poll2
+    else:
+        poll_fun = poll
+
+    if count is None:
+        while map:
+            poll_fun(timeout, map)
+
+    else:
+        while map and count > 0:
+            poll_fun(timeout, map)
+            count = count - 1
+
+class dispatcher:
+
+    debug = False
+    connected = False
+    accepting = False
+    closing = False
+    addr = None
+    ignore_log_types = frozenset(['warning'])
+
+    def __init__(self, sock=None, map=None):
+        if map is None:
+            self._map = socket_map
+        else:
+            self._map = map
+
+        self._fileno = None
+
+        if sock:
+            # Set to nonblocking just to make sure for cases where we
+            # get a socket from a blocking source.
+            sock.setblocking(0)
+            self.set_socket(sock, map)
+            self.connected = True
+            # The constructor no longer requires that the socket
+            # passed be connected.
+            try:
+                self.addr = sock.getpeername()
+            except socket.error as err:
+                if err.args[0] == ENOTCONN:
+                    # To handle the case where we got an unconnected
+                    # socket.
+                    self.connected = False
+                else:
+                    # The socket is broken in some unknown way, alert
+                    # the user and remove it from the map (to prevent
+                    # polling of broken sockets).
+                    self.del_channel(map)
+                    raise
+        else:
+            self.socket = None
+
+    def __repr__(self):
+        status = [self.__class__.__module__+"."+self.__class__.__name__]
+        if self.accepting and self.addr:
+            status.append('listening')
+        elif self.connected:
+            status.append('connected')
+        if self.addr is not None:
+            try:
+                status.append('%s:%d' % self.addr)
+            except TypeError:
+                status.append(repr(self.addr))
+        return '<%s at %#x>' % (' '.join(status), id(self))
+
+    __str__ = __repr__
+
+    def add_channel(self, map=None):
+        #self.log_info('adding channel %s' % self)
+        if map is None:
+            map = self._map
+        map[self._fileno] = self
+
+    def del_channel(self, map=None):
+        fd = self._fileno
+        if map is None:
+            map = self._map
+        if fd in map:
+            #self.log_info('closing channel %d:%s' % (fd, self))
+            del map[fd]
+        self._fileno = None
+
+    def create_socket(self, family, type):
+        self.family_and_type = family, type
+        sock = socket.socket(family, type)
+        sock.setblocking(0)
+        self.set_socket(sock)
+
+    def set_socket(self, sock, map=None):
+        self.socket = sock
+##        self.__dict__['socket'] = sock
+        self._fileno = sock.fileno()
+        self.add_channel(map)
+
+    def set_reuse_addr(self):
+        # try to re-use a server port if possible
+        try:
+            self.socket.setsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEADDR,
+                self.socket.getsockopt(socket.SOL_SOCKET,
+                                       socket.SO_REUSEADDR) | 1
+                )
+        except socket.error:
+            pass
+
+    # ==================================================
+    # predicates for select()
+    # these are used as filters for the lists of sockets
+    # to pass to select().
+    # ==================================================
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return True
+
+    # ==================================================
+    # socket object methods.
+    # ==================================================
+
+    def listen(self, num):
+        self.accepting = True
+        if os.name == 'nt' and num > 5:
+            num = 5
+        return self.socket.listen(num)
+
+    def bind(self, addr):
+        self.addr = addr
+        return self.socket.bind(addr)
+
+    def connect(self, address):
+        self.connected = False
+        err = self.socket.connect_ex(address)
+        if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
+        or err == EINVAL and os.name in ('nt', 'ce'):
+            return
+        if err in (0, EISCONN):
+            self.addr = address
+            self.handle_connect_event()
+        else:
+            raise socket.error(err, errorcode[err])
+
+    def accept(self):
+        # XXX can return either an address pair or None
+        try:
+            conn, addr = self.socket.accept()
+        except TypeError:
+            return None
+        except socket.error as why:
+            if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
+                return None
+            else:
+                raise
+        else:
+            return conn, addr
+
+    def send(self, data):
+        try:
+            result = self.socket.send(data)
+            return result
+        except socket.error as why:
+            if why.args[0] == EWOULDBLOCK:
+                return 0
+            elif why.args[0] in _DISCONNECTED:
+                self.handle_close()
+                return 0
+            else:
+                raise
+
+    def recv(self, buffer_size):
+        try:
+            data = self.socket.recv(buffer_size)
+            if not data:
+                # a closed connection is indicated by signaling
+                # a read condition, and having recv() return 0.
+                self.handle_close()
+                return b''
+            else:
+                return data
+        except socket.error as why:
+            # winsock sometimes throws ENOTCONN
+            if why.args[0] in _DISCONNECTED:
+                self.handle_close()
+                return b''
+            else:
+                raise
+
+    def close(self):
+        self.connected = False
+        self.accepting = False
+        self.del_channel()
+        try:
+            self.socket.close()
+        except socket.error as why:
+            if why.args[0] not in (ENOTCONN, EBADF):
+                raise
+
+    # cheap inheritance, used to pass all other attribute
+    # references to the underlying socket object.
+    def __getattr__(self, attr):
+        try:
+            retattr = getattr(self.socket, attr)
+        except AttributeError:
+            raise AttributeError("%s instance has no attribute '%s'"
+                                 %(self.__class__.__name__, attr))
+        else:
+            msg = "%(me)s.%(attr)s is deprecated; use %(me)s.socket.%(attr)s " \
+                  "instead" % {'me' : self.__class__.__name__, 'attr' : attr}
+            warnings.warn(msg, DeprecationWarning, stacklevel=2)
+            return retattr
+
+    # log and log_info may be overridden to provide more sophisticated
+    # logging and warning methods. In general, log is for 'hit' logging
+    # and 'log_info' is for informational, warning and error logging.
+
+    def log(self, message):
+        sys.stderr.write('log: %s\n' % str(message))
+
+    def log_info(self, message, type='info'):
+        if type not in self.ignore_log_types:
+            print('%s: %s' % (type, message))
+
+    def handle_read_event(self):
+        if self.accepting:
+            # accepting sockets are never connected, they "spawn" new
+            # sockets that are connected
+            self.handle_accept()
+        elif not self.connected:
+            self.handle_connect_event()
+            self.handle_read()
+        else:
+            self.handle_read()
+
+    def handle_connect_event(self):
+        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+        if err != 0:
+            raise socket.error(err, _strerror(err))
+        self.handle_connect()
+        self.connected = True
+
+    def handle_write_event(self):
+        if self.accepting:
+            # Accepting sockets shouldn't get a write event.
+            # We will pretend it didn't happen.
+            return
+
+        if not self.connected:
+            #check for errors
+            err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+            if err != 0:
+                raise socket.error(err, _strerror(err))
+
+            self.handle_connect_event()
+        self.handle_write()
+
+    def handle_expt_event(self):
+        # handle_expt_event() is called if there might be an error on the
+        # socket, or if there is OOB data
+        # check for the error condition first
+        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+        if err != 0:
+            # we can get here when select.select() says that there is an
+            # exceptional condition on the socket
+            # since there is an error, we'll go ahead and close the socket
+            # like we would in a subclassed handle_read() that received no
+            # data
+            self.handle_close()
+        else:
+            self.handle_expt()
+
+    def handle_error(self):
+        nil, t, v, tbinfo = compact_traceback()
+
+        # sometimes a user repr method will crash.
+        try:
+            self_repr = repr(self)
+        except:
+            self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
+
+        self.log_info(
+            'uncaptured python exception, closing channel %s (%s:%s %s)' % (
+                self_repr,
+                t,
+                v,
+                tbinfo
+                ),
+            'error'
+            )
+        self.handle_close()
+
+    def handle_expt(self):
+        self.log_info('unhandled incoming priority event', 'warning')
+
+    def handle_read(self):
+        self.log_info('unhandled read event', 'warning')
+
+    def handle_write(self):
+        self.log_info('unhandled write event', 'warning')
+
+    def handle_connect(self):
+        self.log_info('unhandled connect event', 'warning')
+
+    def handle_accept(self):
+        pair = self.accept()
+        if pair is not None:
+            self.handle_accepted(*pair)
+
+    def handle_accepted(self, sock, addr):
+        sock.close()
+        self.log_info('unhandled accepted event', 'warning')
+
+    def handle_close(self):
+        self.log_info('unhandled close event', 'warning')
+        self.close()
+
+# ---------------------------------------------------------------------------
+# adds simple buffered output capability, useful for simple clients.
+# [for more sophisticated usage use asynchat.async_chat]
+# ---------------------------------------------------------------------------
+
+class dispatcher_with_send(dispatcher):
+
+    def __init__(self, sock=None, map=None):
+        dispatcher.__init__(self, sock, map)
+        self.out_buffer = b''
+
+    def initiate_send(self):
+        num_sent = 0
+        num_sent = dispatcher.send(self, self.out_buffer[:512])
+        self.out_buffer = self.out_buffer[num_sent:]
+
+    def handle_write(self):
+        self.initiate_send()
+
+    def writable(self):
+        return (not self.connected) or len(self.out_buffer)
+
+    def send(self, data):
+        if self.debug:
+            self.log_info('sending %s' % repr(data))
+        self.out_buffer = self.out_buffer + data
+        self.initiate_send()
+
+# ---------------------------------------------------------------------------
+# used for debugging.
+# ---------------------------------------------------------------------------
+
+def compact_traceback():
+    t, v, tb = sys.exc_info()
+    tbinfo = []
+    if not tb: # Must have a traceback
+        raise AssertionError("traceback does not exist")
+    while tb:
+        tbinfo.append((
+            tb.tb_frame.f_code.co_filename,
+            tb.tb_frame.f_code.co_name,
+            str(tb.tb_lineno)
+            ))
+        tb = tb.tb_next
+
+    # just to be safe
+    del tb
+
+    file, function, line = tbinfo[-1]
+    info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
+    return (file, function, line), t, v, info
+
+def close_all(map=None, ignore_all=False):
+    if map is None:
+        map = socket_map
+    for x in list(map.values()):
+        try:
+            x.close()
+        except OSError as x:
+            if x.args[0] == EBADF:
+                pass
+            elif not ignore_all:
+                raise
+        except _reraised_exceptions:
+            raise
+        except:
+            if not ignore_all:
+                raise
+    map.clear()
+
+# Asynchronous File I/O:
+#
+# After a little research (reading man pages on various unixen, and
+# digging through the linux kernel), I've determined that select()
+# isn't meant for doing asynchronous file i/o.
+# Heartening, though - reading linux/mm/filemap.c shows that linux
+# supports asynchronous read-ahead.  So _MOST_ of the time, the data
+# will be sitting in memory for us already when we go to read it.
+#
+# What other OS's (besides NT) support async file i/o?  [VMS?]
+#
+# Regardless, this is useful for pipes, and stdin/stdout...
+
+if os.name == 'posix':
+    import fcntl
+
+    class file_wrapper:
+        # Here we override just enough to make a file
+        # look like a socket for the purposes of asyncore.
+        # The passed fd is automatically os.dup()'d
+
+        def __init__(self, fd):
+            self.fd = os.dup(fd)
+
+        def recv(self, *args):
+            return os.read(self.fd, *args)
+
+        def send(self, *args):
+            return os.write(self.fd, *args)
+
+        def getsockopt(self, level, optname, buflen=None):
+            if (level == socket.SOL_SOCKET and
+                optname == socket.SO_ERROR and
+                not buflen):
+                return 0
+            raise NotImplementedError("Only asyncore specific behaviour "
+                                      "implemented.")
+
+        read = recv
+        write = send
+
+        def close(self):
+            os.close(self.fd)
+
+        def fileno(self):
+            return self.fd
+
+    class file_dispatcher(dispatcher):
+
+        def __init__(self, fd, map=None):
+            dispatcher.__init__(self, None, map)
+            self.connected = True
+            try:
+                fd = fd.fileno()
+            except AttributeError:
+                pass
+            self.set_file(fd)
+            # set it to non-blocking mode
+            flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0)
+            flags = flags | os.O_NONBLOCK
+            fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+        def set_file(self, fd):
+            self.socket = file_wrapper(fd)
+            self._fileno = self.socket.fileno()
+            self.add_channel()
diff --git a/lib-python/3.2/base64.py b/lib-python/3.2/base64.py
new file mode 100755
--- /dev/null
+++ b/lib-python/3.2/base64.py
@@ -0,0 +1,408 @@
+#! /usr/bin/env python3
+
+"""RFC 3548: Base16, Base32, Base64 Data Encodings"""
+
+# Modified 04-Oct-1995 by Jack Jansen to use binascii module
+# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
+# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere
+
+import re
+import struct
+import binascii
+
+
+__all__ = [
+    # Legacy interface exports traditional RFC 1521 Base64 encodings
+    'encode', 'decode', 'encodebytes', 'decodebytes',
+    # Generalized interface for other encodings
+    'b64encode', 'b64decode', 'b32encode', 'b32decode',
+    'b16encode', 'b16decode',
+    # Standard Base64 encoding
+    'standard_b64encode', 'standard_b64decode',
+    # Some common Base64 alternatives.  As referenced by RFC 3458, see thread
+    # starting at:
+    #
+    # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html
+    'urlsafe_b64encode', 'urlsafe_b64decode',
+    ]
+
+
+bytes_types = (bytes, bytearray)  # Types acceptable as binary data
+
+
+def _translate(s, altchars):
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    translation = bytearray(range(256))
+    for k, v in altchars.items():
+        translation[ord(k)] = v[0]
+    return s.translate(translation)
+
+
+
+# Base64 encoding/decoding uses binascii
+
+def b64encode(s, altchars=None):
+    """Encode a byte string using Base64.
+
+    s is the byte string to encode.  Optional altchars must be a byte
+    string of length 2 which specifies an alternative alphabet for the
+    '+' and '/' characters.  This allows an application to
+    e.g. generate url or filesystem safe Base64 strings.
+
+    The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    # Strip off the trailing newline
+    encoded = binascii.b2a_base64(s)[:-1]
+    if altchars is not None:
+        if not isinstance(altchars, bytes_types):
+            raise TypeError("expected bytes, not %s"
+                            % altchars.__class__.__name__)
+        assert len(altchars) == 2, repr(altchars)
+        return _translate(encoded, {'+': altchars[0:1], '/': altchars[1:2]})
+    return encoded
+
+
+def b64decode(s, altchars=None, validate=False):
+    """Decode a Base64 encoded byte string.
+
+    s is the byte string to decode.  Optional altchars must be a
+    string of length 2 which specifies the alternative alphabet used
+    instead of the '+' and '/' characters.
+
+    The decoded string is returned.  A binascii.Error is raised if s is
+    incorrectly padded.
+
+    If validate is False (the default), non-base64-alphabet characters are
+    discarded prior to the padding check.  If validate is True,
+    non-base64-alphabet characters in the input result in a binascii.Error.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    if altchars is not None:
+        if not isinstance(altchars, bytes_types):
+            raise TypeError("expected bytes, not %s"
+                            % altchars.__class__.__name__)
+        assert len(altchars) == 2, repr(altchars)
+        s = _translate(s, {chr(altchars[0]): b'+', chr(altchars[1]): b'/'})
+    if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
+        raise binascii.Error('Non-base64 digit found')
+    return binascii.a2b_base64(s)
+
+
+def standard_b64encode(s):
+    """Encode a byte string using the standard Base64 alphabet.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    return b64encode(s)
+
+def standard_b64decode(s):
+    """Decode a byte string encoded with the standard Base64 alphabet.
+
+    s is the byte string to decode.  The decoded byte string is
+    returned.  binascii.Error is raised if the input is incorrectly
+    padded or if there are non-alphabet characters present in the
+    input.
+    """
+    return b64decode(s)
+
+def urlsafe_b64encode(s):
+    """Encode a byte string using a url-safe Base64 alphabet.
+
+    s is the byte string to encode.  The encoded byte string is
+    returned.  The alphabet uses '-' instead of '+' and '_' instead of
+    '/'.
+    """
+    return b64encode(s, b'-_')
+
+def urlsafe_b64decode(s):
+    """Decode a byte string encoded with the standard Base64 alphabet.
+
+    s is the byte string to decode.  The decoded byte string is
+    returned.  binascii.Error is raised if the input is incorrectly
+    padded or if there are non-alphabet characters present in the
+    input.
+
+    The alphabet uses '-' instead of '+' and '_' instead of '/'.
+    """
+    return b64decode(s, b'-_')
+
+
+
+# Base32 encoding/decoding must be done in Python
+_b32alphabet = {
+    0: b'A',  9: b'J', 18: b'S', 27: b'3',
+    1: b'B', 10: b'K', 19: b'T', 28: b'4',
+    2: b'C', 11: b'L', 20: b'U', 29: b'5',
+    3: b'D', 12: b'M', 21: b'V', 30: b'6',
+    4: b'E', 13: b'N', 22: b'W', 31: b'7',
+    5: b'F', 14: b'O', 23: b'X',
+    6: b'G', 15: b'P', 24: b'Y',
+    7: b'H', 16: b'Q', 25: b'Z',
+    8: b'I', 17: b'R', 26: b'2',
+    }
+
+_b32tab = [v[0] for k, v in sorted(_b32alphabet.items())]
+_b32rev = dict([(v[0], k) for k, v in _b32alphabet.items()])
+
+
+def b32encode(s):
+    """Encode a byte string using Base32.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    quanta, leftover = divmod(len(s), 5)
+    # Pad the last quantum with zero bits if necessary
+    if leftover:
+        s = s + bytes(5 - leftover)  # Don't use += !
+        quanta += 1
+    encoded = bytes()
+    for i in range(quanta):
+        # c1 and c2 are 16 bits wide, c3 is 8 bits wide.  The intent of this
+        # code is to process the 40 bits in units of 5 bits.  So we take the 1
+        # leftover bit of c1 and tack it onto c2.  Then we take the 2 leftover
+        # bits of c2 and tack them onto c3.  The shifts and masks are intended
+        # to give us values of exactly 5 bits in width.
+        c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5])
+        c2 += (c1 & 1) << 16 # 17 bits wide
+        c3 += (c2 & 3) << 8  # 10 bits wide
+        encoded += bytes([_b32tab[c1 >> 11],         # bits 1 - 5
+                          _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10
+                          _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15
+                          _b32tab[c2 >> 12],         # bits 16 - 20 (1 - 5)
+                          _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10)
+                          _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15)
+                          _b32tab[c3 >> 5],          # bits 31 - 35 (1 - 5)
+                          _b32tab[c3 & 0x1f],        # bits 36 - 40 (1 - 5)
+                          ])
+    # Adjust for any leftover partial quanta
+    if leftover == 1:
+        return encoded[:-6] + b'======'
+    elif leftover == 2:
+        return encoded[:-4] + b'===='
+    elif leftover == 3:
+        return encoded[:-3] + b'==='
+    elif leftover == 4:
+        return encoded[:-1] + b'='
+    return encoded
+
+
+def b32decode(s, casefold=False, map01=None):
+    """Decode a Base32 encoded byte string.
+
+    s is the byte string to decode.  Optional casefold is a flag
+    specifying whether a lowercase alphabet is acceptable as input.
+    For security purposes, the default is False.
+
+    RFC 3548 allows for optional mapping of the digit 0 (zero) to the
+    letter O (oh), and for optional mapping of the digit 1 (one) to
+    either the letter I (eye) or letter L (el).  The optional argument
+    map01 when not None, specifies which letter the digit 1 should be
+    mapped to (when map01 is not None, the digit 0 is always mapped to
+    the letter O).  For security purposes the default is None, so that
+    0 and 1 are not allowed in the input.
+
+    The decoded byte string is returned.  binascii.Error is raised if
+    the input is incorrectly padded or if there are non-alphabet
+    characters present in the input.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    quanta, leftover = divmod(len(s), 8)
+    if leftover:
+        raise binascii.Error('Incorrect padding')
+    # Handle section 2.4 zero and one mapping.  The flag map01 will be either
+    # False, or the character to map the digit 1 (one) to.  It should be
+    # either L (el) or I (eye).
+    if map01 is not None:
+        if not isinstance(map01, bytes_types):
+            raise TypeError("expected bytes, not %s" % map01.__class__.__name__)
+        assert len(map01) == 1, repr(map01)
+        s = _translate(s, {b'0': b'O', b'1': map01})
+    if casefold:
+        s = s.upper()
+    # Strip off pad characters from the right.  We need to count the pad
+    # characters because this will tell us how many null bytes to remove from
+    # the end of the decoded string.
+    padchars = 0
+    mo = re.search(b'(?P<pad>[=]*)$', s)
+    if mo:
+        padchars = len(mo.group('pad'))
+        if padchars > 0:
+            s = s[:-padchars]
+    # Now decode the full quanta
+    parts = []
+    acc = 0
+    shift = 35
+    for c in s:
+        val = _b32rev.get(c)
+        if val is None:
+            raise TypeError('Non-base32 digit found')
+        acc += _b32rev[c] << shift
+        shift -= 5
+        if shift < 0:
+            parts.append(binascii.unhexlify(bytes('%010x' % acc, "ascii")))
+            acc = 0
+            shift = 35
+    # Process the last, partial quanta
+    last = binascii.unhexlify(bytes('%010x' % acc, "ascii"))
+    if padchars == 0:
+        last = b''                      # No characters
+    elif padchars == 1:
+        last = last[:-1]
+    elif padchars == 3:
+        last = last[:-2]
+    elif padchars == 4:
+        last = last[:-3]
+    elif padchars == 6:
+        last = last[:-4]
+    else:
+        raise binascii.Error('Incorrect padding')
+    parts.append(last)
+    return b''.join(parts)
+
+
+
+# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
+# lowercase.  The RFC also recommends against accepting input case
+# insensitively.
+def b16encode(s):
+    """Encode a byte string using Base16.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    return binascii.hexlify(s).upper()
+
+
+def b16decode(s, casefold=False):
+    """Decode a Base16 encoded byte string.
+
+    s is the byte string to decode.  Optional casefold is a flag
+    specifying whether a lowercase alphabet is acceptable as input.
+    For security purposes, the default is False.
+
+    The decoded byte string is returned.  binascii.Error is raised if
+    s were incorrectly padded or if there are non-alphabet characters
+    present in the string.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    if casefold:
+        s = s.upper()
+    if re.search(b'[^0-9A-F]', s):
+        raise binascii.Error('Non-base16 digit found')
+    return binascii.unhexlify(s)
+
+
+
+# Legacy interface.  This code could be cleaned up since I don't believe
+# binascii has any line length limitations.  It just doesn't seem worth it
+# though.  The files should be opened in binary mode.
+
+MAXLINESIZE = 76 # Excluding the CRLF
+MAXBINSIZE = (MAXLINESIZE//4)*3
+
+def encode(input, output):
+    """Encode a file; input and output are binary files."""
+    while True:
+        s = input.read(MAXBINSIZE)
+        if not s:
+            break
+        while len(s) < MAXBINSIZE:
+            ns = input.read(MAXBINSIZE-len(s))
+            if not ns:
+                break
+            s += ns
+        line = binascii.b2a_base64(s)
+        output.write(line)
+
+
+def decode(input, output):
+    """Decode a file; input and output are binary files."""
+    while True:
+        line = input.readline()
+        if not line:
+            break
+        s = binascii.a2b_base64(line)
+        output.write(s)
+
+
+def encodebytes(s):
+    """Encode a bytestring into a bytestring containing multiple lines
+    of base-64 data."""
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    pieces = []
+    for i in range(0, len(s), MAXBINSIZE):
+        chunk = s[i : i + MAXBINSIZE]
+        pieces.append(binascii.b2a_base64(chunk))
+    return b"".join(pieces)
+
+def encodestring(s):
+    """Legacy alias of encodebytes()."""
+    import warnings
+    warnings.warn("encodestring() is a deprecated alias, use encodebytes()",
+                  DeprecationWarning, 2)
+    return encodebytes(s)
+
+
+def decodebytes(s):
+    """Decode a bytestring of base-64 data into a bytestring."""
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    return binascii.a2b_base64(s)
+
+def decodestring(s):
+    """Legacy alias of decodebytes()."""
+    import warnings
+    warnings.warn("decodestring() is a deprecated alias, use decodebytes()",
+                  DeprecationWarning, 2)
+    return decodebytes(s)
+
+
+# Usable as a script...
+def main():
+    """Small main program"""
+    import sys, getopt
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'deut')
+    except getopt.error as msg:
+        sys.stdout = sys.stderr
+        print(msg)
+        print("""usage: %s [-d|-e|-u|-t] [file|-]
+        -d, -u: decode
+        -e: encode (default)
+        -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
+        sys.exit(2)
+    func = encode
+    for o, a in opts:
+        if o == '-e': func = encode
+        if o == '-d': func = decode
+        if o == '-u': func = decode
+        if o == '-t': test(); return
+    if args and args[0] != '-':
+        with open(args[0], 'rb') as f:
+            func(f, sys.stdout.buffer)
+    else:
+        func(sys.stdin.buffer, sys.stdout.buffer)
+
+
+def test():
+    s0 = b"Aladdin:open sesame"
+    print(repr(s0))
+    s1 = encodebytes(s0)
+    print(repr(s1))
+    s2 = decodebytes(s1)
+    print(repr(s2))
+    assert s0 == s2
+
+
+if __name__ == '__main__':
+    main()
diff --git a/lib-python/3.2/bdb.py b/lib-python/3.2/bdb.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/bdb.py
@@ -0,0 +1,634 @@
+"""Debugger basics"""
+
+import fnmatch
+import sys
+import os
+
+__all__ = ["BdbQuit", "Bdb", "Breakpoint"]
+
+class BdbQuit(Exception):
+    """Exception to give up completely."""
+
+
+class Bdb:
+    """Generic Python debugger base class.
+
+    This class takes care of details of the trace facility;
+    a derived class should implement user interaction.
+    The standard debugger class (pdb.Pdb) is an example.
+    """
+
+    def __init__(self, skip=None):
+        self.skip = set(skip) if skip else None
+        self.breaks = {}
+        self.fncache = {}
+
+    def canonic(self, filename):
+        if filename == "<" + filename[1:-1] + ">":
+            return filename
+        canonic = self.fncache.get(filename)
+        if not canonic:
+            canonic = os.path.abspath(filename)
+            canonic = os.path.normcase(canonic)
+            self.fncache[filename] = canonic
+        return canonic
+
+    def reset(self):
+        import linecache
+        linecache.checkcache()
+        self.botframe = None
+        self._set_stopinfo(None, None)
+
+    def trace_dispatch(self, frame, event, arg):
+        if self.quitting:
+            return # None
+        if event == 'line':
+            return self.dispatch_line(frame)
+        if event == 'call':
+            return self.dispatch_call(frame, arg)
+        if event == 'return':
+            return self.dispatch_return(frame, arg)
+        if event == 'exception':
+            return self.dispatch_exception(frame, arg)
+        if event == 'c_call':
+            return self.trace_dispatch
+        if event == 'c_exception':
+            return self.trace_dispatch
+        if event == 'c_return':
+            return self.trace_dispatch
+        print('bdb.Bdb.dispatch: unknown debugging event:', repr(event))
+        return self.trace_dispatch
+
+    def dispatch_line(self, frame):
+        if self.stop_here(frame) or self.break_here(frame):
+            self.user_line(frame)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_call(self, frame, arg):
+        # XXX 'arg' is no longer used
+        if self.botframe is None:
+            # First call of dispatch since reset()
+            self.botframe = frame.f_back # (CT) Note that this may also be None!
+            return self.trace_dispatch
+        if not (self.stop_here(frame) or self.break_anywhere(frame)):
+            # No need to trace this function
+            return # None
+        self.user_call(frame, arg)
+        if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_return(self, frame, arg):
+        if self.stop_here(frame) or frame == self.returnframe:
+            self.user_return(frame, arg)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_exception(self, frame, arg):
+        if self.stop_here(frame):
+            self.user_exception(frame, arg)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    # Normally derived classes don't override the following
+    # methods, but they may if they want to redefine the
+    # definition of stopping and breakpoints.
+
+    def is_skipped_module(self, module_name):
+        for pattern in self.skip:
+            if fnmatch.fnmatch(module_name, pattern):
+                return True
+        return False
+
+    def stop_here(self, frame):
+        # (CT) stopframe may now also be None, see dispatch_call.
+        # (CT) the former test for None is therefore removed from here.
+        if self.skip and \
+               self.is_skipped_module(frame.f_globals.get('__name__')):
+            return False
+        if frame is self.stopframe:
+            if self.stoplineno == -1:
+                return False
+            return frame.f_lineno >= self.stoplineno
+        while frame is not None and frame is not self.stopframe:
+            if frame is self.botframe:
+                return True
+            frame = frame.f_back
+        return False
+
+    def break_here(self, frame):
+        filename = self.canonic(frame.f_code.co_filename)
+        if filename not in self.breaks:
+            return False
+        lineno = frame.f_lineno
+        if lineno not in self.breaks[filename]:
+            # The line itself has no breakpoint, but maybe the line is the
+            # first line of a function with breakpoint set by function name.
+            lineno = frame.f_code.co_firstlineno
+            if lineno not in self.breaks[filename]:
+                return False
+
+        # flag says ok to delete temp. bp
+        (bp, flag) = effective(filename, lineno, frame)
+        if bp:
+            self.currentbp = bp.number
+            if (flag and bp.temporary):
+                self.do_clear(str(bp.number))
+            return True
+        else:
+            return False
+
+    def do_clear(self, arg):
+        raise NotImplementedError("subclass of bdb must implement do_clear()")
+
+    def break_anywhere(self, frame):
+        return self.canonic(frame.f_code.co_filename) in self.breaks
+
+    # Derived classes should override the user_* methods
+    # to gain control.
+
+    def user_call(self, frame, argument_list):
+        """This method is called when there is the remote possibility
+        that we ever need to stop in this function."""
+        pass
+
+    def user_line(self, frame):
+        """This method is called when we stop or break at this line."""
+        pass
+
+    def user_return(self, frame, return_value):
+        """This method is called when a return trap is set here."""
+        pass
+
+    def user_exception(self, frame, exc_info):
+        """This method is called if an exception occurs,
+        but only if we are to stop at or just below this level."""
+        pass
+
+    def _set_stopinfo(self, stopframe, returnframe, stoplineno=0):
+        self.stopframe = stopframe
+        self.returnframe = returnframe
+        self.quitting = False
+        # stoplineno >= 0 means: stop at line >= the stoplineno
+        # stoplineno -1 means: don't stop at all
+        self.stoplineno = stoplineno
+
+    # Derived classes and clients can call the following methods
+    # to affect the stepping state.
+
+    def set_until(self, frame, lineno=None):
+        """Stop when the line with the line no greater than the current one is
+        reached or when returning from current frame"""
+        # the name "until" is borrowed from gdb
+        if lineno is None:
+            lineno = frame.f_lineno + 1
+        self._set_stopinfo(frame, frame, lineno)
+
+    def set_step(self):
+        """Stop after one line of code."""
+        self._set_stopinfo(None, None)
+
+    def set_next(self, frame):
+        """Stop on the next line in or below the given frame."""
+        self._set_stopinfo(frame, None)
+
+    def set_return(self, frame):
+        """Stop when returning from the given frame."""
+        self._set_stopinfo(frame.f_back, frame)
+
+    def set_trace(self, frame=None):
+        """Start debugging from `frame`.
+
+        If frame is not specified, debugging starts from caller's frame.
+        """
+        if frame is None:
+            frame = sys._getframe().f_back
+        self.reset()
+        while frame:
+            frame.f_trace = self.trace_dispatch
+            self.botframe = frame
+            frame = frame.f_back
+        self.set_step()
+        sys.settrace(self.trace_dispatch)
+
+    def set_continue(self):
+        # Don't stop except at breakpoints or when finished
+        self._set_stopinfo(self.botframe, None, -1)
+        if not self.breaks:
+            # no breakpoints; run without debugger overhead
+            sys.settrace(None)
+            frame = sys._getframe().f_back
+            while frame and frame is not self.botframe:
+                del frame.f_trace
+                frame = frame.f_back
+
+    def set_quit(self):
+        self.stopframe = self.botframe
+        self.returnframe = None
+        self.quitting = True
+        sys.settrace(None)
+
+    # Derived classes and clients can call the following methods
+    # to manipulate breakpoints.  These methods return an
+    # error message is something went wrong, None if all is well.
+    # Set_break prints out the breakpoint line and file:lineno.
+    # Call self.get_*break*() to see the breakpoints or better
+    # for bp in Breakpoint.bpbynumber: if bp: bp.bpprint().
+
+    def set_break(self, filename, lineno, temporary=False, cond=None,
+                  funcname=None):
+        filename = self.canonic(filename)
+        import linecache # Import as late as possible
+        line = linecache.getline(filename, lineno)
+        if not line:
+            return 'Line %s:%d does not exist' % (filename, lineno)
+        list = self.breaks.setdefault(filename, [])
+        if lineno not in list:
+            list.append(lineno)
+        bp = Breakpoint(filename, lineno, temporary, cond, funcname)
+
+    def _prune_breaks(self, filename, lineno):
+        if (filename, lineno) not in Breakpoint.bplist:
+            self.breaks[filename].remove(lineno)
+        if not self.breaks[filename]:
+            del self.breaks[filename]
+
+    def clear_break(self, filename, lineno):
+        filename = self.canonic(filename)
+        if filename not in self.breaks:
+            return 'There are no breakpoints in %s' % filename
+        if lineno not in self.breaks[filename]:
+            return 'There is no breakpoint at %s:%d' % (filename, lineno)
+        # If there's only one bp in the list for that file,line
+        # pair, then remove the breaks entry
+        for bp in Breakpoint.bplist[filename, lineno][:]:
+            bp.deleteMe()
+        self._prune_breaks(filename, lineno)
+
+    def clear_bpbynumber(self, arg):
+        try:
+            bp = self.get_bpbynumber(arg)
+        except ValueError as err:
+            return str(err)
+        bp.deleteMe()
+        self._prune_breaks(bp.file, bp.line)
+
+    def clear_all_file_breaks(self, filename):
+        filename = self.canonic(filename)
+        if filename not in self.breaks:
+            return 'There are no breakpoints in %s' % filename
+        for line in self.breaks[filename]:
+            blist = Breakpoint.bplist[filename, line]
+            for bp in blist:
+                bp.deleteMe()
+        del self.breaks[filename]
+
+    def clear_all_breaks(self):
+        if not self.breaks:
+            return 'There are no breakpoints'
+        for bp in Breakpoint.bpbynumber:
+            if bp:
+                bp.deleteMe()
+        self.breaks = {}
+
+    def get_bpbynumber(self, arg):
+        if not arg:
+            raise ValueError('Breakpoint number expected')
+        try:
+            number = int(arg)
+        except ValueError:
+            raise ValueError('Non-numeric breakpoint number %s' % arg)
+        try:
+            bp = Breakpoint.bpbynumber[number]
+        except IndexError:
+            raise ValueError('Breakpoint number %d out of range' % number)
+        if bp is None:
+            raise ValueError('Breakpoint %d already deleted' % number)
+        return bp
+
+    def get_break(self, filename, lineno):
+        filename = self.canonic(filename)
+        return filename in self.breaks and \
+            lineno in self.breaks[filename]
+
+    def get_breaks(self, filename, lineno):
+        filename = self.canonic(filename)
+        return filename in self.breaks and \
+            lineno in self.breaks[filename] and \
+            Breakpoint.bplist[filename, lineno] or []

From noreply at buildbot.pypy.org  Sat Apr 14 11:34:54 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 11:34:54 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification/py3k: new branch to test
 stdlib unification for py3k
Message-ID: <20120414093454.A041582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification/py3k
Changeset: r54359:ae3b3a6c6107
Date: 2012-04-14 11:32 +0200
http://bitbucket.org/pypy/pypy/changeset/ae3b3a6c6107/

Log:	new branch to test stdlib unification for py3k

diff too long, truncating to 10000 out of 229939 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return pyc
+def pytest_unconfigure(config):
+    hook = config._assertstate.hook
+    if hook is not None:
+        sys.meta_path.remove(hook)
 
-def before_module_import(mod):
-    if mod.config._assertstate.mode != "on":
-        return
-    # Some deep magic: load the source, rewrite the asserts, and write a
-    # fake pyc, so that it'll be loaded when the module is imported.
-    source = mod.fspath.read()
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
-        return
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, str(mod.fspath), "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the assertion
-        # rewriting, but I don't know of a fast way to tell.
-        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
-        return
-    mod._pyc = _write_pyc(co, mod.fspath)
-    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+def pytest_collection(session):
+    # this hook is only called when test modules are collected
+    # so for example not in the master process of pytest-xdist
+    # (which does not collect test modules)
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.set_session(session)
 
-def after_module_import(mod):
-    if not hasattr(mod, "_pyc"):
-        return
-    state = mod.config._assertstate
-    try:
-        mod._pyc.remove()
-    except py.error.ENOENT:
-        state.trace("couldn't find pyc: %r" % (mod._pyc,))
-    else:
-        state.trace("removed pyc: %r" % (mod._pyc,))
+def pytest_runtest_setup(item):
+    def callbinrepr(op, left, right):
+        hook_result = item.ihook.pytest_assertrepr_compare(
+            config=item.config, op=op, left=left, right=right)
+        for new_expl in hook_result:
+            if new_expl:
+                res = '\n~'.join(new_expl)
+                if item.config.getvalue("assertmode") == "rewrite":
+                    # The result will be fed back a python % formatting
+                    # operation, which will fail if there are extraneous
+                    # '%'s in the string. Escape them here.
+                    res = res.replace("%", "%%")
+                return res
+    util._reprcompare = callbinrepr
 
-def warn_about_missing_assertion():
+def pytest_runtest_teardown(item):
+    util._reprcompare = None
+
+def pytest_sessionfinish(session):
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.session = None
+
+def _load_modules(mode):
+    """Lazily import assertion related code."""
+    global rewrite, reinterpret
+    from _pytest.assertion import reinterpret
+    if mode == "rewrite":
+        from _pytest.assertion import rewrite
+
+def warn_about_missing_assertion(mode):
     try:
         assert False
     except AssertionError:
         pass
     else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
+        if mode == "rewrite":
+            specifically = ("assertions which are not in test modules "
+                            "will be ignored")
+        else:
+            specifically = "failing tests may report as passing"
+
+        sys.stderr.write("WARNING: " + specifically +
+                        " because assert statements are not executed "
+                        "by the underlying Python interpreter "
+                        "(are you using python -O?)\n")
 
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -53,7 +53,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
+                "compute assert expression before the assert or use --assert=plain)")
 
 def run(offending_line, frame=None):
     if frame is None:
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
@@ -482,7 +481,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --nomagic)")
+                "compute assert expression before the assert or use --assert=plain)")
     else:
         return None
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,14 +1,258 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
+import errno
 import itertools
+import imp
+import marshal
+import os
+import struct
 import sys
+import types
 
 import py
 from _pytest.assertion import util
 
 
+# Windows gives ENOENT in places *nix gives ENOTDIR.
+if sys.platform.startswith("win"):
+    PATH_COMPONENT_NOT_DIR = errno.ENOENT
+else:
+    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
+
+# py.test caches rewritten pycs in __pycache__.
+if hasattr(imp, "get_tag"):
+    PYTEST_TAG = imp.get_tag() + "-PYTEST"
+else:
+    if hasattr(sys, "pypy_version_info"):
+        impl = "pypy"
+    elif sys.platform == "java":
+        impl = "jython"
+    else:
+        impl = "cpython"
+    ver = sys.version_info
+    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
+    del ver, impl
+
+PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
+
+REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+
+class AssertionRewritingHook(object):
+    """Import hook which rewrites asserts."""
+
+    def __init__(self):
+        self.session = None
+        self.modules = {}
+
+    def set_session(self, session):
+        self.fnpats = session.config.getini("python_files")
+        self.session = session
+
+    def find_module(self, name, path=None):
+        if self.session is None:
+            return None
+        sess = self.session
+        state = sess.config._assertstate
+        state.trace("find_module called for: %s" % name)
+        names = name.rsplit(".", 1)
+        lastname = names[-1]
+        pth = None
+        if path is not None and len(path) == 1:
+            pth = path[0]
+        if pth is None:
+            try:
+                fd, fn, desc = imp.find_module(lastname, path)
+            except ImportError:
+                return None
+            if fd is not None:
+                fd.close()
+            tp = desc[2]
+            if tp == imp.PY_COMPILED:
+                if hasattr(imp, "source_from_cache"):
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
+            elif tp != imp.PY_SOURCE:
+                # Don't know what this is.
+                return None
+        else:
+            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
+        fn_pypath = py.path.local(fn)
+        # Is this a test file?
+        if not sess.isinitpath(fn):
+            # We have to be very careful here because imports in this code can
+            # trigger a cycle.
+            self.session = None
+            try:
+                for pat in self.fnpats:
+                    if fn_pypath.fnmatch(pat):
+                        state.trace("matched test file %r" % (fn,))
+                        break
+                else:
+                    return None
+            finally:
+                self.session = sess
+        else:
+            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+        # The requested module looks like a test file, so rewrite it. This is
+        # the most magical part of the process: load the source, rewrite the
+        # asserts, and load the rewritten source. We also cache the rewritten
+        # module code in a special pyc. We must be aware of the possibility of
+        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # tricky race conditions, we maintain the following invariant: The
+        # cached pyc is always a complete, valid pyc. Operations on it must be
+        # atomic. POSIX's atomic rename comes in handy.
+        write = not sys.dont_write_bytecode
+        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
+        if write:
+            try:
+                os.mkdir(cache_dir)
+            except OSError:
+                e = sys.exc_info()[1].errno
+                if e == errno.EEXIST:
+                    # Either the __pycache__ directory already exists (the
+                    # common case) or it's blocked by a non-dir node. In the
+                    # latter case, we'll ignore it in _write_pyc.
+                    pass
+                elif e == PATH_COMPONENT_NOT_DIR:
+                    # One of the path components was not a directory, likely
+                    # because we're in a zip file.
+                    write = False
+                elif e == errno.EACCES:
+                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    write = False
+                else:
+                    raise
+        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
+        pyc = os.path.join(cache_dir, cache_name)
+        # Notice that even if we're in a read-only directory, I'm going to check
+        # for a cached pyc. This may not be optimal...
+        co = _read_pyc(fn_pypath, pyc)
+        if co is None:
+            state.trace("rewriting %r" % (fn,))
+            co = _rewrite_test(state, fn_pypath)
+            if co is None:
+                # Probably a SyntaxError in the test.
+                return None
+            if write:
+                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+        else:
+            state.trace("found cached rewritten pyc for %r" % (fn,))
+        self.modules[name] = co, pyc
+        return self
+
+    def load_module(self, name):
+        co, pyc = self.modules.pop(name)
+        # I wish I could just call imp.load_compiled here, but __file__ has to
+        # be set properly. In Python 3.2+, this all would be handled correctly
+        # by load_compiled.
+        mod = sys.modules[name] = imp.new_module(name)
+        try:
+            mod.__file__ = co.co_filename
+            # Normally, this attribute is 3.2+.
+            mod.__cached__ = pyc
+            py.builtin.exec_(co, mod.__dict__)
+        except:
+            del sys.modules[name]
+            raise
+        return sys.modules[name]
+
+def _write_pyc(co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as (C)Python, since
+    # these "pycs" should never be seen by builtin import. However, there's
+    # little reason deviate, and I hope sometime to be able to use
+    # imp.load_compiled to load them. (See the comment in load_module above.)
+    mtime = int(source_path.mtime())
+    try:
+        fp = open(pyc, "wb")
+    except IOError:
+        err = sys.exc_info()[1].errno
+        if err == PATH_COMPONENT_NOT_DIR:
+            # This happens when we get a EEXIST in find_module creating the
+            # __pycache__ directory and __pycache__ is by some non-dir node.
+            return False
+        raise
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return True
+
+RN = "\r\n".encode("utf-8")
+N = "\n".encode("utf-8")
+
+def _rewrite_test(state, fn):
+    """Try to read and rewrite *fn* and return the code object."""
+    try:
+        source = fn.read("rb")
+    except EnvironmentError:
+        return None
+    # On Python versions which are not 2.7 and less than or equal to 3.1, the
+    # parser expects *nix newlines.
+    if REWRITE_NEWLINES:
+        source = source.replace(RN, N) + N
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        state.trace("failed to parse: %r" % (fn,))
+        return None
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, fn.strpath, "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the
+        # assertion rewriting, but I don't know of a fast way to tell.
+        state.trace("failed to compile: %r" % (fn,))
+        return None
+    return co
+
+def _make_rewritten_pyc(state, fn, pyc, co):
+    """Try to dump rewritten code to *pyc*."""
+    if sys.platform.startswith("win"):
+        # Windows grants exclusive access to open files and doesn't have atomic
+        # rename, so just write into the final file.
+        _write_pyc(co, fn, pyc)
+    else:
+        # When not on windows, assume rename is atomic. Dump the code object
+        # into a file specific to this process and atomically replace it.
+        proc_pyc = pyc + "." + str(os.getpid())
+        if _write_pyc(co, fn, proc_pyc):
+            os.rename(proc_pyc, pyc)
+
+def _read_pyc(source, pyc):
+    """Possibly read a py.test pyc containing rewritten code.
+
+    Return rewritten code if successful or None if not.
+    """
+    try:
+        fp = open(pyc, "rb")
+    except IOError:
+        return None
+    try:
+        try:
+            mtime = int(source.mtime())
+            data = fp.read(8)
+        except EnvironmentError:
+            return None
+        # Check for invalid or out of date pyc file.
+        if (len(data) != 8 or
+            data[:4] != imp.get_magic() or
+            struct.unpack("<l", data[4:])[0] != mtime):
+            return None
+        co = marshal.load(fp)
+        if not isinstance(co, types.CodeType):
+            # That's interesting....
+            return None
+        return co
+    finally:
+        fp.close()
+
+
 def rewrite_asserts(mod):
     """Rewrite the assert statements in mod."""
     AssertionRewriter().run(mod)
@@ -17,13 +261,8 @@
 _saferepr = py.io.saferepr
 from _pytest.assertion.util import format_explanation as _format_explanation
 
-def _format_boolop(operands, explanations, is_or):
-    show_explanations = []
-    for operand, expl in zip(operands, explanations):
-        show_explanations.append(expl)
-        if operand == is_or:
-            break
-    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+def _format_boolop(explanations, is_or):
+    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -109,8 +348,8 @@
                     return
                 lineno += len(doc) - 1
                 expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +357,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +382,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +437,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +460,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +480,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +545,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,21 +25,39 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +86,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("</testcase>")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="xfail-marked test passes unexpectedly"/>')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('<failure message="test failure">%s</failure>',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<failure message="collection failure">%s</failure>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<skipped message="collection skipped">%s</skipped>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('<error message="test setup failure">%s</error>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="expected test failure">%s</skipped>',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('<skipped type="pytest.skip" '
-                           'message="%s">%s</skipped>',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +185,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n<testcase classname="pytest" name="internal">'
-            '    <error message="internal error">'
-            '%s</error></testcase>' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +203,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write('<testsuite ')
-        logfile.write('name="" ')
-        logfile.write('errors="%i" ' % self.errors)
-        logfile.write('failures="%i" ' % self.failed)
-        logfile.write('skips="%i" ' % self.skipped)
-        logfile.write('tests="%i" ' % numtests)
-        logfile.write('time="%.3f"' % suite_time_delta)
-        logfile.write(' >')
-        logfile.writelines(self.test_logs)
-        logfile.write('</testsuite>')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -399,6 +410,7 @@
         self._notfound = []
         self._initialpaths = set()
         self._initialparts = []
+        self.items = items = []
         for arg in args:
             parts = self._parsearg(arg)
             self._initialparts.append(parts)
@@ -414,7 +426,6 @@
         if not genitems:
             return rep.result
         else:
-            self.items = items = []
             if rep.passed:
                 for node in rep.result:
                     self.items.extend(self.genitems(node))
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "<CallSpec id=%r param=%r funcargs=%r>" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,71 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        if not indirect:
+            #XXX should we also check for the opposite case?
+            for arg in argnames:
+                if arg not in self.funcargnames:
+                    raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +678,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +769,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +781,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +830,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +956,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         <ExceptionInfo ...>
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         <ExceptionInfo ...>
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,60 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
-
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
+def pytest_sessionfinish(session):
+    session._setupstate.teardown_all()
 
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +76,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +93,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +114,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +143,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +166,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +188,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +208,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "<TestReport %r when=%r outcome=%r>" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +234,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +259,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +319,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -117,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
@@ -169,21 +192,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -289,10 +282,18 @@
         # we take care to leave out Instances aka ()
         # because later versions are going to get rid of them anyway
         if self.config.option.verbose < 0:
-            for item in items:
-                nodeid = item.nodeid
-                nodeid = nodeid.replace("::()::", "::")
-                self._tw.line(nodeid)
+            if self.config.option.verbose < -1:
+                counts = {}
+                for item in items:
+                    name = item.nodeid.split('::', 1)[0]
+                    counts[name] = counts.get(name, 0) + 1
+                for name, count in sorted(counts.items()):
+                    self._tw.line("%s: %d" % (name, count))
+            else:
+                for item in items:
+                    nodeid = item.nodeid
+                    nodeid = nodeid.replace("::()::", "::")
+                    self._tw.line(nodeid)
             return
         stack = []
         indent = ""
@@ -318,12 +319,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +394,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +412,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +431,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +445,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -83,28 +91,37 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +137,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib-python/2.7/UserDict.py b/lib-python/2.7/UserDict.py
--- a/lib-python/2.7/UserDict.py
+++ b/lib-python/2.7/UserDict.py
@@ -80,8 +80,12 @@
     def __iter__(self):
         return iter(self.data)
 
-import _abcoll
-_abcoll.MutableMapping.register(IterableUserDict)
+try:
+    import _abcoll
+except ImportError:
+    pass    # e.g. no '_weakref' module on this pypy
+else:
+    _abcoll.MutableMapping.register(IterableUserDict)
 
 
 class DictMixin:
diff --git a/lib-python/2.7/_threading_local.py b/lib-python/2.7/_threading_local.py
--- a/lib-python/2.7/_threading_local.py
+++ b/lib-python/2.7/_threading_local.py
@@ -155,7 +155,7 @@
         object.__setattr__(self, '_local__args', (args, kw))
         object.__setattr__(self, '_local__lock', RLock())
 
-        if (args or kw) and (cls.__init__ is object.__init__):
+        if (args or kw) and (cls.__init__ == object.__init__):
             raise TypeError("Initialization arguments are not supported")
 
         # We need to create the thread dict in anticipation of
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -7,6 +7,7 @@
 
 __version__ = "1.1.0"
 
+import _ffi
 from _ctypes import Union, Structure, Array
 from _ctypes import _Pointer
 from _ctypes import CFuncPtr as _CFuncPtr
@@ -350,16 +351,17 @@
         self._FuncPtr = _FuncPtr
 
         if handle is None:
-            self._handle = _dlopen(self._name, mode)
+            self._handle = _ffi.CDLL(name, mode)
         else:
             self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %x at %x>" % \
+        return "<%s '%s', handle %r at %x>" % \
                (self.__class__.__name__, self._name,
-                (self._handle & (_sys.maxint*2 + 1)),
+                (self._handle),
                 id(self) & (_sys.maxint*2 + 1))
 
+
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
             raise AttributeError(name)
@@ -487,9 +489,12 @@
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
-    return _cast(obj, obj, typ)
+    try:
+        c_void_p.from_param(obj)
+    except TypeError, e:
+        raise ArgumentError(str(e))
+    return _cast_addr(obj, obj, typ)
 
 _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
diff --git a/lib-python/2.7/ctypes/test/__init__.py b/lib-python/2.7/ctypes/test/__init__.py
--- a/lib-python/2.7/ctypes/test/__init__.py
+++ b/lib-python/2.7/ctypes/test/__init__.py
@@ -206,3 +206,16 @@
         result = unittest.TestResult()
         test(result)
         return result
+
+def xfail(method):
+    """
+    Poor's man xfail: remove it when all the failures have been fixed
+    """
+    def new_method(self, *args, **kwds):
+        try:
+            method(self, *args, **kwds)
+        except:
+            pass
+        else:
+            self.assertTrue(False, "DID NOT RAISE")
+    return new_method
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -1,12 +1,23 @@
 import unittest
 from ctypes import *
+from test.test_support import impl_detail
 
 formats = "bBhHiIlLqQfd"
 
+# c_longdouble commented out for PyPy, look at the commend in test_longdouble
 formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \
-          c_long, c_ulonglong, c_float, c_double, c_longdouble
+          c_long, c_ulonglong, c_float, c_double #, c_longdouble
 
 class ArrayTestCase(unittest.TestCase):
+
+    @impl_detail('long double not supported by PyPy', pypy=False)
+    def test_longdouble(self):
+        """
+        This test is empty. It's just here to remind that we commented out
+        c_longdouble in "formats". If pypy will ever supports c_longdouble, we
+        should kill this test and uncomment c_longdouble inside formats.
+        """
+
     def test_simple(self):
         # create classes holding simple numeric types, and check
         # various properties.
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -115,17 +115,21 @@
     def test_nonint_types(self):
         # bit fields are not allowed on non-integer types.
         result = self.fail_fields(("a", c_char_p, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         result = self.fail_fields(("a", c_void_p, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         if c_int != c_long:
             result = self.fail_fields(("a", POINTER(c_int), 1))
-            self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int'))
+            self.assertEqual(result[0], TypeError)
+            self.assertIn('bit fields not allowed for type', result[1])
 
         result = self.fail_fields(("a", c_char, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         try:
             c_wchar
@@ -133,13 +137,15 @@
             pass
         else:
             result = self.fail_fields(("a", c_wchar, 1))
-            self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_wchar'))
+            self.assertEqual(result[0], TypeError)
+            self.assertIn('bit fields not allowed for type', result[1])
 
         class Dummy(Structure):
             _fields_ = []
 
         result = self.fail_fields(("a", Dummy, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
     def test_single_bitfield_size(self):
         for c_typ in int_types:
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -2,6 +2,7 @@
 from binascii import hexlify
 
 from ctypes import *
+from ctypes.test import xfail
 
 def bin(s):
     return hexlify(memoryview(s)).upper()
@@ -21,6 +22,7 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
+    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertTrue(c_short.__ctype_le__ is c_short)
@@ -48,6 +50,7 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
+    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertTrue(c_int.__ctype_le__ is c_int)
@@ -76,6 +79,7 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
+    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertTrue(c_longlong.__ctype_le__ is c_longlong)
@@ -104,6 +108,7 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
+    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertTrue(c_float.__ctype_le__ is c_float)
@@ -122,6 +127,7 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
+    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertTrue(c_double.__ctype_le__ is c_double)
@@ -149,6 +155,7 @@
         self.assertTrue(c_char.__ctype_le__ is c_char)
         self.assertTrue(c_char.__ctype_be__ is c_char)
 
+    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -198,6 +205,7 @@
             pass
         self.assertRaises(TypeError, setattr, S, "_fields_", [("s", T)])
 
+    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -221,6 +229,7 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
+    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import xfail
 import _ctypes_test
 
 class Callbacks(unittest.TestCase):
@@ -98,6 +99,7 @@
 ##        self.check_type(c_char_p, "abc")
 ##        self.check_type(c_char_p, "def")
 
+    @xfail
     def test_pyobject(self):
         o = ()
         from sys import getrefcount as grc
diff --git a/lib-python/2.7/ctypes/test/test_cfuncs.py b/lib-python/2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/2.7/ctypes/test/test_cfuncs.py
@@ -3,8 +3,8 @@
 
 import unittest
 from ctypes import *
-
 import _ctypes_test
+from test.test_support import impl_detail
 
 class CFunctions(unittest.TestCase):
     _dll = CDLL(_ctypes_test.__file__)
@@ -158,12 +158,14 @@
         self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
         self.assertEqual(self.S(), 42)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble(self):
         self._dll.tf_D.restype = c_longdouble
         self._dll.tf_D.argtypes = (c_longdouble,)
         self.assertEqual(self._dll.tf_D(42.), 14.)
         self.assertEqual(self.S(), 42)
-
+        
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble_plus(self):
         self._dll.tf_bD.restype = c_longdouble
         self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
diff --git a/lib-python/2.7/ctypes/test/test_delattr.py b/lib-python/2.7/ctypes/test/test_delattr.py
--- a/lib-python/2.7/ctypes/test/test_delattr.py
+++ b/lib-python/2.7/ctypes/test/test_delattr.py
@@ -6,15 +6,15 @@
 
 class TestCase(unittest.TestCase):
     def test_simple(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, c_int(42), "value")
 
     def test_chararray(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, (c_char * 5)(), "value")
 
     def test_struct(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, X(), "foo")
 
 if __name__ == "__main__":
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,6 +2,7 @@
 import array
 import gc
 import unittest
+from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -10,6 +11,7 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
+    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -35,6 +37,7 @@
         self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
+    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -43,6 +46,7 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
+    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -67,6 +71,7 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
+    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py
--- a/lib-python/2.7/ctypes/test/test_functions.py
+++ b/lib-python/2.7/ctypes/test/test_functions.py
@@ -7,6 +7,8 @@
 
 from ctypes import *
 import sys, unittest
+from ctypes.test import xfail
+from test.test_support import impl_detail
 
 try:
     WINFUNCTYPE
@@ -143,6 +145,7 @@
         self.assertEqual(result, -21)
         self.assertEqual(type(result), float)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdoubleresult(self):
         f = dll._testfunc_D_bhilfD
         f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
@@ -393,6 +396,7 @@
             self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
                                  (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
 
+    @xfail
     def test_sf1651235(self):
         # see http://www.python.org/sf/1651235
 
diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py
--- a/lib-python/2.7/ctypes/test/test_internals.py
+++ b/lib-python/2.7/ctypes/test/test_internals.py
@@ -33,7 +33,13 @@
         refcnt = grc(s)
         cs = c_char_p(s)
         self.assertEqual(refcnt + 1, grc(s))
-        self.assertSame(cs._objects, s)
+        try:
+            # Moving gcs need to allocate a nonmoving buffer
+            cs._objects._obj
+        except AttributeError:
+            self.assertSame(cs._objects, s)
+        else:
+            self.assertSame(cs._objects._obj, s)
 
     def test_simple_struct(self):
         class X(Structure):
diff --git a/lib-python/2.7/ctypes/test/test_libc.py b/lib-python/2.7/ctypes/test/test_libc.py
--- a/lib-python/2.7/ctypes/test/test_libc.py
+++ b/lib-python/2.7/ctypes/test/test_libc.py
@@ -25,5 +25,14 @@
         lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort))
         self.assertEqual(chars.raw, "   ,,aaaadmmmnpppsss\x00")
 
+    def SKIPPED_test_no_more_xfail(self):
+        # We decided to not explicitly support the whole ctypes-2.7
+        # and instead go for a case-by-case, demand-driven approach.
+        # So this test is skipped instead of failing.
+        import socket
+        import ctypes.test
+        self.assertTrue(not hasattr(ctypes.test, 'xfail'),
+                        "You should incrementally grep for '@xfail' and remove them, they are real failures")
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_loading.py b/lib-python/2.7/ctypes/test/test_loading.py
--- a/lib-python/2.7/ctypes/test/test_loading.py
+++ b/lib-python/2.7/ctypes/test/test_loading.py
@@ -2,7 +2,7 @@
 import sys, unittest
 import os
 from ctypes.util import find_library
-from ctypes.test import is_resource_enabled
+from ctypes.test import is_resource_enabled, xfail
 
 libc_name = None
 if os.name == "nt":
@@ -75,6 +75,7 @@
             self.assertRaises(AttributeError, dll.__getitem__, 1234)
 
     if os.name == "nt":
+        @xfail
         def test_1703286_A(self):
             from _ctypes import LoadLibrary, FreeLibrary
             # On winXP 64-bit, advapi32 loads at an address that does
@@ -85,6 +86,7 @@
             handle = LoadLibrary("advapi32")
             FreeLibrary(handle)
 
+        @xfail
         def test_1703286_B(self):
             # Since on winXP 64-bit advapi32 loads like described
             # above, the (arbitrarily selected) CloseEventLog function
diff --git a/lib-python/2.7/ctypes/test/test_macholib.py b/lib-python/2.7/ctypes/test/test_macholib.py
--- a/lib-python/2.7/ctypes/test/test_macholib.py
+++ b/lib-python/2.7/ctypes/test/test_macholib.py
@@ -52,7 +52,6 @@
                                  '/usr/lib/libSystem.B.dylib')
 
             result = find_lib('z')
-            self.assertTrue(result.startswith('/usr/lib/libz.1'))
             self.assertTrue(result.endswith('.dylib'))
 
             self.assertEqual(find_lib('IOKit'),
diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py
--- a/lib-python/2.7/ctypes/test/test_numbers.py
+++ b/lib-python/2.7/ctypes/test/test_numbers.py
@@ -1,6 +1,7 @@
 from ctypes import *
 import unittest
 import struct
+from ctypes.test import xfail
 
 def valid_ranges(*types):
     # given a sequence of numeric types, collect their _type_
@@ -89,12 +90,14 @@
 ##            self.assertRaises(ValueError, t, l-1)
 ##            self.assertRaises(ValueError, t, h+1)
 
+    @xfail
     def test_from_param(self):
         # the from_param class method attribute always
         # returns PyCArgObject instances
         for t in signed_types + unsigned_types + float_types:
             self.assertEqual(ArgType, type(t.from_param(0)))
 
+    @xfail
     def test_byref(self):
         # calling byref returns also a PyCArgObject instance
         for t in signed_types + unsigned_types + float_types + bool_types:
@@ -102,6 +105,7 @@
             self.assertEqual(ArgType, type(parm))
 
 
+    @xfail
     def test_floats(self):
         # c_float and c_double can be created from
         # Python int, long and float
@@ -115,6 +119,7 @@
             self.assertEqual(t(2L).value, 2.0)
             self.assertEqual(t(f).value, 2.0)
 
+    @xfail
     def test_integers(self):
         class FloatLike(object):
             def __float__(self):
diff --git a/lib-python/2.7/ctypes/test/test_objects.py b/lib-python/2.7/ctypes/test/test_objects.py
--- a/lib-python/2.7/ctypes/test/test_objects.py
+++ b/lib-python/2.7/ctypes/test/test_objects.py
@@ -22,7 +22,7 @@
 
 >>> array[4] = 'foo bar'
 >>> array._objects
-{'4': 'foo bar'}
+{'4': <CArgObject 'foo bar'>}
 >>> array[4]
 'foo bar'
 >>>
@@ -47,9 +47,9 @@
 
 >>> x.array[0] = 'spam spam spam'
 >>> x._objects
-{'0:2': 'spam spam spam'}
+{'0:2': <CArgObject 'spam spam spam'>}
 >>> x.array._b_base_._objects
-{'0:2': 'spam spam spam'}
+{'0:2': <CArgObject 'spam spam spam'>}
 >>>
 
 '''
diff --git a/lib-python/2.7/ctypes/test/test_parameters.py b/lib-python/2.7/ctypes/test/test_parameters.py
--- a/lib-python/2.7/ctypes/test/test_parameters.py
+++ b/lib-python/2.7/ctypes/test/test_parameters.py
@@ -1,5 +1,7 @@
 import unittest, sys
 
+from ctypes.test import xfail
+
 class SimpleTypesTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -49,6 +51,7 @@
         self.assertEqual(CWCHARP.from_param("abc"), "abcabcabc")
 
     # XXX Replace by c_char_p tests
+    @xfail
     def test_cstrings(self):
         from ctypes import c_char_p, byref
 
@@ -86,7 +89,10 @@
 
         pa = c_wchar_p.from_param(c_wchar_p(u"123"))
         self.assertEqual(type(pa), c_wchar_p)
+    if sys.platform == "win32":
+        test_cw_strings = xfail(test_cw_strings)
 
+    @xfail
     def test_int_pointers(self):
         from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer
         LPINT = POINTER(c_int)
diff --git a/lib-python/2.7/ctypes/test/test_pep3118.py b/lib-python/2.7/ctypes/test/test_pep3118.py
--- a/lib-python/2.7/ctypes/test/test_pep3118.py
+++ b/lib-python/2.7/ctypes/test/test_pep3118.py
@@ -1,6 +1,7 @@
 import unittest
 from ctypes import *
 import re, sys
+from ctypes.test import xfail
 
 if sys.byteorder == "little":
     THIS_ENDIAN = "<"
@@ -19,6 +20,7 @@
 
 class Test(unittest.TestCase):
 
+    @xfail
     def test_native_types(self):
         for tp, fmt, shape, itemtp in native_types:
             ob = tp()
@@ -46,6 +48,7 @@
                 print(tp)
                 raise
 
+    @xfail
     def test_endian_types(self):
         for tp, fmt, shape, itemtp in endian_types:
             ob = tp()
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -3,6 +3,7 @@
 from ctypes import *
 import _ctypes_test
 dll = CDLL(_ctypes_test.__file__)
+from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("a", c_int), ("b", c_double)]
@@ -21,6 +22,7 @@
     def loads(self, item):
         return pickle.loads(item)
 
+    @xfail
     def test_simple(self):
         for src in [
             c_int(42),
@@ -31,6 +33,7 @@
             self.assertEqual(memoryview(src).tobytes(),
                                  memoryview(dst).tobytes())
 
+    @xfail
     def test_struct(self):
         X.init_called = 0
 
@@ -49,6 +52,7 @@
         self.assertEqual(memoryview(y).tobytes(),
                              memoryview(x).tobytes())
 
+    @xfail
     def test_unpickable(self):
         # ctypes objects that are pointers or contain pointers are
         # unpickable.
@@ -66,6 +70,7 @@
             ]:
             self.assertRaises(ValueError, lambda: self.dumps(item))
 
+    @xfail
     def test_wchar(self):
         pickle.dumps(c_char("x"))
         # Issue 5049
diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py
--- a/lib-python/2.7/ctypes/test/test_python_api.py
+++ b/lib-python/2.7/ctypes/test/test_python_api.py
@@ -1,6 +1,6 @@
 from ctypes import *
 import unittest, sys
-from ctypes.test import is_resource_enabled
+from ctypes.test import is_resource_enabled, xfail
 
 ################################################################
 # This section should be moved into ctypes\__init__.py, when it's ready.
@@ -17,6 +17,7 @@
 
 class PythonAPITestCase(unittest.TestCase):
 
+    @xfail
     def test_PyString_FromStringAndSize(self):
         PyString_FromStringAndSize = pythonapi.PyString_FromStringAndSize
 
@@ -25,6 +26,7 @@
 
         self.assertEqual(PyString_FromStringAndSize("abcdefghi", 3), "abc")
 
+    @xfail
     def test_PyString_FromString(self):
         pythonapi.PyString_FromString.restype = py_object
         pythonapi.PyString_FromString.argtypes = (c_char_p,)
@@ -56,6 +58,7 @@
             del res
             self.assertEqual(grc(42), ref42)
 
+    @xfail
     def test_PyObj_FromPtr(self):
         s = "abc def ghi jkl"
         ref = grc(s)
@@ -81,6 +84,7 @@
         # not enough arguments
         self.assertRaises(TypeError, PyOS_snprintf, buf)
 
+    @xfail
     def test_pyobject_repr(self):
         self.assertEqual(repr(py_object()), "py_object(<NULL>)")
         self.assertEqual(repr(py_object(42)), "py_object(42)")
diff --git a/lib-python/2.7/ctypes/test/test_refcounts.py b/lib-python/2.7/ctypes/test/test_refcounts.py
--- a/lib-python/2.7/ctypes/test/test_refcounts.py
+++ b/lib-python/2.7/ctypes/test/test_refcounts.py
@@ -90,6 +90,7 @@
             return a * b * 2
         f = proto(func)
 
+        gc.collect()
         a = sys.getrefcount(ctypes.c_int)
         f(1, 2)
         self.assertEqual(sys.getrefcount(ctypes.c_int), a)
diff --git a/lib-python/2.7/ctypes/test/test_stringptr.py b/lib-python/2.7/ctypes/test/test_stringptr.py
--- a/lib-python/2.7/ctypes/test/test_stringptr.py
+++ b/lib-python/2.7/ctypes/test/test_stringptr.py
@@ -2,11 +2,13 @@
 from ctypes import *
 
 import _ctypes_test
+from ctypes.test import xfail
 
 lib = CDLL(_ctypes_test.__file__)
 
 class StringPtrTestCase(unittest.TestCase):
 
+    @xfail
     def test__POINTER_c_char(self):
         class X(Structure):
             _fields_ = [("str", POINTER(c_char))]
@@ -27,6 +29,7 @@
 
         self.assertRaises(TypeError, setattr, x, "str", "Hello, World")
 
+    @xfail
     def test__c_char_p(self):
         class X(Structure):
             _fields_ = [("str", c_char_p)]
diff --git a/lib-python/2.7/ctypes/test/test_strings.py b/lib-python/2.7/ctypes/test/test_strings.py
--- a/lib-python/2.7/ctypes/test/test_strings.py
+++ b/lib-python/2.7/ctypes/test/test_strings.py
@@ -31,8 +31,9 @@
         buf.value = "Hello, World"
         self.assertEqual(buf.value, "Hello, World")
 
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("Hello, World"))
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
+        if test_support.check_impl_detail():
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("Hello, World"))
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
         self.assertRaises(ValueError, setattr, buf, "raw", memoryview("x" * 100))
 
     def test_c_buffer_raw(self, memoryview=memoryview):
@@ -40,7 +41,8 @@
 
         buf.raw = memoryview("Hello, World")
         self.assertEqual(buf.value, "Hello, World")
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
+        if test_support.check_impl_detail():
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
         self.assertRaises(ValueError, setattr, buf, "raw", memoryview("x" * 100))
 
     def test_c_buffer_deprecated(self):
diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py
--- a/lib-python/2.7/ctypes/test/test_structures.py
+++ b/lib-python/2.7/ctypes/test/test_structures.py
@@ -194,8 +194,8 @@
         self.assertEqual(X.b.offset, min(8, longlong_align))
 
 
-        d = {"_fields_": [("a", "b"),
-                          ("b", "q")],
+        d = {"_fields_": [("a", c_byte),
+                          ("b", c_longlong)],
              "_pack_": -1}
         self.assertRaises(ValueError, type(Structure), "X", (Structure,), d)
 
diff --git a/lib-python/2.7/ctypes/test/test_varsize_struct.py b/lib-python/2.7/ctypes/test/test_varsize_struct.py
--- a/lib-python/2.7/ctypes/test/test_varsize_struct.py
+++ b/lib-python/2.7/ctypes/test/test_varsize_struct.py
@@ -1,7 +1,9 @@
 from ctypes import *
 import unittest
+from ctypes.test import xfail
 
 class VarSizeTest(unittest.TestCase):
+    @xfail
     def test_resize(self):
         class X(Structure):
             _fields_ = [("item", c_int),
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -72,8 +72,8 @@
         return name
 
 if os.name == "posix" and sys.platform == "darwin":
-    from ctypes.macholib.dyld import dyld_find as _dyld_find
     def find_library(name):
+        from ctypes.macholib.dyld import dyld_find as _dyld_find
         possible = ['lib%s.dylib' % name,
                     '%s.dylib' % name,
                     '%s.framework/%s' % (name, name)]
diff --git a/lib-python/2.7/distutils/command/bdist_wininst.py b/lib-python/2.7/distutils/command/bdist_wininst.py
--- a/lib-python/2.7/distutils/command/bdist_wininst.py
+++ b/lib-python/2.7/distutils/command/bdist_wininst.py
@@ -298,7 +298,8 @@
                              bitmaplen,        # number of bytes in bitmap
                              )
         file.write(header)
-        file.write(open(arcname, "rb").read())
+        with open(arcname, "rb") as arcfile:
+            file.write(arcfile.read())
 
     # create_exe()
 
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -184,7 +184,7 @@
             # the 'libs' directory is for binary installs - we assume that
             # must be the *native* platform.  But we don't really support
             # cross-compiling via a binary install anyway, so we let it go.
-            self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs'))
+            self.library_dirs.append(os.path.join(sys.exec_prefix, 'include'))
             if self.debug:
                 self.build_temp = os.path.join(self.build_temp, "Debug")
             else:
@@ -192,8 +192,13 @@
 
             # Append the source distribution include and library directories,
             # this allows distutils on windows to work in the source tree
-            self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
-            if MSVC_VERSION == 9:
+            if 0:
+                # pypy has no PC directory
+                self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
+            if 1:
+                # pypy has no PCBuild directory
+                pass
+            elif MSVC_VERSION == 9:
                 # Use the .lib files for the correct architecture
                 if self.plat_name == 'win32':
                     suffix = ''
@@ -695,24 +700,14 @@
         shared extension.  On most platforms, this is just 'ext.libraries';
         on Windows and OS/2, we add the Python library (eg. python20.dll).
         """
-        # The python library is always needed on Windows.  For MSVC, this
-        # is redundant, since the library is mentioned in a pragma in
-        # pyconfig.h that MSVC groks.  The other Windows compilers all seem
-        # to need it mentioned explicitly, though, so that's what we do.
-        # Append '_d' to the python import library on debug builds.
+        # The python library is always needed on Windows.
         if sys.platform == "win32":
-            from distutils.msvccompiler import MSVCCompiler
-            if not isinstance(self.compiler, MSVCCompiler):
-                template = "python%d%d"
-                if self.debug:
-                    template = template + '_d'
-                pythonlib = (template %
-                       (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
-                # don't extend ext.libraries, it may be shared with other
-                # extensions, it is a reference to the original list
-                return ext.libraries + [pythonlib]
-            else:
-                return ext.libraries
+            template = "python%d%d"
+            pythonlib = (template %
+                   (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
+            # don't extend ext.libraries, it may be shared with other
+            # extensions, it is a reference to the original list
+            return ext.libraries + [pythonlib]
         elif sys.platform == "os2emx":
             # EMX/GCC requires the python library explicitly, and I
             # believe VACPP does as well (though not confirmed) - AIM Apr01
diff --git a/lib-python/2.7/distutils/command/install.py b/lib-python/2.7/distutils/command/install.py
--- a/lib-python/2.7/distutils/command/install.py
+++ b/lib-python/2.7/distutils/command/install.py
@@ -83,6 +83,13 @@
         'scripts': '$userbase/bin',
         'data'   : '$userbase',
         },
+    'pypy': {
+        'purelib': '$base/site-packages',
+        'platlib': '$base/site-packages',
+        'headers': '$base/include',
+        'scripts': '$base/bin',
+        'data'   : '$base',
+        },
     }
 
 # The keys to an installation scheme; if any new types of files are to be
@@ -467,6 +474,8 @@
 
     def select_scheme (self, name):
         # it's the caller's problem if they supply a bad name!
+        if hasattr(sys, 'pypy_version_info'):
+            name = 'pypy'
         scheme = INSTALL_SCHEMES[name]
         for key in SCHEME_KEYS:
             attrname = 'install_' + key
diff --git a/lib-python/2.7/distutils/cygwinccompiler.py b/lib-python/2.7/distutils/cygwinccompiler.py
--- a/lib-python/2.7/distutils/cygwinccompiler.py
+++ b/lib-python/2.7/distutils/cygwinccompiler.py
@@ -75,6 +75,9 @@
         elif msc_ver == '1500':
             # VS2008 / MSVC 9.0
             return ['msvcr90']
+        elif msc_ver == '1600':
+            # VS2010 / MSVC 10.0
+            return ['msvcr100']
         else:
             raise ValueError("Unknown MS Compiler version %s " % msc_ver)
 
diff --git a/lib-python/2.7/distutils/msvc9compiler.py b/lib-python/2.7/distutils/msvc9compiler.py
--- a/lib-python/2.7/distutils/msvc9compiler.py
+++ b/lib-python/2.7/distutils/msvc9compiler.py
@@ -648,6 +648,7 @@
             temp_manifest = os.path.join(
                     build_temp,
                     os.path.basename(output_filename) + ".manifest")
+            ld_args.append('/MANIFEST')
             ld_args.append('/MANIFESTFILE:' + temp_manifest)
 
             if extra_preargs:
diff --git a/lib-python/2.7/distutils/spawn.py b/lib-python/2.7/distutils/spawn.py
--- a/lib-python/2.7/distutils/spawn.py
+++ b/lib-python/2.7/distutils/spawn.py
@@ -58,7 +58,6 @@
 
 def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0):
     executable = cmd[0]
-    cmd = _nt_quote_args(cmd)
     if search_path:
         # either we find one or it stays the same
         executable = find_executable(executable) or executable
@@ -66,7 +65,8 @@
     if not dry_run:
         # spawn for NT requires a full path to the .exe
         try:
-            rc = os.spawnv(os.P_WAIT, executable, cmd)
+            import subprocess
+            rc = subprocess.call(cmd)
         except OSError, exc:
             # this seems to happen when the command isn't found
             raise DistutilsExecError, \
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -9,563 +9,21 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
-import os
-import re
-import string
 import sys
 
-from distutils.errors import DistutilsPlatformError
 
-# These are needed in a couple of spots, so just compute them once.
-PREFIX = os.path.normpath(sys.prefix)
-EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+# The content of this file is redirected from
+# sysconfig_cpython or sysconfig_pypy.
 
-# Path to the base directory of the project. On Windows the binary may
-# live in project/PCBuild9.  If we're dealing with an x64 Windows build,
-# it'll live in project/PCbuild/amd64.
-project_base = os.path.dirname(os.path.abspath(sys.executable))
-if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
-# PC/VS7.1
-if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
-# PC/AMD64
-if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
+if '__pypy__' in sys.builtin_module_names:
+    from distutils.sysconfig_pypy import *
+    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
+    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+else:
+    from distutils.sysconfig_cpython import *
+    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
+    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
 
-# python_build: (Boolean) if true, we're either building Python or
-# building an extension with an un-installed Python, so we use
-# different (hard-wired) directories.
-# Setup.local is available for Makefile builds including VPATH builds,
-# Setup.dist is available on Windows
-def _python_build():
-    for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
-            return True
-    return False
-python_build = _python_build()
 
-
-def get_python_version():
-    """Return a string containing the major and minor Python version,
-    leaving off the patchlevel.  Sample return values could be '1.5'
-    or '2.2'.
-    """
-    return sys.version[:3]
-
-
-def get_python_inc(plat_specific=0, prefix=None):
-    """Return the directory containing installed Python header files.
-
-    If 'plat_specific' is false (the default), this is the path to the
-    non-platform-specific header files, i.e. Python.h and so on;
-    otherwise, this is the path to platform-specific header files
-    (namely pyconfig.h).
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        if python_build:
-            buildir = os.path.dirname(sys.executable)
-            if plat_specific:
-                # python.h is located in the buildir
-                inc_dir = buildir
-            else:
-                # the source dir is relative to the buildir
-                srcdir = os.path.abspath(os.path.join(buildir,
-                                         get_config_var('srcdir')))
-                # Include is located in the srcdir
-                inc_dir = os.path.join(srcdir, "Include")
-            return inc_dir
-        return os.path.join(prefix, "include", "python" + get_python_version())
-    elif os.name == "nt":
-        return os.path.join(prefix, "include")
-    elif os.name == "os2":
-        return os.path.join(prefix, "Include")
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its C header files "
-            "on platform '%s'" % os.name)
-
-
-def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
-    """Return the directory containing the Python library (standard or
-    site additions).
-
-    If 'plat_specific' is true, return the directory containing
-    platform-specific modules, i.e. any module from a non-pure-Python
-    module distribution; otherwise, return the platform-shared library
-    directory.  If 'standard_lib' is true, return the directory
-    containing standard Python library modules; otherwise, return the
-    directory for site-specific modules.
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
-        if standard_lib:
-            return libpython
-        else:
-            return os.path.join(libpython, "site-packages")
-
-    elif os.name == "nt":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            if get_python_version() < "2.2":
-                return prefix
-            else:
-                return os.path.join(prefix, "Lib", "site-packages")
-
-    elif os.name == "os2":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            return os.path.join(prefix, "Lib", "site-packages")
-
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its library "
-            "on platform '%s'" % os.name)
-
-
-def customize_compiler(compiler):
-    """Do any platform-specific customization of a CCompiler instance.
-
-    Mainly needed on Unix, so we can plug in the information that
-    varies across Unices and is stored in Python's Makefile.
-    """
-    if compiler.compiler_type == "unix":
-        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext) = \
-            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SO')
-
-        if 'CC' in os.environ:
-            cc = os.environ['CC']
-        if 'CXX' in os.environ:
-            cxx = os.environ['CXX']
-        if 'LDSHARED' in os.environ:
-            ldshared = os.environ['LDSHARED']
-        if 'CPP' in os.environ:
-            cpp = os.environ['CPP']
-        else:
-            cpp = cc + " -E"           # not always
-        if 'LDFLAGS' in os.environ:
-            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        if 'CPPFLAGS' in os.environ:
-            cpp = cpp + ' ' + os.environ['CPPFLAGS']
-            cflags = cflags + ' ' + os.environ['CPPFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-
-        cc_cmd = cc + ' ' + cflags
-        compiler.set_executables(
-            preprocessor=cpp,
-            compiler=cc_cmd,
-            compiler_so=cc_cmd + ' ' + ccshared,
-            compiler_cxx=cxx,
-            linker_so=ldshared,
-            linker_exe=cc)
-
-        compiler.shared_lib_extension = so_ext
-
-
-def get_config_h_filename():
-    """Return full pathname of installed pyconfig.h file."""
-    if python_build:
-        if os.name == "nt":
-            inc_dir = os.path.join(project_base, "PC")
-        else:
-            inc_dir = project_base
-    else:
-        inc_dir = get_python_inc(plat_specific=1)
-    if get_python_version() < '2.2':
-        config_h = 'config.h'
-    else:
-        # The name of the config.h file changed in 2.2
-        config_h = 'pyconfig.h'
-    return os.path.join(inc_dir, config_h)
-
-
-def get_makefile_filename():
-    """Return full pathname of installed Makefile from the Python build."""
-    if python_build:
-        return os.path.join(os.path.dirname(sys.executable), "Makefile")
-    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
-    return os.path.join(lib_dir, "config", "Makefile")
-
-
-def parse_config_h(fp, g=None):
-    """Parse a config.h-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    if g is None:
-        g = {}
-    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
-    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
-    #
-    while 1:
-        line = fp.readline()
-        if not line:
-            break
-        m = define_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            try: v = int(v)
-            except ValueError: pass
-            g[n] = v
-        else:
-            m = undef_rx.match(line)
-            if m:
-                g[m.group(1)] = 0
-    return g
-
-
-# Regexes needed for parsing Makefile (and similar syntaxes,
-# like old-style Setup files).
-_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-def parse_makefile(fn, g=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    from distutils.text_file import TextFile
-    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)
-
-    if g is None:
-        g = {}
-    done = {}
-    notdone = {}
-
-    while 1:
-        line = fp.readline()
-        if line is None:  # eof
-            break
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-
-    fp.close()
-
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    g.update(done)
-    return g
-
-
-def expand_makefile_vars(s, vars):
-    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
-    'string' according to 'vars' (a dictionary mapping variable names to
-    values).  Variables not present in 'vars' are silently expanded to the
-    empty string.  The variable values in 'vars' should not contain further
-    variable expansions; if 'vars' is the output of 'parse_makefile()',
-    you're fine.  Returns a variable-expanded version of 's'.
-    """
-
-    # This algorithm does multiple expansion, so if vars['foo'] contains
-    # "${bar}", it will expand ${foo} to ${bar}, and then expand
-    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
-    # 'parse_makefile()', which takes care of such expansions eagerly,
-    # according to make's variable expansion semantics.
-
-    while 1:
-        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
-        if m:
-            (beg, end) = m.span()
-            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
-        else:
-            break
-    return s
-
-
-_config_vars = None
-
-def _init_posix():
-    """Initialize the module as appropriate for POSIX systems."""
-    g = {}
-    # load the installed Makefile:
-    try:
-        filename = get_makefile_filename()
-        parse_makefile(filename, g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # load the installed pyconfig.h:
-    try:
-        filename = get_config_h_filename()
-        parse_config_h(file(filename), g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # On MacOSX we need to check the setting of the environment variable
-    # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
-    # it needs to be compatible.
-    # If it isn't set we set it to the configure-time value
-    if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g:
-        cfg_target = g['MACOSX_DEPLOYMENT_TARGET']
-        cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
-        if cur_target == '':
-            cur_target = cfg_target
-            os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target
-        elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
-            my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
-                % (cur_target, cfg_target))
-            raise DistutilsPlatformError(my_msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if python_build:
-        g['LDSHARED'] = g['BLDSHARED']
-
-    elif get_python_version() < '2.1':
-        # The following two branches are for 1.5.2 compatibility.
-        if sys.platform == 'aix4':          # what about AIX 3.x ?
-            # Linker script is in the config directory, not in Modules as the
-            # Makefile says.
-            python_lib = get_python_lib(standard_lib=1)
-            ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
-            python_exp = os.path.join(python_lib, 'config', 'python.exp')
-
-            g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
-
-        elif sys.platform == 'beos':
-            # Linker script is in the config directory.  In the Makefile it is
-            # relative to the srcdir, which after installation no longer makes
-            # sense.
-            python_lib = get_python_lib(standard_lib=1)
-            linkerscript_path = string.split(g['LDSHARED'])[0]
-            linkerscript_name = os.path.basename(linkerscript_path)
-            linkerscript = os.path.join(python_lib, 'config',
-                                        linkerscript_name)
-
-            # XXX this isn't the right place to do this: adding the Python
-            # library to the link, if needed, should be in the "build_ext"
-            # command.  (It's also needed for non-MS compilers on Windows, and
-            # it's taken care of for them by the 'build_ext.get_libraries()'
-            # method.)
-            g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
-                             (linkerscript, PREFIX, get_python_version()))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_nt():
-    """Initialize the module as appropriate for NT"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-    g['VERSION'] = get_python_version().replace(".", "")
-    g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_os2():
-    """Initialize the module as appropriate for OS/2"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-
-    global _config_vars
-    _config_vars = g
-
-
-def get_config_vars(*args):
-    """With no arguments, return a dictionary of all configuration
-    variables relevant for the current platform.  Generally this includes
-    everything needed to build extensions and install both pure modules and
-    extensions.  On Unix, this means every variable defined in Python's
-    installed Makefile; on Windows and Mac OS it's a much smaller set.
-
-    With arguments, return a list of values that result from looking up
-    each argument in the configuration variable dictionary.
-    """
-    global _config_vars
-    if _config_vars is None:
-        func = globals().get("_init_" + os.name)
-        if func:
-            func()
-        else:
-            _config_vars = {}
-
-        # Normalized versions of prefix and exec_prefix are handy to have;
-        # in fact, these are the standard versions used most places in the
-        # Distutils.
-        _config_vars['prefix'] = PREFIX
-        _config_vars['exec_prefix'] = EXEC_PREFIX
-
-        if sys.platform == 'darwin':
-            kernel_version = os.uname()[2] # Kernel version (8.4.3)
-            major_version = int(kernel_version.split('.')[0])
-
-            if major_version < 8:
-                # On Mac OS X before 10.4, check if -arch and -isysroot
-                # are in CFLAGS or LDFLAGS and remove them if they are.
-                # This is needed when building extensions on a 10.3 system
-                # using a universal build of python.
-                for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-                    flags = _config_vars[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
-                    _config_vars[key] = flags
-
-            else:
-
-                # Allow the user to override the architecture flags using
-                # an environment variable.
-                # NOTE: This name was introduced by Apple in OSX 10.5 and
-                # is used by several scripting languages distributed with
-                # that OS release.
-
-                if 'ARCHFLAGS' in os.environ:
-                    arch = os.environ['ARCHFLAGS']
-                    for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                        flags = _config_vars[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _config_vars[key] = flags
-
-                # If we're on OSX 10.5 or later and the user tries to
-                # compiles an extension using an SDK that is not present
-                # on the current machine it is better to not use an SDK
-                # than to fail.
-                #
-                # The major usecase for this is users using a Python.org
-                # binary installer  on OSX 10.6: that installer uses
-                # the 10.4u SDK, but that SDK is not installed by default
-                # when you install Xcode.
-                #
-                m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS'])
-                if m is not None:
-                    sdk = m.group(1)
-                    if not os.path.exists(sdk):
-                        for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                             # a number of derived variables. These need to be
-                             # patched up as well.
-                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                            flags = _config_vars[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
-                            _config_vars[key] = flags
-
-    if args:
-        vals = []
-        for name in args:
-            vals.append(_config_vars.get(name))
-        return vals
-    else:
-        return _config_vars
-
-def get_config_var(name):
-    """Return the value of a single variable using the dictionary
-    returned by 'get_config_vars()'.  Equivalent to
-    get_config_vars().get(name)
-    """
-    return get_config_vars().get(name)
diff --git a/lib-python/modified-2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
rename from lib-python/modified-2.7/distutils/sysconfig_cpython.py
rename to lib-python/2.7/distutils/sysconfig_cpython.py
diff --git a/lib-python/modified-2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
rename from lib-python/modified-2.7/distutils/sysconfig_pypy.py
rename to lib-python/2.7/distutils/sysconfig_pypy.py
diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py
--- a/lib-python/2.7/distutils/tests/test_build_ext.py
+++ b/lib-python/2.7/distutils/tests/test_build_ext.py
@@ -293,7 +293,7 @@
         finally:
             os.chdir(old_wd)
         self.assertTrue(os.path.exists(so_file))
-        self.assertEqual(os.path.splitext(so_file)[-1],
+        self.assertEqual(so_file[so_file.index(os.path.extsep):],
                          sysconfig.get_config_var('SO'))
         so_dir = os.path.dirname(so_file)
         self.assertEqual(so_dir, other_tmp_dir)
@@ -302,7 +302,7 @@
         cmd.run()
         so_file = cmd.get_outputs()[0]
         self.assertTrue(os.path.exists(so_file))
-        self.assertEqual(os.path.splitext(so_file)[-1],
+        self.assertEqual(so_file[so_file.index(os.path.extsep):],
                          sysconfig.get_config_var('SO'))
         so_dir = os.path.dirname(so_file)
         self.assertEqual(so_dir, cmd.build_lib)
diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py
--- a/lib-python/2.7/distutils/tests/test_install.py
+++ b/lib-python/2.7/distutils/tests/test_install.py
@@ -2,6 +2,7 @@
 
 import os
 import unittest
+from test import test_support
 
 from test.test_support import run_unittest
 
@@ -40,14 +41,15 @@
             expected = os.path.normpath(expected)
             self.assertEqual(got, expected)
 
-        libdir = os.path.join(destination, "lib", "python")
-        check_path(cmd.install_lib, libdir)
-        check_path(cmd.install_platlib, libdir)
-        check_path(cmd.install_purelib, libdir)
-        check_path(cmd.install_headers,
-                   os.path.join(destination, "include", "python", "foopkg"))
-        check_path(cmd.install_scripts, os.path.join(destination, "bin"))
-        check_path(cmd.install_data, destination)
+        if test_support.check_impl_detail():
+            libdir = os.path.join(destination, "lib", "python")
+            check_path(cmd.install_lib, libdir)
+            check_path(cmd.install_platlib, libdir)
+            check_path(cmd.install_purelib, libdir)
+            check_path(cmd.install_headers,
+                       os.path.join(destination, "include", "python", "foopkg"))
+            check_path(cmd.install_scripts, os.path.join(destination, "bin"))
+            check_path(cmd.install_data, destination)
 
 
 def test_suite():
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -125,7 +125,22 @@
                   }
 
     if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
         executables['ranlib'] = ["ranlib"]
+        executables['linker_so'] += ['-undefined', 'dynamic_lookup']
+
+        for k, v in executables.iteritems():
+            if v and v[0] == 'cc':
+                v += ['-arch', arch]
+
 
     # Needed for the filename generation methods provided by the base
     # class, CCompiler.  NB. whoever instantiates/uses a particular
@@ -309,7 +324,7 @@
             # On OSX users can specify an alternate SDK using
             # '-isysroot', calculate the SDK root if it is specified
             # (and use it further on)
-            cflags = sysconfig.get_config_var('CFLAGS')
+            cflags = sysconfig.get_config_var('CFLAGS') or ''
             m = re.search(r'-isysroot\s+(\S+)', cflags)
             if m is None:
                 sysroot = '/'
diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py
--- a/lib-python/2.7/heapq.py
+++ b/lib-python/2.7/heapq.py
@@ -193,6 +193,8 @@
 
     Equivalent to:  sorted(iterable, reverse=True)[:n]
     """
+    if n < 0: # for consistency with the c impl
+        return []
     it = iter(iterable)
     result = list(islice(it, n))
     if not result:
@@ -209,6 +211,8 @@
 
     Equivalent to:  sorted(iterable)[:n]
     """
+    if n < 0: # for consistency with the c impl
+        return []
     if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
         # For smaller values of n, the bisect method is faster than a minheap.
         # It is also memory efficient, consuming only n elements of space.
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -1024,7 +1024,11 @@
             kwds["buffering"] = True;
         response = self.response_class(*args, **kwds)
 
-        response.begin()
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
         assert response.will_close != _UNKNOWN
         self.__state = _CS_IDLE
 
diff --git a/lib-python/2.7/idlelib/Delegator.py b/lib-python/2.7/idlelib/Delegator.py
--- a/lib-python/2.7/idlelib/Delegator.py
+++ b/lib-python/2.7/idlelib/Delegator.py
@@ -12,6 +12,14 @@
         self.__cache[name] = attr
         return attr
 
+    def __nonzero__(self):
+        # this is needed for PyPy: else, if self.delegate is None, the
+        # __getattr__ above picks NoneType.__nonzero__, which returns
+        # False. Thus, bool(Delegator()) is False as well, but it's not what
+        # we want.  On CPython, bool(Delegator()) is True because NoneType
+        # does not have __nonzero__
+        return True
+
     def resetcache(self):
         for key in self.__cache.keys():
             try:
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -746,8 +746,15 @@
     'varargs' and 'varkw' are the names of the * and ** arguments or None."""
 
     if not iscode(co):
-        raise TypeError('{!r} is not a code object'.format(co))
+        if hasattr(len, 'func_code') and type(co) is type(len.func_code):
+            # PyPy extension: built-in function objects have a func_code too.
+            # There is no co_code on it, but co_argcount and co_varnames and
+            # co_flags are present.
+            pass
+        else:
+            raise TypeError('{!r} is not a code object'.format(co))
 
+    code = getattr(co, 'co_code', '')
     nargs = co.co_argcount
     names = co.co_varnames
     args = list(names[:nargs])
@@ -757,12 +764,12 @@
     for i in range(nargs):
         if args[i][:1] in ('', '.'):
             stack, remain, count = [], [], []
-            while step < len(co.co_code):
-                op = ord(co.co_code[step])
+            while step < len(code):
+                op = ord(code[step])
                 step = step + 1
                 if op >= dis.HAVE_ARGUMENT:
                     opname = dis.opname[op]
-                    value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
+                    value = ord(code[step]) + ord(code[step+1])*256
                     step = step + 2
                     if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
                         remain.append(value)
@@ -809,7 +816,9 @@
 
     if ismethod(func):
         func = func.im_func
-    if not isfunction(func):
+    if not (isfunction(func) or
+            isbuiltin(func) and hasattr(func, 'func_code')):
+            # PyPy extension: this works for built-in functions too
         raise TypeError('{!r} is not a Python function'.format(func))
     args, varargs, varkw = getargs(func.func_code)
     return ArgSpec(args, varargs, varkw, func.func_defaults)
@@ -949,7 +958,7 @@
                 raise TypeError('%s() takes exactly 0 arguments '
                                 '(%d given)' % (f_name, num_total))
         else:
-            raise TypeError('%s() takes no arguments (%d given)' %
+            raise TypeError('%s() takes no argument (%d given)' %
                             (f_name, num_total))
     for arg in args:
         if isinstance(arg, str) and arg in named:
diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py
--- a/lib-python/2.7/json/encoder.py
+++ b/lib-python/2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,23 +17,22 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
 FLOAT_REPR = repr
 
-def encode_basestring(s):
+def raw_encode_basestring(s):
     """Return a JSON representation of a Python string
 
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
+encode_basestring = lambda s: '"' + raw_encode_basestring(s) + '"'
 
-
-def py_encode_basestring_ascii(s):
+def raw_encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,21 +45,19 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+encode_basestring_ascii = lambda s: '"' + raw_encode_basestring_ascii(s) + '"'
 
 
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
-
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
 
@@ -147,6 +137,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = raw_encode_basestring_ascii
+        else:
+            self.encoder = raw_encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +185,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +320,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and self.indent is None and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +375,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +385,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +431,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +440,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +448,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +461,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +492,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/2.7/json/tests/test_unicode.py b/lib-python/2.7/json/tests/test_unicode.py
--- a/lib-python/2.7/json/tests/test_unicode.py
+++ b/lib-python/2.7/json/tests/test_unicode.py
@@ -80,6 +80,12 @@
         # Issue 10038.
         self.assertEqual(type(self.loads('"foo"')), unicode)
 
+    def test_encode_not_utf_8(self):
+        self.assertEqual(self.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(self.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
+
 
 class TestPyUnicode(TestUnicode, PyTest): pass
 class TestCUnicode(TestUnicode, CTest): pass
diff --git a/lib-python/2.7/multiprocessing/forking.py b/lib-python/2.7/multiprocessing/forking.py
--- a/lib-python/2.7/multiprocessing/forking.py
+++ b/lib-python/2.7/multiprocessing/forking.py
@@ -73,15 +73,12 @@
         return getattr, (m.im_self, m.im_func.func_name)
 ForkingPickler.register(type(ForkingPickler.save), _reduce_method)
 
-def _reduce_method_descriptor(m):
-    return getattr, (m.__objclass__, m.__name__)
-ForkingPickler.register(type(list.append), _reduce_method_descriptor)
-ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
-
-#def _reduce_builtin_function_or_method(m):
-#    return getattr, (m.__self__, m.__name__)
-#ForkingPickler.register(type(list().append), _reduce_builtin_function_or_method)
-#ForkingPickler.register(type(int().__add__), _reduce_builtin_function_or_method)
+if type(list.append) is not type(ForkingPickler.save):
+    # Some python implementations have unbound methods even for builtin types
+    def _reduce_method_descriptor(m):
+        return getattr, (m.__objclass__, m.__name__)
+    ForkingPickler.register(type(list.append), _reduce_method_descriptor)
+    ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
 
 try:
     from functools import partial
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -1,4 +1,3 @@
-
 """
 opcode module - potentially shared between dis and other modules which
 operate on bytecodes (e.g. peephole optimizers).
@@ -189,4 +188,10 @@
 def_op('SET_ADD', 146)
 def_op('MAP_ADD', 147)
 
+# pypy modification, experimental bytecode
+def_op('LOOKUP_METHOD', 201)          # Index in name list
+hasname.append(201)
+def_op('CALL_METHOD', 202)            # #args not including 'self'
+def_op('BUILD_LIST_FROM_ARG', 203)
+
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/pickle.py b/lib-python/2.7/pickle.py
--- a/lib-python/2.7/pickle.py
+++ b/lib-python/2.7/pickle.py
@@ -168,7 +168,7 @@
 
 # Pickling machinery
 
-class Pickler:
+class Pickler(object):
 
     def __init__(self, file, protocol=None):
         """This takes a file-like object for writing a pickle data stream.
@@ -638,6 +638,10 @@
             # else tmp is empty, and we're done
 
     def save_dict(self, obj):
+        modict_saver = self._pickle_moduledict(obj)
+        if modict_saver is not None:
+            return self.save_reduce(*modict_saver)
+
         write = self.write
 
         if self.bin:
@@ -687,6 +691,29 @@
                 write(SETITEM)
             # else tmp is empty, and we're done
 
+    def _pickle_moduledict(self, obj):
+        # save module dictionary as "getattr(module, '__dict__')"
+
+        # build index of module dictionaries
+        try:
+            modict = self.module_dict_ids
+        except AttributeError:
+            modict = {}
+            from sys import modules
+            for mod in modules.values():
+                if isinstance(mod, ModuleType):
+                    modict[id(mod.__dict__)] = mod
+            self.module_dict_ids = modict
+
+        thisid = id(obj)
+        try:
+            themodule = modict[thisid]
+        except KeyError:
+            return None
+        from __builtin__ import getattr
+        return getattr, (themodule, '__dict__')
+
+
     def save_inst(self, obj):
         cls = obj.__class__
 
@@ -727,6 +754,29 @@
 
     dispatch[InstanceType] = save_inst
 
+    def save_function(self, obj):
+        try:
+            return self.save_global(obj)
+        except PicklingError, e:
+            pass
+        # Check copy_reg.dispatch_table
+        reduce = dispatch_table.get(type(obj))
+        if reduce:
+            rv = reduce(obj)
+        else:
+            # Check for a __reduce_ex__ method, fall back to __reduce__
+            reduce = getattr(obj, "__reduce_ex__", None)
+            if reduce:
+                rv = reduce(self.proto)
+            else:
+                reduce = getattr(obj, "__reduce__", None)
+                if reduce:
+                    rv = reduce()
+                else:
+                    raise e
+        return self.save_reduce(obj=obj, *rv)
+    dispatch[FunctionType] = save_function
+
     def save_global(self, obj, name=None, pack=struct.pack):
         write = self.write
         memo = self.memo
@@ -768,7 +818,6 @@
         self.memoize(obj)
 
     dispatch[ClassType] = save_global
-    dispatch[FunctionType] = save_global
     dispatch[BuiltinFunctionType] = save_global
     dispatch[TypeType] = save_global
 
@@ -824,7 +873,7 @@
 
 # Unpickling machinery
 
-class Unpickler:
+class Unpickler(object):
 
     def __init__(self, file):
         """This takes a file-like object for reading a pickle data stream.
diff --git a/lib-python/2.7/pkgutil.py b/lib-python/2.7/pkgutil.py
--- a/lib-python/2.7/pkgutil.py
+++ b/lib-python/2.7/pkgutil.py
@@ -244,7 +244,8 @@
         return mod
 
     def get_data(self, pathname):
-        return open(pathname, "rb").read()
+        with open(pathname, "rb") as f:
+            return f.read()
 
     def _reopen(self):
         if self.file and self.file.closed:
diff --git a/lib-python/2.7/pprint.py b/lib-python/2.7/pprint.py
--- a/lib-python/2.7/pprint.py
+++ b/lib-python/2.7/pprint.py
@@ -144,7 +144,7 @@
             return
 
         r = getattr(typ, "__repr__", None)
-        if issubclass(typ, dict) and r is dict.__repr__:
+        if issubclass(typ, dict) and r == dict.__repr__:
             write('{')
             if self._indent_per_level > 1:
                 write((self._indent_per_level - 1) * ' ')
@@ -173,10 +173,10 @@
             write('}')
             return
 
-        if ((issubclass(typ, list) and r is list.__repr__) or
-            (issubclass(typ, tuple) and r is tuple.__repr__) or
-            (issubclass(typ, set) and r is set.__repr__) or
-            (issubclass(typ, frozenset) and r is frozenset.__repr__)
+        if ((issubclass(typ, list) and r == list.__repr__) or
+            (issubclass(typ, tuple) and r == tuple.__repr__) or
+            (issubclass(typ, set) and r == set.__repr__) or
+            (issubclass(typ, frozenset) and r == frozenset.__repr__)
            ):
             length = _len(object)
             if issubclass(typ, list):
@@ -266,7 +266,7 @@
         return ("%s%s%s" % (closure, sio.getvalue(), closure)), True, False
 
     r = getattr(typ, "__repr__", None)
-    if issubclass(typ, dict) and r is dict.__repr__:
+    if issubclass(typ, dict) and r == dict.__repr__:
         if not object:
             return "{}", True, False
         objid = _id(object)
@@ -291,8 +291,8 @@
         del context[objid]
         return "{%s}" % _commajoin(components), readable, recursive
 
-    if (issubclass(typ, list) and r is list.__repr__) or \
-       (issubclass(typ, tuple) and r is tuple.__repr__):
+    if (issubclass(typ, list) and r == list.__repr__) or \
+       (issubclass(typ, tuple) and r == tuple.__repr__):
         if issubclass(typ, list):
             if not object:
                 return "[]", True, False
diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py
--- a/lib-python/2.7/pydoc.py
+++ b/lib-python/2.7/pydoc.py
@@ -623,7 +623,9 @@
             head, '#ffffff', '#7799ee',
             '<a href=".">index</a><br>' + filelink + docloc)
 
-        modules = inspect.getmembers(object, inspect.ismodule)
+        def isnonbuiltinmodule(obj):
+            return inspect.ismodule(obj) and obj is not __builtin__
+        modules = inspect.getmembers(object, isnonbuiltinmodule)
 
         classes, cdict = [], {}
         for key, value in inspect.getmembers(object, inspect.isclass):
diff --git a/lib-python/2.7/random.py b/lib-python/2.7/random.py
--- a/lib-python/2.7/random.py
+++ b/lib-python/2.7/random.py
@@ -41,7 +41,6 @@
 
 from __future__ import division
 from warnings import warn as _warn
-from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethodType
 from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil
 from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin
 from os import urandom as _urandom
@@ -240,8 +239,7 @@
 
         return self.randrange(a, b+1)
 
-    def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L<<BPF,
-                   _Method=_MethodType, _BuiltinMethod=_BuiltinMethodType):
+    def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L<<BPF):
         """Return a random int in the range [0,n)
 
         Handles the case where n has more bits than returned
@@ -256,7 +254,8 @@
             # Only call self.getrandbits if the original random() builtin method
             # has not been overridden or if a new getrandbits() was supplied.
             # This assures that the two methods correspond.
-            if type(self.random) is _BuiltinMethod or type(getrandbits) is _Method:
+            if (self.random == super(Random, self).random or
+                getrandbits != super(Random, self).getrandbits):
                 k = int(1.00001 + _log(n-1, 2.0))   # 2**k > n-1 > 2**(k-2)
                 r = getrandbits(k)
                 while r >= n:
diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py
--- a/lib-python/2.7/site.py
+++ b/lib-python/2.7/site.py
@@ -75,7 +75,6 @@
 USER_SITE = None
 USER_BASE = None
 
-
 def makepath(*paths):
     dir = os.path.join(*paths)
     try:
@@ -91,7 +90,10 @@
         if hasattr(m, '__loader__'):
             continue   # don't mess with a PEP 302-supplied __file__
         try:
-            m.__file__ = os.path.abspath(m.__file__)
+            prev = m.__file__
+            new = os.path.abspath(m.__file__)
+            if prev != new:
+                m.__file__ = new
         except (AttributeError, OSError):
             pass
 
@@ -289,6 +291,7 @@
     will find its `site-packages` subdirectory depending on the system
     environment, and will return a list of full paths.
     """
+    is_pypy = '__pypy__' in sys.builtin_module_names
     sitepackages = []
     seen = set()
 
@@ -299,6 +302,10 @@
 
         if sys.platform in ('os2emx', 'riscos'):
             sitepackages.append(os.path.join(prefix, "Lib", "site-packages"))
+        elif is_pypy:
+            from distutils.sysconfig import get_python_lib
+            sitedir = get_python_lib(standard_lib=False, prefix=prefix)
+            sitepackages.append(sitedir)
         elif os.sep == '/':
             sitepackages.append(os.path.join(prefix, "lib",
                                         "python" + sys.version[:3],
@@ -435,22 +442,33 @@
                 if key == 'q':
                     break
 
+##def setcopyright():
+##    """Set 'copyright' and 'credits' in __builtin__"""
+##    __builtin__.copyright = _Printer("copyright", sys.copyright)
+##    if sys.platform[:4] == 'java':
+##        __builtin__.credits = _Printer(
+##            "credits",
+##            "Jython is maintained by the Jython developers (www.jython.org).")
+##    else:
+##        __builtin__.credits = _Printer("credits", """\
+##    Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
+##    for supporting Python development.  See www.python.org for more information.""")
+##    here = os.path.dirname(os.__file__)
+##    __builtin__.license = _Printer(
+##        "license", "See http://www.python.org/%.3s/license.html" % sys.version,
+##        ["LICENSE.txt", "LICENSE"],
+##        [os.path.join(here, os.pardir), here, os.curdir])
+
 def setcopyright():
-    """Set 'copyright' and 'credits' in __builtin__"""
+    # XXX this is the PyPy-specific version.  Should be unified with the above.
     __builtin__.copyright = _Printer("copyright", sys.copyright)
-    if sys.platform[:4] == 'java':
-        __builtin__.credits = _Printer(
-            "credits",
-            "Jython is maintained by the Jython developers (www.jython.org).")
-    else:
-        __builtin__.credits = _Printer("credits", """\
-    Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
-    for supporting Python development.  See www.python.org for more information.""")
-    here = os.path.dirname(os.__file__)
+    __builtin__.credits = _Printer(
+        "credits",
+        "PyPy is maintained by the PyPy developers: http://pypy.org/")
     __builtin__.license = _Printer(
-        "license", "See http://www.python.org/%.3s/license.html" % sys.version,
-        ["LICENSE.txt", "LICENSE"],
-        [os.path.join(here, os.pardir), here, os.curdir])
+        "license",
+        "See https://bitbucket.org/pypy/pypy/src/default/LICENSE")
+
 
 
 class _Helper(object):
@@ -476,7 +494,7 @@
     if sys.platform == 'win32':
         import locale, codecs
         enc = locale.getdefaultlocale()[1]
-        if enc.startswith('cp'):            # "cp***" ?
+        if enc is not None and enc.startswith('cp'):            # "cp***" ?
             try:
                 codecs.lookup(enc)
             except LookupError:
@@ -532,9 +550,18 @@
                 "'import usercustomize' failed; use -v for traceback"
 
 
+def import_builtin_stuff():
+    """PyPy specific: pre-import a few built-in modules, because
+    some programs actually rely on them to be in sys.modules :-("""
+    import exceptions
+    if 'zipimport' in sys.builtin_module_names:
+        import zipimport
+
+
 def main():
     global ENABLE_USER_SITE
 
+    import_builtin_stuff()
     abs__file__()
     known_paths = removeduppaths()
     if (os.name == "posix" and sys.path and
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -46,8 +46,6 @@
 
 import _socket
 from _socket import *
-from functools import partial
-from types import MethodType
 
 try:
     import _ssl
@@ -159,11 +157,6 @@
 if sys.platform == "riscos":
     _socketmethods = _socketmethods + ('sleeptaskw',)
 
-# All the method names that must be delegated to either the real socket
-# object or the _closedsocket object.
-_delegate_methods = ("recv", "recvfrom", "recv_into", "recvfrom_into",
-                     "send", "sendto")
-
 class _closedsocket(object):
     __slots__ = []
     def _dummy(*args):
@@ -180,22 +173,43 @@
 
     __doc__ = _realsocket.__doc__
 
-    __slots__ = ["_sock", "__weakref__"] + list(_delegate_methods)
-
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
         self._sock = _sock
-        for method in _delegate_methods:
-            setattr(self, method, getattr(_sock, method))
+        self._io_refs = 0
+        self._closed = False
 
-    def close(self, _closedsocket=_closedsocket,
-              _delegate_methods=_delegate_methods, setattr=setattr):
+    def send(self, data, flags=0):
+        return self._sock.send(data, flags=flags)
+    send.__doc__ = _realsocket.send.__doc__
+
+    def recv(self, buffersize, flags=0):
+        return self._sock.recv(buffersize, flags=flags)
+    recv.__doc__ = _realsocket.recv.__doc__
+
+    def recv_into(self, buffer, nbytes=0, flags=0):
+        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+    recv_into.__doc__ = _realsocket.recv_into.__doc__
+
+    def recvfrom(self, buffersize, flags=0):
+        return self._sock.recvfrom(buffersize, flags=flags)
+    recvfrom.__doc__ = _realsocket.recvfrom.__doc__
+
+    def recvfrom_into(self, buffer, nbytes=0, flags=0):
+        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+    recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
+
+    def sendto(self, data, param2, param3=None):
+        if param3 is None:
+            return self._sock.sendto(data, param2)
+        else:
+            return self._sock.sendto(data, param2, param3)
+    sendto.__doc__ = _realsocket.sendto.__doc__
+
+    def close(self):
         # This function should not reference any globals. See issue #808164.
         self._sock = _closedsocket()
-        dummy = self._sock._dummy
-        for method in _delegate_methods:
-            setattr(self, method, dummy)
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
@@ -214,21 +228,49 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        return _fileobject(self._sock, mode, bufsize)
+        self._io_refs += 1
+        return _fileobject(self, mode, bufsize)
+
+    def _decref_socketios(self):
+        if self._io_refs > 0:
+            self._io_refs -= 1
+        if self._closed:
+            self.close()
+
+    def _real_close(self):
+        # This function should not reference any globals. See issue #808164.
+        self._sock.close()
+
+    def close(self):
+        # This function should not reference any globals. See issue #808164.
+        self._closed = True
+        if self._io_refs <= 0:
+            self._real_close()
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
     proto = property(lambda self: self._sock.proto, doc="the socket protocol")
 
-def meth(name,self,*args):
-    return getattr(self._sock,name)(*args)
+    # Delegate many calls to the raw socket object.
+    _s = ("def %(name)s(self, %(args)s): return self._sock.%(name)s(%(args)s)\n\n"
+          "%(name)s.__doc__ = _realsocket.%(name)s.__doc__\n")
+    for _m in _socketmethods:
+        # yupi! we're on pypy, all code objects have this interface
+        argcount = getattr(_realsocket, _m).im_func.func_code.co_argcount - 1
+        exec _s % {'name': _m, 'args': ', '.join('arg%d' % i for i in range(argcount))}
+    del _m, _s, argcount
 
-for _m in _socketmethods:
-    p = partial(meth,_m)
-    p.__name__ = _m
-    p.__doc__ = getattr(_realsocket,_m).__doc__
-    m = MethodType(p,None,_socketobject)
-    setattr(_socketobject,_m,m)
+    # Delegation methods with default arguments, that the code above
+    # cannot handle correctly
+    def sendall(self, data, flags=0):
+        self._sock.sendall(data, flags)
+    sendall.__doc__ = _realsocket.sendall.__doc__
+
+    def getsockopt(self, level, optname, buflen=None):
+        if buflen is None:
+            return self._sock.getsockopt(level, optname)
+        return self._sock.getsockopt(level, optname, buflen)
+    getsockopt.__doc__ = _realsocket.getsockopt.__doc__
 
 socket = SocketType = _socketobject
 
@@ -278,8 +320,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._close:
-                self._sock.close()
+            if self._sock:
+                if self._close:
+                    self._sock.close()
+                else:
+                    self._sock._decref_socketios()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/dbapi.py: tests for DB-API compliance
 #
 # Copyright (C) 2004-2010 Gerhard H&#65533;ring <gh at ghaering.de>
@@ -332,6 +332,9 @@
             def __init__(self):
                 self.value = 5
 
+            def __iter__(self):
+                return self
+
             def next(self):
                 if self.value == 10:
                     raise StopIteration
@@ -826,7 +829,7 @@
         con = sqlite.connect(":memory:")
         con.close()
         try:
-            con()
+            con("select 1")
             self.fail("Should have raised a ProgrammingError")
         except sqlite.ProgrammingError:
             pass
diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -264,6 +264,28 @@
         """
         self.assertRaises(sqlite.Warning, self.con, 1)
 
+    def CheckUpdateDescriptionNone(self):
+        """
+        Call Cursor.update with an UPDATE query and check that it sets the
+        cursor's description to be None.
+        """
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        cur.execute("UPDATE foo SET id = 3 WHERE id = 1")
+        self.assertEqual(cur.description, None)
+
+    def CheckStatementCache(self):
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        values = [(i,) for i in xrange(5)]
+        cur.executemany("INSERT INTO foo (id) VALUES (?)", values)
+
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+        self.con.commit()
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+
 def suite():
     regression_suite = unittest.makeSuite(RegressionTests, "Check")
     return unittest.TestSuite((regression_suite,))
diff --git a/lib-python/2.7/sqlite3/test/userfunctions.py b/lib-python/2.7/sqlite3/test/userfunctions.py
--- a/lib-python/2.7/sqlite3/test/userfunctions.py
+++ b/lib-python/2.7/sqlite3/test/userfunctions.py
@@ -275,12 +275,14 @@
             pass
 
     def CheckAggrNoStep(self):
+        # XXX it's better to raise OperationalError in order to stop
+        # the query earlier.
         cur = self.con.cursor()
         try:
             cur.execute("select nostep(t) from test")
-            self.fail("should have raised an AttributeError")
-        except AttributeError, e:
-            self.assertEqual(e.args[0], "AggrNoStep instance has no attribute 'step'")
+            self.fail("should have raised an OperationalError")
+        except sqlite.OperationalError, e:
+            self.assertEqual(e.args[0], "user-defined aggregate's 'step' method raised error")
 
     def CheckAggrNoFinalize(self):
         cur = self.con.cursor()
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -86,7 +86,7 @@
 else:
     _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
-from socket import socket, _fileobject, _delegate_methods, error as socket_error
+from socket import socket, _fileobject, error as socket_error
 from socket import getnameinfo as _getnameinfo
 import base64        # for DER-to-PEM translation
 import errno
@@ -103,14 +103,6 @@
                  do_handshake_on_connect=True,
                  suppress_ragged_eofs=True, ciphers=None):
         socket.__init__(self, _sock=sock._sock)
-        # The initializer for socket overrides the methods send(), recv(), etc.
-        # in the instancce, which we don't need -- but we want to provide the
-        # methods defined in SSLSocket.
-        for attr in _delegate_methods:
-            try:
-                delattr(self, attr)
-            except AttributeError:
-                pass
 
         if certfile and not keyfile:
             keyfile = certfile
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -803,7 +803,7 @@
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
             elif stderr == STDOUT:
-                errwrite = c2pwrite
+                errwrite = c2pwrite.handle # pass id to not close it
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
@@ -818,9 +818,13 @@
 
         def _make_inheritable(self, handle):
             """Return a duplicate of handle, which is inheritable"""
-            return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
+            dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
+            # If the initial handle was obtained with CreatePipe, close it.
+            if not isinstance(handle, int):
+                handle.Close()
+            return dupl
 
 
         def _find_w9xpopen(self):
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -26,6 +26,16 @@
         'scripts': '{base}/bin',
         'data'   : '{base}',
         },
+    'pypy': {
+        'stdlib': '{base}/lib-python',
+        'platstdlib': '{base}/lib-python',
+        'purelib': '{base}/lib-python',
+        'platlib': '{base}/lib-python',
+        'include': '{base}/include',
+        'platinclude': '{base}/include',
+        'scripts': '{base}/bin',
+        'data'   : '{base}',
+        },
     'nt': {
         'stdlib': '{base}/Lib',
         'platstdlib': '{base}/Lib',
@@ -158,7 +168,9 @@
     return res
 
 def _get_default_scheme():
-    if os.name == 'posix':
+    if '__pypy__' in sys.builtin_module_names:
+        return 'pypy'
+    elif os.name == 'posix':
         # the default scheme for posix is posix_prefix
         return 'posix_prefix'
     return os.name
@@ -182,126 +194,9 @@
     return env_base if env_base else joinuser("~", ".local")
 
 
-def _parse_makefile(filename, vars=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    import re
-    # Regexes needed for parsing Makefile (and similar syntaxes,
-    # like old-style Setup files).
-    _variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-    _findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-    _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-    if vars is None:
-        vars = {}
-    done = {}
-    notdone = {}
-
-    with open(filename) as f:
-        lines = f.readlines()
-
-    for line in lines:
-        if line.startswith('#') or line.strip() == '':
-            continue
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    vars.update(done)
-    return vars
-
-
-def _get_makefile_filename():
-    if _PYTHON_BUILD:
-        return os.path.join(_PROJECT_BASE, "Makefile")
-    return os.path.join(get_path('platstdlib'), "config", "Makefile")
-
-
 def _init_posix(vars):
     """Initialize the module as appropriate for POSIX systems."""
-    # load the installed Makefile:
-    makefile = _get_makefile_filename()
-    try:
-        _parse_makefile(makefile, vars)
-    except IOError, e:
-        msg = "invalid Python installation: unable to open %s" % makefile
-        if hasattr(e, "strerror"):
-            msg = msg + " (%s)" % e.strerror
-        raise IOError(msg)
-
-    # load the installed pyconfig.h:
-    config_h = get_config_h_filename()
-    try:
-        with open(config_h) as f:
-            parse_config_h(f, vars)
-    except IOError, e:
-        msg = "invalid Python installation: unable to open %s" % config_h
-        if hasattr(e, "strerror"):
-            msg = msg + " (%s)" % e.strerror
-        raise IOError(msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if _PYTHON_BUILD:
-        vars['LDSHARED'] = vars['BLDSHARED']
+    return
 
 def _init_non_posix(vars):
     """Initialize the module as appropriate for NT"""
@@ -474,10 +369,11 @@
                         # patched up as well.
                         'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
 
-                        flags = _CONFIG_VARS[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _CONFIG_VARS[key] = flags
+                        if key in _CONFIG_VARS: 
+                            flags = _CONFIG_VARS[key]
+                            flags = re.sub('-arch\s+\w+\s', ' ', flags)
+                            flags = flags + ' ' + arch
+                            _CONFIG_VARS[key] = flags
 
                 # If we're on OSX 10.5 or later and the user tries to
                 # compiles an extension using an SDK that is not present
diff --git a/lib-python/2.7/tarfile.py b/lib-python/2.7/tarfile.py
--- a/lib-python/2.7/tarfile.py
+++ b/lib-python/2.7/tarfile.py
@@ -1716,9 +1716,6 @@
         except (ImportError, AttributeError):
             raise CompressionError("gzip module is not available")
 
-        if fileobj is None:
-            fileobj = bltn_open(name, mode + "b")
-
         try:
             t = cls.taropen(name, mode,
                 gzip.GzipFile(name, mode, compresslevel, fileobj),
diff --git a/lib-python/2.7/test/list_tests.py b/lib-python/2.7/test/list_tests.py
--- a/lib-python/2.7/test/list_tests.py
+++ b/lib-python/2.7/test/list_tests.py
@@ -45,8 +45,12 @@
         self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
         self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
 
+        if test_support.check_impl_detail():
+            depth = sys.getrecursionlimit() + 100
+        else:
+            depth = 1000 * 1000 # should be enough to exhaust the stack
         l0 = []
-        for i in xrange(sys.getrecursionlimit() + 100):
+        for i in xrange(depth):
             l0 = [l0]
         self.assertRaises(RuntimeError, repr, l0)
 
@@ -472,7 +476,11 @@
         u += "eggs"
         self.assertEqual(u, self.type2test("spameggs"))
 
-        self.assertRaises(TypeError, u.__iadd__, None)
+        def f_iadd(u, x):
+            u += x
+            return u
+
+        self.assertRaises(TypeError, f_iadd, u, None)
 
     def test_imul(self):
         u = self.type2test([0, 1])
diff --git a/lib-python/2.7/test/mapping_tests.py b/lib-python/2.7/test/mapping_tests.py
--- a/lib-python/2.7/test/mapping_tests.py
+++ b/lib-python/2.7/test/mapping_tests.py
@@ -531,7 +531,10 @@
                     self.assertEqual(va, int(ka))
                     kb, vb = tb = b.popitem()
                     self.assertEqual(vb, int(kb))
-                    self.assertTrue(not(copymode < 0 and ta != tb))
+                    if copymode < 0 and test_support.check_impl_detail():
+                        # popitem() is not guaranteed to be deterministic on
+                        # all implementations
+                        self.assertEqual(ta, tb)
                 self.assertTrue(not a)
                 self.assertTrue(not b)
 
diff --git a/lib-python/2.7/test/pickletester.py b/lib-python/2.7/test/pickletester.py
--- a/lib-python/2.7/test/pickletester.py
+++ b/lib-python/2.7/test/pickletester.py
@@ -6,7 +6,7 @@
 import pickletools
 import copy_reg
 
-from test.test_support import TestFailed, have_unicode, TESTFN
+from test.test_support import TestFailed, have_unicode, TESTFN, impl_detail
 
 # Tests that try a number of pickle protocols should have a
 #     for proto in protocols:
@@ -949,6 +949,7 @@
                              "Failed protocol %d: %r != %r"
                              % (proto, obj, loaded))
 
+    @impl_detail("pypy does not store attribute names", pypy=False)
     def test_attribute_name_interning(self):
         # Test that attribute names of pickled objects are interned when
         # unpickling.
@@ -1091,6 +1092,7 @@
         s = StringIO.StringIO("X''.")
         self.assertRaises(EOFError, self.module.load, s)
 
+    @impl_detail("no full restricted mode in pypy", pypy=False)
     def test_restricted(self):
         # issue7128: cPickle failed in restricted mode
         builtins = {self.module.__name__: self.module,
diff --git a/lib-python/2.7/test/regrtest.py b/lib-python/2.7/test/regrtest.py
--- a/lib-python/2.7/test/regrtest.py
+++ b/lib-python/2.7/test/regrtest.py
@@ -1388,7 +1388,26 @@
         test_zipimport
         test_zlib
         """,
-    'openbsd3':
+    'openbsd4':
+        """
+        test_ascii_formatd
+        test_bsddb
+        test_bsddb3
+        test_ctypes
+        test_dl
+        test_epoll
+        test_gdbm
+        test_locale
+        test_normalization
+        test_ossaudiodev
+        test_pep277
+        test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
+        test_multiprocessing
+        """,
+    'openbsd5':
         """
         test_ascii_formatd
         test_bsddb
@@ -1503,13 +1522,7 @@
         return self.expected
 
 if __name__ == '__main__':
-    # findtestdir() gets the dirname out of __file__, so we have to make it
-    # absolute before changing the working directory.
-    # For example __file__ may be relative when running trace or profile.
-    # See issue #9323.
-    __file__ = os.path.abspath(__file__)
-
-    # sanity check
+    # Simplification for findtestdir().
     assert __file__ == os.path.abspath(sys.argv[0])
 
     # When tests are run from the Python build directory, it is best practice
diff --git a/lib-python/2.7/test/seq_tests.py b/lib-python/2.7/test/seq_tests.py
--- a/lib-python/2.7/test/seq_tests.py
+++ b/lib-python/2.7/test/seq_tests.py
@@ -307,12 +307,18 @@
 
     def test_bigrepeat(self):
         import sys
-        if sys.maxint <= 2147483647:
-            x = self.type2test([0])
-            x *= 2**16
-            self.assertRaises(MemoryError, x.__mul__, 2**16)
-            if hasattr(x, '__imul__'):
-                self.assertRaises(MemoryError, x.__imul__, 2**16)
+        # we chose an N such as 2**16 * N does not fit into a cpu word
+        if sys.maxint == 2147483647:
+            # 32 bit system
+            N = 2**16
+        else:
+            # 64 bit system
+            N = 2**48
+        x = self.type2test([0])
+        x *= 2**16
+        self.assertRaises(MemoryError, x.__mul__, N)
+        if hasattr(x, '__imul__'):
+            self.assertRaises(MemoryError, x.__imul__, N)
 
     def test_subscript(self):
         a = self.type2test([10, 11])
diff --git a/lib-python/2.7/test/string_tests.py b/lib-python/2.7/test/string_tests.py
--- a/lib-python/2.7/test/string_tests.py
+++ b/lib-python/2.7/test/string_tests.py
@@ -1024,7 +1024,10 @@
         self.checkequal('abc', 'abc', '__mul__', 1)
         self.checkequal('abcabcabc', 'abc', '__mul__', 3)
         self.checkraises(TypeError, 'abc', '__mul__')
-        self.checkraises(TypeError, 'abc', '__mul__', '')
+        class Mul(object):
+            def mul(self, a, b):
+                return a * b
+        self.checkraises(TypeError, Mul(), 'mul', 'abc', '')
         # XXX: on a 64-bit system, this doesn't raise an overflow error,
         # but either raises a MemoryError, or succeeds (if you have 54TiB)
         #self.checkraises(OverflowError, 10000*'abc', '__mul__', 2000000000)
diff --git a/lib-python/2.7/test/test_abstract_numbers.py b/lib-python/2.7/test/test_abstract_numbers.py
--- a/lib-python/2.7/test/test_abstract_numbers.py
+++ b/lib-python/2.7/test/test_abstract_numbers.py
@@ -40,7 +40,8 @@
 
         c1, c2 = complex(3, 2), complex(4,1)
         # XXX: This is not ideal, but see the comment in math_trunc().
-        self.assertRaises(AttributeError, math.trunc, c1)
+        # Modified to suit PyPy, which gives TypeError in all cases
+        self.assertRaises((AttributeError, TypeError), math.trunc, c1)
         self.assertRaises(TypeError, float, c1)
         self.assertRaises(TypeError, int, c1)
 
diff --git a/lib-python/2.7/test/test_aifc.py b/lib-python/2.7/test/test_aifc.py
--- a/lib-python/2.7/test/test_aifc.py
+++ b/lib-python/2.7/test/test_aifc.py
@@ -1,4 +1,4 @@
-from test.test_support import findfile, run_unittest, TESTFN
+from test.test_support import findfile, run_unittest, TESTFN, impl_detail
 import unittest
 import os
 
@@ -68,6 +68,7 @@
         self.assertEqual(f.getparams(), fout.getparams())
         self.assertEqual(f.readframes(5), fout.readframes(5))
 
+    @impl_detail("PyPy has no audioop module yet", pypy=False)
     def test_compress(self):
         f = self.f = aifc.open(self.sndfilepath)
         fout = self.fout = aifc.open(TESTFN, 'wb')
diff --git a/lib-python/2.7/test/test_array.py b/lib-python/2.7/test/test_array.py
--- a/lib-python/2.7/test/test_array.py
+++ b/lib-python/2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, a.__add__, b)
-
-        self.assertRaises(TypeError, a.__add__, "bad")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, a.__add__, b)
-
-        self.assertRaises(TypeError, a.__iadd__, "bad")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, a.__mul__, "bad")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, a.__imul__, "bad")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
@@ -769,6 +773,7 @@
         p = proxy(s)
         self.assertEqual(p.tostring(), s.tostring())
         s = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, len, p)
 
     def test_bug_782369(self):
diff --git a/lib-python/2.7/test/test_ascii_formatd.py b/lib-python/2.7/test/test_ascii_formatd.py
--- a/lib-python/2.7/test/test_ascii_formatd.py
+++ b/lib-python/2.7/test/test_ascii_formatd.py
@@ -4,6 +4,10 @@
 
 import unittest
 from test.test_support import check_warnings, run_unittest, import_module
+from test.test_support import check_impl_detail
+
+if not check_impl_detail(cpython=True):
+    raise unittest.SkipTest("this test is only for CPython")
 
 # Skip tests if _ctypes module does not exist
 import_module('_ctypes')
diff --git a/lib-python/2.7/test/test_ast.py b/lib-python/2.7/test/test_ast.py
--- a/lib-python/2.7/test/test_ast.py
+++ b/lib-python/2.7/test/test_ast.py
@@ -20,10 +20,24 @@
 # These tests are compiled through "exec"
 # There should be atleast one test per statement
 exec_tests = [
+    # None
+    "None",
     # FunctionDef
     "def f(): pass",
+    # FunctionDef with arg
+    "def f(a): pass",
+    # FunctionDef with arg and default value
+    "def f(a=0): pass",
+    # FunctionDef with varargs
+    "def f(*args): pass",
+    # FunctionDef with kwargs
+    "def f(**kwargs): pass",
+    # FunctionDef with all kind of args
+    "def f(a, b=1, c=None, d=[], e={}, *args, **kwargs): pass",
     # ClassDef
     "class C:pass",
+    # ClassDef, new style class
+    "class C(object): pass",
     # Return
     "def f():return 1",
     # Delete
@@ -68,6 +82,27 @@
     "for a,b in c: pass",
     "[(a,b) for a,b in c]",
     "((a,b) for a,b in c)",
+    "((a,b) for (a,b) in c)",
+    # Multiline generator expression
+    """(
+    (
+    Aa
+    ,
+       Bb
+    )
+    for
+    Aa
+    ,
+    Bb in Cc
+    )""",
+    # dictcomp
+    "{a : b for w in x for m in p if g}",
+    # dictcomp with naked tuple
+    "{a : b for v,w in x}",
+    # setcomp
+    "{r for l in x if g}",
+    # setcomp with naked tuple
+    "{r for l,m in x}",
 ]
 
 # These are compiled through "single"
@@ -80,6 +115,8 @@
 # These are compiled through "eval"
 # It should test all expressions
 eval_tests = [
+  # None
+  "None",
   # BoolOp
   "a and b",
   # BinOp
@@ -90,6 +127,16 @@
   "lambda:None",
   # Dict
   "{ 1:2 }",
+  # Empty dict
+  "{}",
+  # Set
+  "{None,}",
+  # Multiline dict
+  """{
+      1
+        :
+          2
+     }""",
   # ListComp
   "[a for b in c if d]",
   # GeneratorExp
@@ -114,8 +161,14 @@
   "v",
   # List
   "[1,2,3]",
+  # Empty list
+  "[]",
   # Tuple
   "1,2,3",
+  # Tuple
+  "(1,2,3)",
+  # Empty tuple
+  "()",
   # Combination
   "a.b.c.d(a.b[1:2])",
 
@@ -141,6 +194,35 @@
             elif value is not None:
                 self._assertTrueorder(value, parent_pos)
 
+    def test_AST_objects(self):
+        if test_support.check_impl_detail():
+            # PyPy also provides a __dict__ to the ast.AST base class.
+
+            x = ast.AST()
+            try:
+                x.foobar = 21
+            except AttributeError, e:
+                self.assertEquals(e.args[0],
+                                  "'_ast.AST' object has no attribute 'foobar'")
+            else:
+                self.assert_(False)
+
+            try:
+                ast.AST(lineno=2)
+            except AttributeError, e:
+                self.assertEquals(e.args[0],
+                                  "'_ast.AST' object has no attribute 'lineno'")
+            else:
+                self.assert_(False)
+
+        try:
+            ast.AST(2)
+        except TypeError, e:
+            self.assertEquals(e.args[0],
+                              "_ast.AST constructor takes 0 positional arguments")
+        else:
+            self.assert_(False)
+
     def test_snippets(self):
         for input, output, kind in ((exec_tests, exec_results, "exec"),
                                     (single_tests, single_results, "single"),
@@ -169,6 +251,114 @@
         self.assertTrue(issubclass(ast.comprehension, ast.AST))
         self.assertTrue(issubclass(ast.Gt, ast.AST))
 
+    def test_field_attr_existence(self):
+        for name, item in ast.__dict__.iteritems():
+            if isinstance(item, type) and name != 'AST' and name[0].isupper(): # XXX: pypy does not allow abstract ast class instanciation
+                x = item()
+                if isinstance(x, ast.AST):
+                    self.assertEquals(type(x._fields), tuple)
+
+    def test_arguments(self):
+        x = ast.arguments()
+        self.assertEquals(x._fields, ('args', 'vararg', 'kwarg', 'defaults'))
+        try:
+            x.vararg
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'arguments' object has no attribute 'vararg'")
+        else:
+            self.assert_(False)
+        x = ast.arguments(1, 2, 3, 4)
+        self.assertEquals(x.vararg, 2)
+
+    def test_field_attr_writable(self):
+        x = ast.Num()
+        # We can assign to _fields
+        x._fields = 666
+        self.assertEquals(x._fields, 666)
+
+    def test_classattrs(self):
+        x = ast.Num()
+        self.assertEquals(x._fields, ('n',))
+        try:
+            x.n
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'n'")
+        else:
+            self.assert_(False)
+
+        x = ast.Num(42)
+        self.assertEquals(x.n, 42)
+        try:
+            x.lineno
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'lineno'")
+        else:
+            self.assert_(False)
+
+        y = ast.Num()
+        x.lineno = y
+        self.assertEquals(x.lineno, y)
+
+        try:
+            x.foobar
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'foobar'")
+        else:
+            self.assert_(False)
+
+        x = ast.Num(lineno=2)
+        self.assertEquals(x.lineno, 2)
+
+        x = ast.Num(42, lineno=0)
+        self.assertEquals(x.lineno, 0)
+        self.assertEquals(x._fields, ('n',))
+        self.assertEquals(x.n, 42)
+
+        self.assertRaises(TypeError, ast.Num, 1, 2)
+        self.assertRaises(TypeError, ast.Num, 1, 2, lineno=0)
+
+    def test_module(self):
+        body = [ast.Num(42)]
+        x = ast.Module(body)
+        self.assertEquals(x.body, body)
+
+    def test_nodeclass(self):
+        x = ast.BinOp()
+        self.assertEquals(x._fields, ('left', 'op', 'right'))
+
+        # Zero arguments constructor explicitely allowed
+        x = ast.BinOp()
+        # Random attribute allowed too
+        x.foobarbaz = 5
+        self.assertEquals(x.foobarbaz, 5)
+
+        n1 = ast.Num(1)
+        n3 = ast.Num(3)
+        addop = ast.Add()
+        x = ast.BinOp(n1, addop, n3)
+        self.assertEquals(x.left, n1)
+        self.assertEquals(x.op, addop)
+        self.assertEquals(x.right, n3)
+        
+        x = ast.BinOp(1, 2, 3)
+        self.assertEquals(x.left, 1)
+        self.assertEquals(x.op, 2)
+        self.assertEquals(x.right, 3)
+
+        x = ast.BinOp(1, 2, 3, lineno=0)
+        self.assertEquals(x.lineno, 0)
+
+    def test_nodeclasses(self):
+        x = ast.BinOp(1, 2, 3, lineno=0)
+        self.assertEquals(x.left, 1)
+        self.assertEquals(x.op, 2)
+        self.assertEquals(x.right, 3)
+        self.assertEquals(x.lineno, 0)
+
     def test_nodeclasses(self):
         x = ast.BinOp(1, 2, 3, lineno=0)
         self.assertEqual(x.left, 1)
@@ -178,6 +368,12 @@
 
         # node raises exception when not given enough arguments
         self.assertRaises(TypeError, ast.BinOp, 1, 2)
+        # node raises exception when given too many arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4)
+        # node raises exception when not given enough arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, lineno=0)
+        # node raises exception when given too many arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4, lineno=0)
 
         # can set attributes through kwargs too
         x = ast.BinOp(left=1, op=2, right=3, lineno=0)
@@ -186,8 +382,14 @@
         self.assertEqual(x.right, 3)
         self.assertEqual(x.lineno, 0)
 
+        # Random kwargs also allowed
+        x = ast.BinOp(1, 2, 3, foobarbaz=42)
+        self.assertEquals(x.foobarbaz, 42)
+
+    def test_no_fields(self):
         # this used to fail because Sub._fields was None
         x = ast.Sub()
+        self.assertEquals(x._fields, ())
 
     def test_pickling(self):
         import pickle
@@ -330,8 +532,15 @@
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
+('Module', [('Expr', (1, 0), ('Name', (1, 0), 'None', ('Load',)))]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',))], None, None, []), [('Pass', (1, 10))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',))], None, None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, []), [('Pass', (1, 14))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, 'kwargs', []), [('Pass', (1, 17))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',)), ('Name', (1, 9), 'b', ('Param',)), ('Name', (1, 14), 'c', ('Param',)), ('Name', (1, 22), 'd', ('Param',)), ('Name', (1, 28), 'e', ('Param',))], 'args', 'kwargs', [('Num', (1, 11), 1), ('Name', (1, 16), 'None', ('Load',)), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 52))], [])]),
 ('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [('Pass', (1, 17))], [])]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
 ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
 ('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
@@ -355,16 +564,26 @@
 ('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]),
 ('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]),
 ('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), []), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))])]))]),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))])]))]),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [])]))]),
 ]
 single_results = [
 ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
 ]
 eval_results = [
+('Expression', ('Name', (1, 0), 'None', ('Load',))),
 ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
 ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
 ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
 ('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
 ('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
+('Expression', ('Dict', (1, 0), [], [])),
+('Expression', ('Set', (1, 0), [('Name', (1, 1), 'None', ('Load',))])),
+('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
 ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
@@ -376,7 +595,10 @@
 ('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
 ('Expression', ('Name', (1, 0), 'v', ('Load',))),
 ('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('List', (1, 0), [], ('Load',))),
 ('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 1), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 0), [], ('Load',))),
 ('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
 ]
 main()
diff --git a/lib-python/2.7/test/test_builtin.py b/lib-python/2.7/test/test_builtin.py
--- a/lib-python/2.7/test/test_builtin.py
+++ b/lib-python/2.7/test/test_builtin.py
@@ -3,7 +3,8 @@
 import platform
 import unittest
 from test.test_support import fcmp, have_unicode, TESTFN, unlink, \
-                              run_unittest, check_py3k_warnings
+                              run_unittest, check_py3k_warnings, \
+                              check_impl_detail
 import warnings
 from operator import neg
 
@@ -247,12 +248,14 @@
         self.assertRaises(TypeError, compile)
         self.assertRaises(ValueError, compile, 'print 42\n', '<string>', 'badmode')
         self.assertRaises(ValueError, compile, 'print 42\n', '<string>', 'single', 0xff)
-        self.assertRaises(TypeError, compile, chr(0), 'f', 'exec')
+        if check_impl_detail(cpython=True):
+            self.assertRaises(TypeError, compile, chr(0), 'f', 'exec')
         self.assertRaises(TypeError, compile, 'pass', '?', 'exec',
                           mode='eval', source='0', filename='tmp')
         if have_unicode:
             compile(unicode('print u"\xc3\xa5"\n', 'utf8'), '', 'exec')
-            self.assertRaises(TypeError, compile, unichr(0), 'f', 'exec')
+            if check_impl_detail(cpython=True):
+                self.assertRaises(TypeError, compile, unichr(0), 'f', 'exec')
             self.assertRaises(ValueError, compile, unicode('a = 1'), 'f', 'bad')
 
 
@@ -395,12 +398,16 @@
         self.assertEqual(eval('dir()', g, m), list('xyz'))
         self.assertEqual(eval('globals()', g, m), g)
         self.assertEqual(eval('locals()', g, m), m)
-        self.assertRaises(TypeError, eval, 'a', m)
+        # on top of CPython, the first dictionary (the globals) has to
+        # be a real dict.  This is not the case on top of PyPy.
+        if check_impl_detail(pypy=False):
+            self.assertRaises(TypeError, eval, 'a', m)
+
         class A:
             "Non-mapping"
             pass
         m = A()
-        self.assertRaises(TypeError, eval, 'a', g, m)
+        self.assertRaises((TypeError, AttributeError), eval, 'a', g, m)
 
         # Verify that dict subclasses work as well
         class D(dict):
@@ -491,9 +498,10 @@
         execfile(TESTFN, globals, locals)
         self.assertEqual(locals['z'], 2)
 
+        self.assertRaises(TypeError, execfile, TESTFN, {}, ())
         unlink(TESTFN)
         self.assertRaises(TypeError, execfile)
-        self.assertRaises(TypeError, execfile, TESTFN, {}, ())
+        self.assertRaises((TypeError, IOError), execfile, TESTFN, {}, ())
         import os
         self.assertRaises(IOError, execfile, os.curdir)
         self.assertRaises(IOError, execfile, "I_dont_exist")
@@ -1108,7 +1116,8 @@
             def __cmp__(self, other):
                 raise RuntimeError
             __hash__ = None # Invalid cmp makes this unhashable
-        self.assertRaises(RuntimeError, range, a, a + 1, badzero(1))
+        if check_impl_detail(cpython=True):
+            self.assertRaises(RuntimeError, range, a, a + 1, badzero(1))
 
         # Reject floats.
         self.assertRaises(TypeError, range, 1., 1., 1.)
diff --git a/lib-python/2.7/test/test_bytes.py b/lib-python/2.7/test/test_bytes.py
--- a/lib-python/2.7/test/test_bytes.py
+++ b/lib-python/2.7/test/test_bytes.py
@@ -694,6 +694,7 @@
         self.assertEqual(b, b1)
         self.assertTrue(b is b1)
 
+    @test.test_support.impl_detail("undocumented bytes.__alloc__()")
     def test_alloc(self):
         b = bytearray()
         alloc = b.__alloc__()
@@ -821,6 +822,8 @@
         self.assertEqual(b, b"")
         self.assertEqual(c, b"")
 
+    @test.test_support.impl_detail(
+        "resizing semantics of CPython rely on refcounting")
     def test_resize_forbidden(self):
         # #4509: can't resize a bytearray when there are buffer exports, even
         # if it wouldn't reallocate the underlying buffer.
@@ -853,6 +856,26 @@
         self.assertRaises(BufferError, delslice)
         self.assertEqual(b, orig)
 
+    @test.test_support.impl_detail("resizing semantics", cpython=False)
+    def test_resize_forbidden_non_cpython(self):
+        # on non-CPython implementations, we cannot prevent changes to
+        # bytearrays just because there are buffers around.  Instead,
+        # we get (on PyPy) a buffer that follows the changes and resizes.
+        b = bytearray(range(10))
+        for v in [memoryview(b), buffer(b)]:
+            b[5] = 99
+            self.assertIn(v[5], (99, chr(99)))
+            b[5] = 100
+            b += b
+            b += b
+            b += b
+            self.assertEquals(len(v), 80)
+            self.assertIn(v[5], (100, chr(100)))
+            self.assertIn(v[79], (9, chr(9)))
+            del b[10:]
+            self.assertRaises(IndexError, lambda: v[10])
+            self.assertEquals(len(v), 10)
+
     def test_empty_bytearray(self):
         # Issue #7561: operations on empty bytearrays could crash in many
         # situations, due to a fragile implementation of the
diff --git a/lib-python/2.7/test/test_bz2.py b/lib-python/2.7/test/test_bz2.py
--- a/lib-python/2.7/test/test_bz2.py
+++ b/lib-python/2.7/test/test_bz2.py
@@ -50,6 +50,7 @@
         self.filename = TESTFN
 
     def tearDown(self):
+        test_support.gc_collect()
         if os.path.isfile(self.filename):
             os.unlink(self.filename)
 
@@ -246,6 +247,8 @@
         for i in xrange(10000):
             o = BZ2File(self.filename)
             del o
+            if i % 100 == 0:
+                test_support.gc_collect()
 
     def testOpenNonexistent(self):
         # "Test opening a nonexistent file"
@@ -310,6 +313,7 @@
             for t in threads:
                 t.join()
 
+    @test_support.impl_detail()
     def testMixedIterationReads(self):
         # Issue #8397: mixed iteration and reads should be forbidden.
         with bz2.BZ2File(self.filename, 'wb') as f:
diff --git a/lib-python/2.7/test/test_cmd_line_script.py b/lib-python/2.7/test/test_cmd_line_script.py
--- a/lib-python/2.7/test/test_cmd_line_script.py
+++ b/lib-python/2.7/test/test_cmd_line_script.py
@@ -112,6 +112,8 @@
             self._check_script(script_dir, script_name, script_dir, '')
 
     def test_directory_compiled(self):
+        if test.test_support.check_impl_detail(pypy=True):
+            raise unittest.SkipTest("pypy won't load lone .pyc files")
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             compiled_name = compile_script(script_name)
@@ -173,6 +175,8 @@
                                script_name, 'test_pkg')
 
     def test_package_compiled(self):
+        if test.test_support.check_impl_detail(pypy=True):
+            raise unittest.SkipTest("pypy won't load lone .pyc files")
         with temp_dir() as script_dir:
             pkg_dir = os.path.join(script_dir, 'test_pkg')
             make_pkg(pkg_dir)
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -82,7 +82,7 @@
 
 import unittest
 import weakref
-import _testcapi
+from test import test_support
 
 
 def consts(t):
@@ -104,7 +104,9 @@
 
 class CodeTest(unittest.TestCase):
 
+    @test_support.impl_detail("test for PyCode_NewEmpty")
     def test_newempty(self):
+        import _testcapi
         co = _testcapi.code_newempty("filename", "funcname", 15)
         self.assertEqual(co.co_filename, "filename")
         self.assertEqual(co.co_name, "funcname")
@@ -132,6 +134,7 @@
         coderef = weakref.ref(f.__code__, callback)
         self.assertTrue(bool(coderef()))
         del f
+        test_support.gc_collect()
         self.assertFalse(bool(coderef()))
         self.assertTrue(self.called)
 
diff --git a/lib-python/2.7/test/test_codeop.py b/lib-python/2.7/test/test_codeop.py
--- a/lib-python/2.7/test/test_codeop.py
+++ b/lib-python/2.7/test/test_codeop.py
@@ -3,7 +3,7 @@
    Nick Mathewson
 """
 import unittest
-from test.test_support import run_unittest, is_jython
+from test.test_support import run_unittest, is_jython, check_impl_detail
 
 from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
 
@@ -270,7 +270,9 @@
         ai("a = 'a\\\n")
 
         ai("a = 1","eval")
-        ai("a = (","eval")
+        if check_impl_detail():   # on PyPy it asks for more data, which is not
+            ai("a = (","eval")    # completely correct but hard to fix and
+                                  # really a detail (in my opinion <arigo>)
         ai("]","eval")
         ai("())","eval")
         ai("[}","eval")
diff --git a/lib-python/2.7/test/test_coercion.py b/lib-python/2.7/test/test_coercion.py
--- a/lib-python/2.7/test/test_coercion.py
+++ b/lib-python/2.7/test/test_coercion.py
@@ -1,6 +1,7 @@
 import copy
 import unittest
-from test.test_support import run_unittest, TestFailed, check_warnings
+from test.test_support import (
+    run_unittest, TestFailed, check_warnings, check_impl_detail)
 
 
 # Fake a number that implements numeric methods through __coerce__
@@ -306,12 +307,18 @@
         self.assertNotEqual(cmp(u'fish', evil_coercer), 0)
         self.assertNotEqual(cmp(slice(1), evil_coercer), 0)
         # ...but that this still works
-        class WackyComparer(object):
-            def __cmp__(slf, other):
-                self.assertTrue(other == 42, 'expected evil_coercer, got %r' % other)
-                return 0
-            __hash__ = None # Invalid cmp makes this unhashable
-        self.assertEqual(cmp(WackyComparer(), evil_coercer), 0)
+        if check_impl_detail():
+            # NB. I (arigo) would consider the following as implementation-
+            # specific.  For example, in CPython, if we replace 42 with 42.0
+            # both below and in CoerceTo() above, then the test fails.  This
+            # hints that the behavior is really dependent on some obscure
+            # internal details.
+            class WackyComparer(object):
+                def __cmp__(slf, other):
+                    self.assertTrue(other == 42, 'expected evil_coercer, got %r' % other)
+                    return 0
+                __hash__ = None # Invalid cmp makes this unhashable
+            self.assertEqual(cmp(WackyComparer(), evil_coercer), 0)
         # ...and classic classes too, since that code path is a little different
         class ClassicWackyComparer:
             def __cmp__(slf, other):
diff --git a/lib-python/2.7/test/test_compile.py b/lib-python/2.7/test/test_compile.py
--- a/lib-python/2.7/test/test_compile.py
+++ b/lib-python/2.7/test/test_compile.py
@@ -3,6 +3,7 @@
 import _ast
 from test import test_support
 import textwrap
+from test.test_support import check_impl_detail
 
 class TestSpecifics(unittest.TestCase):
 
@@ -90,12 +91,13 @@
         self.assertEqual(m.results, ('z', g))
         exec 'z = locals()' in g, m
         self.assertEqual(m.results, ('z', m))
-        try:
-            exec 'z = b' in m
-        except TypeError:
-            pass
-        else:
-            self.fail('Did not validate globals as a real dict')
+        if check_impl_detail():
+            try:
+                exec 'z = b' in m
+            except TypeError:
+                pass
+            else:
+                self.fail('Did not validate globals as a real dict')
 
         class A:
             "Non-mapping"
diff --git a/lib-python/2.7/test/test_copy.py b/lib-python/2.7/test/test_copy.py
--- a/lib-python/2.7/test/test_copy.py
+++ b/lib-python/2.7/test/test_copy.py
@@ -637,6 +637,7 @@
         self.assertEqual(v[c], d)
         self.assertEqual(len(v), 2)
         del c, d
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
         x, y = C(), C()
         # The underlying containers are decoupled
@@ -666,6 +667,7 @@
         self.assertEqual(v[a].i, b.i)
         self.assertEqual(v[c].i, d.i)
         del c
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
 
     def test_deepcopy_weakvaluedict(self):
@@ -689,6 +691,7 @@
         self.assertTrue(t is d)
         del x, y, z, t
         del d
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
 
     def test_deepcopy_bound_method(self):
diff --git a/lib-python/2.7/test/test_cpickle.py b/lib-python/2.7/test/test_cpickle.py
--- a/lib-python/2.7/test/test_cpickle.py
+++ b/lib-python/2.7/test/test_cpickle.py
@@ -61,27 +61,27 @@
     error = cPickle.BadPickleGet
 
     def test_recursive_list(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_list,
                           self)
 
     def test_recursive_tuple(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_tuple,
                           self)
 
     def test_recursive_inst(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_inst,
                           self)
 
     def test_recursive_dict(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_dict,
                           self)
 
     def test_recursive_multi(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_multi,
                           self)
 
diff --git a/lib-python/2.7/test/test_csv.py b/lib-python/2.7/test/test_csv.py
--- a/lib-python/2.7/test/test_csv.py
+++ b/lib-python/2.7/test/test_csv.py
@@ -54,8 +54,10 @@
         self.assertEqual(obj.dialect.skipinitialspace, False)
         self.assertEqual(obj.dialect.strict, False)
         # Try deleting or changing attributes (they are read-only)
-        self.assertRaises(TypeError, delattr, obj.dialect, 'delimiter')
-        self.assertRaises(TypeError, setattr, obj.dialect, 'delimiter', ':')
+        self.assertRaises((TypeError, AttributeError), delattr, obj.dialect,
+                          'delimiter')
+        self.assertRaises((TypeError, AttributeError), setattr, obj.dialect,
+                          'delimiter', ':')
         self.assertRaises(AttributeError, delattr, obj.dialect, 'quoting')
         self.assertRaises(AttributeError, setattr, obj.dialect,
                           'quoting', None)
diff --git a/lib-python/2.7/test/test_deque.py b/lib-python/2.7/test/test_deque.py
--- a/lib-python/2.7/test/test_deque.py
+++ b/lib-python/2.7/test/test_deque.py
@@ -109,7 +109,7 @@
         self.assertEqual(deque('abc', maxlen=4).maxlen, 4)
         self.assertEqual(deque('abc', maxlen=2).maxlen, 2)
         self.assertEqual(deque('abc', maxlen=0).maxlen, 0)
-        with self.assertRaises(AttributeError):
+        with self.assertRaises((AttributeError, TypeError)):
             d = deque('abc')
             d.maxlen = 10
 
@@ -352,7 +352,10 @@
         for match in (True, False):
             d = deque(['ab'])
             d.extend([MutateCmp(d, match), 'c'])
-            self.assertRaises(IndexError, d.remove, 'c')
+            # On CPython we get IndexError: deque mutated during remove().
+            # Why is it an IndexError during remove() only???
+            # On PyPy it is a RuntimeError, as in the other operations.
+            self.assertRaises((IndexError, RuntimeError), d.remove, 'c')
             self.assertEqual(d, deque())
 
     def test_repr(self):
@@ -514,7 +517,7 @@
                 container = reversed(deque([obj, 1]))
             obj.x = iter(container)
             del obj, container
-            gc.collect()
+            test_support.gc_collect()
             self.assertTrue(ref() is None, "Cycle was not collected")
 
 class TestVariousIteratorArgs(unittest.TestCase):
@@ -630,6 +633,7 @@
         p = weakref.proxy(d)
         self.assertEqual(str(p), str(d))
         d = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, str, p)
 
     def test_strange_subclass(self):
diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py
--- a/lib-python/2.7/test/test_descr.py
+++ b/lib-python/2.7/test/test_descr.py
@@ -2,6 +2,7 @@
 import sys
 import types
 import unittest
+import popen2     # trigger early the warning from popen2.py
 
 from copy import deepcopy
 from test import test_support
@@ -1128,7 +1129,7 @@
 
         # Test lookup leaks [SF bug 572567]
         import gc
-        if hasattr(gc, 'get_objects'):
+        if test_support.check_impl_detail():
             class G(object):
                 def __cmp__(self, other):
                     return 0
@@ -1741,6 +1742,10 @@
                 raise MyException
 
         for name, runner, meth_impl, ok, env in specials:
+            if name == '__length_hint__' or name == '__sizeof__':
+                if not test_support.check_impl_detail():
+                    continue
+
             class X(Checker):
                 pass
             for attr, obj in env.iteritems():
@@ -1980,7 +1985,9 @@
         except TypeError, msg:
             self.assertTrue(str(msg).find("weak reference") >= 0)
         else:
-            self.fail("weakref.ref(no) should be illegal")
+            if test_support.check_impl_detail(pypy=False):
+                self.fail("weakref.ref(no) should be illegal")
+            #else: pypy supports taking weakrefs to some more objects
         class Weak(object):
             __slots__ = ['foo', '__weakref__']
         yes = Weak()
@@ -3092,7 +3099,16 @@
         class R(J):
             __slots__ = ["__dict__", "__weakref__"]
 
-        for cls, cls2 in ((G, H), (G, I), (I, H), (Q, R), (R, Q)):
+        if test_support.check_impl_detail(pypy=False):
+            lst = ((G, H), (G, I), (I, H), (Q, R), (R, Q))
+        else:
+            # Not supported in pypy: changing the __class__ of an object
+            # to another __class__ that just happens to have the same slots.
+            # If needed, we can add the feature, but what we'll likely do
+            # then is to allow mostly any __class__ assignment, even if the
+            # classes have different __slots__, because we it's easier.
+            lst = ((Q, R), (R, Q))
+        for cls, cls2 in lst:
             x = cls()
             x.a = 1
             x.__class__ = cls2
@@ -3175,7 +3191,8 @@
             except TypeError:
                 pass
             else:
-                self.fail("%r's __dict__ can be modified" % cls)
+                if test_support.check_impl_detail(pypy=False):
+                    self.fail("%r's __dict__ can be modified" % cls)
 
         # Modules also disallow __dict__ assignment
         class Module1(types.ModuleType, Base):
@@ -4383,13 +4400,10 @@
         self.assertTrue(l.__add__ != [5].__add__)
         self.assertTrue(l.__add__ != l.__mul__)
         self.assertTrue(l.__add__.__name__ == '__add__')
-        if hasattr(l.__add__, '__self__'):
-            # CPython
-            self.assertTrue(l.__add__.__self__ is l)
+        self.assertTrue(l.__add__.__self__ is l)
+        if hasattr(l.__add__, '__objclass__'):   # CPython
             self.assertTrue(l.__add__.__objclass__ is list)
-        else:
-            # Python implementations where [].__add__ is a normal bound method
-            self.assertTrue(l.__add__.im_self is l)
+        else:                                    # PyPy
             self.assertTrue(l.__add__.im_class is list)
         self.assertEqual(l.__add__.__doc__, list.__add__.__doc__)
         try:
@@ -4578,8 +4592,12 @@
             str.split(fake_str)
 
         # call a slot wrapper descriptor
-        with self.assertRaises(TypeError):
-            str.__add__(fake_str, "abc")
+        try:
+            r = str.__add__(fake_str, "abc")
+        except TypeError:
+            pass
+        else:
+            self.assertEqual(r, NotImplemented)
 
 
 class DictProxyTests(unittest.TestCase):
diff --git a/lib-python/2.7/test/test_descrtut.py b/lib-python/2.7/test/test_descrtut.py
--- a/lib-python/2.7/test/test_descrtut.py
+++ b/lib-python/2.7/test/test_descrtut.py
@@ -172,46 +172,12 @@
     AttributeError: 'list' object has no attribute '__methods__'
     >>>
 
-Instead, you can get the same information from the list type:
+Instead, you can get the same information from the list type
+(the following example filters out the numerous method names
+starting with '_'):
 
-    >>> pprint.pprint(dir(list))    # like list.__dict__.keys(), but sorted
-    ['__add__',
-     '__class__',
-     '__contains__',
-     '__delattr__',
-     '__delitem__',
-     '__delslice__',
-     '__doc__',
-     '__eq__',
-     '__format__',
-     '__ge__',
-     '__getattribute__',
-     '__getitem__',
-     '__getslice__',
-     '__gt__',
-     '__hash__',
-     '__iadd__',
-     '__imul__',
-     '__init__',
-     '__iter__',
-     '__le__',
-     '__len__',
-     '__lt__',
-     '__mul__',
-     '__ne__',
-     '__new__',
-     '__reduce__',
-     '__reduce_ex__',
-     '__repr__',
-     '__reversed__',
-     '__rmul__',
-     '__setattr__',
-     '__setitem__',
-     '__setslice__',
-     '__sizeof__',
-     '__str__',
-     '__subclasshook__',
-     'append',
+    >>> pprint.pprint([name for name in dir(list) if not name.startswith('_')])
+    ['append',
      'count',
      'extend',
      'index',
diff --git a/lib-python/2.7/test/test_dict.py b/lib-python/2.7/test/test_dict.py
--- a/lib-python/2.7/test/test_dict.py
+++ b/lib-python/2.7/test/test_dict.py
@@ -319,7 +319,8 @@
                     self.assertEqual(va, int(ka))
                     kb, vb = tb = b.popitem()
                     self.assertEqual(vb, int(kb))
-                    self.assertFalse(copymode < 0 and ta != tb)
+                    if test_support.check_impl_detail():
+                        self.assertFalse(copymode < 0 and ta != tb)
                 self.assertFalse(a)
                 self.assertFalse(b)
 
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -56,8 +56,8 @@
  %-4d         0 LOAD_CONST               1 (0)
               3 POP_JUMP_IF_TRUE        38
               6 LOAD_GLOBAL              0 (AssertionError)
-              9 BUILD_LIST               0
-             12 LOAD_FAST                0 (x)
+              9 LOAD_FAST                0 (x)
+             12 BUILD_LIST_FROM_ARG      0
              15 GET_ITER
         >>   16 FOR_ITER                12 (to 31)
              19 STORE_FAST               1 (s)
diff --git a/lib-python/2.7/test/test_doctest.py b/lib-python/2.7/test/test_doctest.py
--- a/lib-python/2.7/test/test_doctest.py
+++ b/lib-python/2.7/test/test_doctest.py
@@ -782,7 +782,7 @@
     ...     >>> x = 12
     ...     >>> print x//0
     ...     Traceback (most recent call last):
-    ...     ZeroDivisionError: integer division or modulo by zero
+    ...     ZeroDivisionError: integer division by zero
     ...     '''
     >>> test = doctest.DocTestFinder().find(f)[0]
     >>> doctest.DocTestRunner(verbose=False).run(test)
@@ -799,7 +799,7 @@
     ...     >>> print 'pre-exception output', x//0
     ...     pre-exception output
     ...     Traceback (most recent call last):
-    ...     ZeroDivisionError: integer division or modulo by zero
+    ...     ZeroDivisionError: integer division by zero
     ...     '''
     >>> test = doctest.DocTestFinder().find(f)[0]
     >>> doctest.DocTestRunner(verbose=False).run(test)
@@ -810,7 +810,7 @@
         print 'pre-exception output', x//0
     Exception raised:
         ...
-        ZeroDivisionError: integer division or modulo by zero
+        ZeroDivisionError: integer division by zero
     TestResults(failed=1, attempted=2)
 
 Exception messages may contain newlines:
@@ -978,7 +978,7 @@
     Exception raised:
         Traceback (most recent call last):
         ...
-        ZeroDivisionError: integer division or modulo by zero
+        ZeroDivisionError: integer division by zero
     TestResults(failed=1, attempted=1)
 """
     def displayhook(): r"""
@@ -1924,7 +1924,7 @@
     > <doctest foo-b&#228;r at baz[1]>(1)<module>()
     -> calls_set_trace()
     (Pdb) print foo
-    *** NameError: name 'foo' is not defined
+    *** NameError: global name 'foo' is not defined
     (Pdb) continue
     TestResults(failed=0, attempted=2)
 """
@@ -2229,7 +2229,7 @@
           favorite_color
       Exception raised:
           ...
-          NameError: name 'favorite_color' is not defined
+          NameError: global name 'favorite_color' is not defined
       <BLANKLINE>
       <BLANKLINE>
 
@@ -2289,7 +2289,7 @@
         favorite_color
     Exception raised:
         ...
-        NameError: name 'favorite_color' is not defined
+        NameError: global name 'favorite_color' is not defined
     **********************************************************************
     1 items had failures:
        1 of   2 in test_doctest.txt
@@ -2382,7 +2382,7 @@
         favorite_color
     Exception raised:
         ...
-        NameError: name 'favorite_color' is not defined
+        NameError: global name 'favorite_color' is not defined
     TestResults(failed=1, attempted=2)
     >>> doctest.master = None  # Reset master.
 
diff --git a/lib-python/2.7/test/test_dumbdbm.py b/lib-python/2.7/test/test_dumbdbm.py
--- a/lib-python/2.7/test/test_dumbdbm.py
+++ b/lib-python/2.7/test/test_dumbdbm.py
@@ -107,9 +107,11 @@
         f.close()
 
         # Mangle the file by adding \r before each newline
-        data = open(_fname + '.dir').read()
+        with open(_fname + '.dir') as f:
+            data = f.read()
         data = data.replace('\n', '\r\n')
-        open(_fname + '.dir', 'wb').write(data)
+        with open(_fname + '.dir', 'wb') as f:
+            f.write(data)
 
         f = dumbdbm.open(_fname)
         self.assertEqual(f['1'], 'hello')
diff --git a/lib-python/2.7/test/test_extcall.py b/lib-python/2.7/test/test_extcall.py
--- a/lib-python/2.7/test/test_extcall.py
+++ b/lib-python/2.7/test/test_extcall.py
@@ -90,19 +90,19 @@
 
     >>> class Nothing: pass
     ...
-    >>> g(*Nothing())
+    >>> g(*Nothing())                     #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: g() argument after * must be a sequence, not instance
+    TypeError: ...argument after * must be a sequence, not instance
 
     >>> class Nothing:
     ...     def __len__(self): return 5
     ...
 
-    >>> g(*Nothing())
+    >>> g(*Nothing())                     #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: g() argument after * must be a sequence, not instance
+    TypeError: ...argument after * must be a sequence, not instance
 
     >>> class Nothing():
     ...     def __len__(self): return 5
@@ -154,52 +154,50 @@
       ...
     TypeError: g() got multiple values for keyword argument 'x'
 
-    >>> f(**{1:2})
+    >>> f(**{1:2})                             #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: f() keywords must be strings
+    TypeError: ...keywords must be strings
 
     >>> h(**{'e': 2})
     Traceback (most recent call last):
       ...
     TypeError: h() got an unexpected keyword argument 'e'
 
-    >>> h(*h)
+    >>> h(*h)                                  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after * must be a sequence, not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> dir(*h)
+    >>> dir(*h)                                #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() argument after * must be a sequence, not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> None(*h)
+    >>> None(*h)                               #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: NoneType object argument after * must be a sequence, \
-not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> h(**h)
+    >>> h(**h)                                 #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> dir(**h)
+    >>> dir(**h)                               #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> None(**h)
+    >>> None(**h)                              #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: NoneType object argument after ** must be a mapping, \
-not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> dir(b=1, **{'b': 1})
+    >>> dir(b=1, **{'b': 1})                   #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() got multiple values for keyword argument 'b'
+    TypeError: ...got multiple values for keyword argument 'b'
 
 Another helper function
 
@@ -247,10 +245,10 @@
     ...     False
     True
 
-    >>> id(1, **{'foo': 1})
+    >>> id(1, **{'foo': 1})                 #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: id() takes no keyword arguments
+    TypeError: id() ... keyword argument...
 
 A corner case of keyword dictionary items being deleted during
 the function call setup. See <http://bugs.python.org/issue2016>.
diff --git a/lib-python/2.7/test/test_fcntl.py b/lib-python/2.7/test/test_fcntl.py
--- a/lib-python/2.7/test/test_fcntl.py
+++ b/lib-python/2.7/test/test_fcntl.py
@@ -32,7 +32,7 @@
                         'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5',
                         'freebsd6', 'freebsd7', 'freebsd8',
                         'bsdos2', 'bsdos3', 'bsdos4',
-                        'openbsd', 'openbsd2', 'openbsd3', 'openbsd4'):
+                        'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', 'openbsd5'):
         if struct.calcsize('l') == 8:
             off_t = 'l'
             pid_t = 'i'
diff --git a/lib-python/2.7/test/test_file.py b/lib-python/2.7/test/test_file.py
--- a/lib-python/2.7/test/test_file.py
+++ b/lib-python/2.7/test/test_file.py
@@ -12,7 +12,7 @@
 import io
 import _pyio as pyio
 
-from test.test_support import TESTFN, run_unittest
+from test.test_support import TESTFN, run_unittest, gc_collect
 from UserList import UserList
 
 class AutoFileTests(unittest.TestCase):
@@ -33,6 +33,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testAttributes(self):
@@ -157,7 +158,12 @@
     def testStdin(self):
         # This causes the interpreter to exit on OSF1 v5.1.
         if sys.platform != 'osf1V5':
-            self.assertRaises((IOError, ValueError), sys.stdin.seek, -1)
+            if sys.stdin.isatty():
+                self.assertRaises((IOError, ValueError), sys.stdin.seek, -1)
+            else:
+                print((
+                    '  Skipping sys.stdin.seek(-1): stdin is not a tty.'
+                    ' Test manually.'), file=sys.__stdout__)
         else:
             print((
                 '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
diff --git a/lib-python/2.7/test/test_file2k.py b/lib-python/2.7/test/test_file2k.py
--- a/lib-python/2.7/test/test_file2k.py
+++ b/lib-python/2.7/test/test_file2k.py
@@ -11,7 +11,7 @@
     threading = None
 
 from test import test_support
-from test.test_support import TESTFN, run_unittest
+from test.test_support import TESTFN, run_unittest, gc_collect
 from UserList import UserList
 
 class AutoFileTests(unittest.TestCase):
@@ -32,6 +32,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testAttributes(self):
@@ -116,8 +117,12 @@
 
         for methodname in methods:
             method = getattr(self.f, methodname)
+            args = {'readinto': (bytearray(''),),
+                    'seek':     (0,),
+                    'write':    ('',),
+                    }.get(methodname, ())
             # should raise on closed file
-            self.assertRaises(ValueError, method)
+            self.assertRaises(ValueError, method, *args)
         with test_support.check_py3k_warnings():
             for methodname in deprecated_methods:
                 method = getattr(self.f, methodname)
@@ -216,7 +221,12 @@
     def testStdin(self):
         # This causes the interpreter to exit on OSF1 v5.1.
         if sys.platform != 'osf1V5':
-            self.assertRaises(IOError, sys.stdin.seek, -1)
+            if sys.stdin.isatty():
+                self.assertRaises(IOError, sys.stdin.seek, -1)
+            else:
+                print >>sys.__stdout__, (
+                    '  Skipping sys.stdin.seek(-1): stdin is not a tty.'
+                    ' Test manualy.')
         else:
             print >>sys.__stdout__, (
                 '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
@@ -336,8 +346,9 @@
                 except ValueError:
                     pass
                 else:
-                    self.fail("%s%r after next() didn't raise ValueError" %
-                                     (methodname, args))
+                    if test_support.check_impl_detail():
+                        self.fail("%s%r after next() didn't raise ValueError" %
+                                  (methodname, args))
                 f.close()
 
             # Test to see if harmless (by accident) mixing of read* and
@@ -388,6 +399,7 @@
             if lines != testlines:
                 self.fail("readlines() after next() with empty buffer "
                           "failed. Got %r, expected %r" % (line, testline))
+            f.close()
             # Reading after iteration hit EOF shouldn't hurt either
             f = open(TESTFN)
             try:
@@ -438,6 +450,9 @@
         self.close_count = 0
         self.close_success_count = 0
         self.use_buffering = False
+        # to prevent running out of file descriptors on PyPy,
+        # we only keep the 50 most recent files open
+        self.all_files = [None] * 50
 
     def tearDown(self):
         if self.f:
@@ -453,9 +468,14 @@
 
     def _create_file(self):
         if self.use_buffering:
-            self.f = open(self.filename, "w+", buffering=1024*16)
+            f = open(self.filename, "w+", buffering=1024*16)
         else:
-            self.f = open(self.filename, "w+")
+            f = open(self.filename, "w+")
+        self.f = f
+        self.all_files.append(f)
+        oldf = self.all_files.pop(0)
+        if oldf is not None:
+            oldf.close()
 
     def _close_file(self):
         with self._count_lock:
@@ -496,7 +516,6 @@
 
     def _test_close_open_io(self, io_func, nb_workers=5):
         def worker():
-            self._create_file()
             funcs = itertools.cycle((
                 lambda: io_func(),
                 lambda: self._close_and_reopen_file(),
@@ -508,7 +527,11 @@
                     f()
                 except (IOError, ValueError):
                     pass
+        self._create_file()
         self._run_workers(worker, nb_workers)
+        # make sure that all files can be closed now
+        del self.all_files
+        gc_collect()
         if test_support.verbose:
             # Useful verbose statistics when tuning this test to take
             # less time to run but still ensuring that its still useful.
diff --git a/lib-python/2.7/test/test_fileio.py b/lib-python/2.7/test/test_fileio.py
--- a/lib-python/2.7/test/test_fileio.py
+++ b/lib-python/2.7/test/test_fileio.py
@@ -12,6 +12,7 @@
 
 from test.test_support import TESTFN, check_warnings, run_unittest, make_bad_fd
 from test.test_support import py3k_bytes as bytes
+from test.test_support import gc_collect
 from test.script_helper import run_python
 
 from _io import FileIO as _FileIO
@@ -34,6 +35,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testSeekTell(self):
@@ -104,8 +106,8 @@
         self.assertTrue(f.closed)
 
     def testMethods(self):
-        methods = ['fileno', 'isatty', 'read', 'readinto',
-                   'seek', 'tell', 'truncate', 'write', 'seekable',
+        methods = ['fileno', 'isatty', 'read',
+                   'tell', 'truncate', 'seekable',
                    'readable', 'writable']
         if sys.platform.startswith('atheos'):
             methods.remove('truncate')
@@ -117,6 +119,10 @@
             method = getattr(self.f, methodname)
             # should raise on closed file
             self.assertRaises(ValueError, method)
+        # methods with one argument
+        self.assertRaises(ValueError, self.f.readinto, 0)
+        self.assertRaises(ValueError, self.f.write, 0)
+        self.assertRaises(ValueError, self.f.seek, 0)
 
     def testOpendir(self):
         # Issue 3703: opening a directory should fill the errno
diff --git a/lib-python/2.7/test/test_format.py b/lib-python/2.7/test/test_format.py
--- a/lib-python/2.7/test/test_format.py
+++ b/lib-python/2.7/test/test_format.py
@@ -242,7 +242,7 @@
             try:
                 testformat(formatstr, args)
             except exception, exc:
-                if str(exc) == excmsg:
+                if str(exc) == excmsg or not test_support.check_impl_detail():
                     if verbose:
                         print "yes"
                 else:
@@ -272,13 +272,16 @@
         test_exc(u'no format', u'1', TypeError,
                  "not all arguments converted during string formatting")
 
-        class Foobar(long):
-            def __oct__(self):
-                # Returning a non-string should not blow up.
-                return self + 1
-
-        test_exc('%o', Foobar(), TypeError,
-                 "expected string or Unicode object, long found")
+        if test_support.check_impl_detail():
+            # __oct__() is called if Foobar inherits from 'long', but
+            # not, say, 'object' or 'int' or 'str'.  This seems strange
+            # enough to consider it a complete implementation detail.
+            class Foobar(long):
+                def __oct__(self):
+                    # Returning a non-string should not blow up.
+                    return self + 1
+            test_exc('%o', Foobar(), TypeError,
+                     "expected string or Unicode object, long found")
 
         if maxsize == 2**31-1:
             # crashes 2.2.1 and earlier:
diff --git a/lib-python/2.7/test/test_funcattrs.py b/lib-python/2.7/test/test_funcattrs.py
--- a/lib-python/2.7/test/test_funcattrs.py
+++ b/lib-python/2.7/test/test_funcattrs.py
@@ -14,6 +14,8 @@
         self.b = b
 
     def cannot_set_attr(self, obj, name, value, exceptions):
+        if not test_support.check_impl_detail():
+            exceptions = (TypeError, AttributeError)
         # Helper method for other tests.
         try:
             setattr(obj, name, value)
@@ -286,13 +288,13 @@
     def test_delete_func_dict(self):
         try:
             del self.b.__dict__
-        except TypeError:
+        except (AttributeError, TypeError):
             pass
         else:
             self.fail("deleting function dictionary should raise TypeError")
         try:
             del self.b.func_dict
-        except TypeError:
+        except (AttributeError, TypeError):
             pass
         else:
             self.fail("deleting function dictionary should raise TypeError")
diff --git a/lib-python/2.7/test/test_functools.py b/lib-python/2.7/test/test_functools.py
--- a/lib-python/2.7/test/test_functools.py
+++ b/lib-python/2.7/test/test_functools.py
@@ -45,6 +45,8 @@
         # attributes should not be writable
         if not isinstance(self.thetype, type):
             return
+        if not test_support.check_impl_detail():
+            return
         self.assertRaises(TypeError, setattr, p, 'func', map)
         self.assertRaises(TypeError, setattr, p, 'args', (1, 2))
         self.assertRaises(TypeError, setattr, p, 'keywords', dict(a=1, b=2))
@@ -136,6 +138,7 @@
         p = proxy(f)
         self.assertEqual(f.func, p.func)
         f = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'func')
 
     def test_with_bound_and_unbound_methods(self):
@@ -172,7 +175,7 @@
                       updated=functools.WRAPPER_UPDATES):
         # Check attributes were assigned
         for name in assigned:
-            self.assertTrue(getattr(wrapper, name) is getattr(wrapped, name))
+            self.assertTrue(getattr(wrapper, name) == getattr(wrapped, name), name)
         # Check attributes were updated
         for name in updated:
             wrapper_attr = getattr(wrapper, name)
diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py
--- a/lib-python/2.7/test/test_generators.py
+++ b/lib-python/2.7/test/test_generators.py
@@ -190,7 +190,7 @@
       File "<stdin>", line 1, in ?
       File "<stdin>", line 2, in g
       File "<stdin>", line 2, in f
-    ZeroDivisionError: integer division or modulo by zero
+    ZeroDivisionError: integer division by zero
     >>> k.next()  # and the generator cannot be resumed
     Traceback (most recent call last):
       File "<stdin>", line 1, in ?
@@ -733,14 +733,16 @@
 ...     yield 1
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[0]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[0]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 >>> def f():
 ...     yield 1
 ...     return 22
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[1]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[1]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 "return None" is not the same as "return" in a generator:
 
@@ -749,7 +751,8 @@
 ...     return None
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[2]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[2]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 These are fine:
 
@@ -878,7 +881,9 @@
 ...     if 0:
 ...         yield 2             # because it's a generator (line 10)
 Traceback (most recent call last):
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[24]>, line 10)
+  ...
+  File "<doctest test.test_generators.__test__.syntax[24]>", line 10
+SyntaxError: 'return' with argument inside generator
 
 This one caused a crash (see SF bug 567538):
 
@@ -1496,6 +1501,10 @@
 """
 
 coroutine_tests = """\
+A helper function to call gc.collect() without printing
+>>> import gc
+>>> def gc_collect(): gc.collect()
+
 Sending a value into a started generator:
 
 >>> def f():
@@ -1570,13 +1579,14 @@
 >>> def f(): return lambda x=(yield): 1
 Traceback (most recent call last):
   ...
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.coroutine[22]>, line 1)
+  File "<doctest test.test_generators.__test__.coroutine[22]>", line 1
+SyntaxError: 'return' with argument inside generator
 
 >>> def f(): x = yield = y
 Traceback (most recent call last):
   ...
   File "<doctest test.test_generators.__test__.coroutine[23]>", line 1
-SyntaxError: assignment to yield expression not possible
+SyntaxError: can't assign to yield expression
 
 >>> def f(): (yield bar) = y
 Traceback (most recent call last):
@@ -1665,7 +1675,7 @@
 >>> f().throw("abc")     # throw on just-opened generator
 Traceback (most recent call last):
   ...
-TypeError: exceptions must be classes, or instances, not str
+TypeError: exceptions must be old-style classes or derived from BaseException, not str
 
 Now let's try closing a generator:
 
@@ -1697,7 +1707,7 @@
 
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 exiting
 
 >>> class context(object):
@@ -1708,7 +1718,7 @@
 ...          yield
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 exiting
 
 
@@ -1721,7 +1731,7 @@
 
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 finally
 
 
@@ -1747,6 +1757,7 @@
 >>> g = f()
 >>> g.next()
 >>> del g
+>>> gc_collect()
 >>> sys.stderr.getvalue().startswith(
 ...     "Exception RuntimeError: 'generator ignored GeneratorExit' in "
 ... )
@@ -1812,6 +1823,9 @@
 references. We add it to the standard suite so the routine refleak-tests
 would trigger if it starts being uncleanable again.
 
+>>> import gc
+>>> def gc_collect(): gc.collect()
+
 >>> import itertools
 >>> def leak():
 ...     class gen:
@@ -1863,9 +1877,10 @@
 ...
 ...     l = Leaker()
 ...     del l
+...     gc_collect()
 ...     err = sys.stderr.getvalue().strip()
 ...     err.startswith(
-...         "Exception RuntimeError: RuntimeError() in <"
+...         "Exception RuntimeError: RuntimeError() in "
 ...     )
 ...     err.endswith("> ignored")
 ...     len(err.splitlines())
diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py
--- a/lib-python/2.7/test/test_genexps.py
+++ b/lib-python/2.7/test/test_genexps.py
@@ -128,8 +128,9 @@
 
 Verify re-use of tuples (a side benefit of using genexps over listcomps)
 
+    >>> from test.test_support import check_impl_detail
     >>> tupleids = map(id, ((i,i) for i in xrange(10)))
-    >>> int(max(tupleids) - min(tupleids))
+    >>> int(max(tupleids) - min(tupleids)) if check_impl_detail() else 0
     0
 
 Verify that syntax error's are raised for genexps used as lvalues
@@ -198,13 +199,13 @@
     >>> g = (10 // i for i in (5, 0, 2))
     >>> g.next()
     2
-    >>> g.next()
+    >>> g.next()       # doctest: +ELLIPSIS
     Traceback (most recent call last):
       File "<pyshell#37>", line 1, in -toplevel-
         g.next()
       File "<pyshell#35>", line 1, in <generator expression>
         g = (10 // i for i in (5, 0, 2))
-    ZeroDivisionError: integer division or modulo by zero
+    ZeroDivisionError: integer division...by zero
     >>> g.next()
     Traceback (most recent call last):
       File "<pyshell#38>", line 1, in -toplevel-
diff --git a/lib-python/2.7/test/test_heapq.py b/lib-python/2.7/test/test_heapq.py
--- a/lib-python/2.7/test/test_heapq.py
+++ b/lib-python/2.7/test/test_heapq.py
@@ -215,6 +215,11 @@
 class TestHeapPython(TestHeap):
     module = py_heapq
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 @skipUnless(c_heapq, 'requires _heapq')
 class TestHeapC(TestHeap):
diff --git a/lib-python/2.7/test/test_import.py b/lib-python/2.7/test/test_import.py
--- a/lib-python/2.7/test/test_import.py
+++ b/lib-python/2.7/test/test_import.py
@@ -7,7 +7,8 @@
 import sys
 import unittest
 from test.test_support import (unlink, TESTFN, unload, run_unittest, rmtree,
-                               is_jython, check_warnings, EnvironmentVarGuard)
+                               is_jython, check_warnings, EnvironmentVarGuard,
+                               impl_detail, check_impl_detail)
 import textwrap
 from test import script_helper
 
@@ -69,7 +70,8 @@
                 self.assertEqual(mod.b, b,
                     "module loaded (%s) but contents invalid" % mod)
             finally:
-                unlink(source)
+                if check_impl_detail(pypy=False):
+                    unlink(source)
 
             try:
                 imp.reload(mod)
@@ -149,13 +151,16 @@
         # Compile & remove .py file, we only need .pyc (or .pyo).
         with open(filename, 'r') as f:
             py_compile.compile(filename)
-        unlink(filename)
+        if check_impl_detail(pypy=False):
+            # pypy refuses to import a .pyc if the .py does not exist
+            unlink(filename)
 
         # Need to be able to load from current dir.
         sys.path.append('')
 
         # This used to crash.
         exec 'import ' + module
+        reload(longlist)
 
         # Cleanup.
         del sys.path[-1]
@@ -326,6 +331,7 @@
         self.assertEqual(mod.code_filename, self.file_name)
         self.assertEqual(mod.func_filename, self.file_name)
 
+    @impl_detail("pypy refuses to import without a .py source", pypy=False)
     def test_module_without_source(self):
         target = "another_module.py"
         py_compile.compile(self.file_name, dfile=target)
diff --git a/lib-python/2.7/test/test_inspect.py b/lib-python/2.7/test/test_inspect.py
--- a/lib-python/2.7/test/test_inspect.py
+++ b/lib-python/2.7/test/test_inspect.py
@@ -4,11 +4,11 @@
 import unittest
 import inspect
 import linecache
-import datetime
 from UserList import UserList
 from UserDict import UserDict
 
 from test.test_support import run_unittest, check_py3k_warnings
+from test.test_support import check_impl_detail
 
 with check_py3k_warnings(
         ("tuple parameter unpacking has been removed", SyntaxWarning),
@@ -74,7 +74,8 @@
 
     def test_excluding_predicates(self):
         self.istest(inspect.isbuiltin, 'sys.exit')
-        self.istest(inspect.isbuiltin, '[].append')
+        if check_impl_detail():
+            self.istest(inspect.isbuiltin, '[].append')
         self.istest(inspect.iscode, 'mod.spam.func_code')
         self.istest(inspect.isframe, 'tb.tb_frame')
         self.istest(inspect.isfunction, 'mod.spam')
@@ -92,9 +93,9 @@
         else:
             self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals))
         if hasattr(types, 'MemberDescriptorType'):
-            self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days')
+            self.istest(inspect.ismemberdescriptor, 'type(lambda: None).func_globals')
         else:
-            self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days))
+            self.assertFalse(inspect.ismemberdescriptor(type(lambda: None).func_globals))
 
     def test_isroutine(self):
         self.assertTrue(inspect.isroutine(mod.spam))
@@ -567,7 +568,8 @@
         else:
             self.fail('Exception not raised')
         self.assertIs(type(ex1), type(ex2))
-        self.assertEqual(str(ex1), str(ex2))
+        if check_impl_detail():
+            self.assertEqual(str(ex1), str(ex2))
 
     def makeCallable(self, signature):
         """Create a function that returns its locals(), excluding the
diff --git a/lib-python/2.7/test/test_int.py b/lib-python/2.7/test/test_int.py
--- a/lib-python/2.7/test/test_int.py
+++ b/lib-python/2.7/test/test_int.py
@@ -1,7 +1,7 @@
 import sys
 
 import unittest
-from test.test_support import run_unittest, have_unicode
+from test.test_support import run_unittest, have_unicode, check_impl_detail
 import math
 
 L = [
@@ -392,9 +392,10 @@
                 try:
                     int(TruncReturnsNonIntegral())
                 except TypeError as e:
-                    self.assertEqual(str(e),
-                                      "__trunc__ returned non-Integral"
-                                      " (type NonIntegral)")
+                    if check_impl_detail(cpython=True):
+                        self.assertEqual(str(e),
+                                          "__trunc__ returned non-Integral"
+                                          " (type NonIntegral)")
                 else:
                     self.fail("Failed to raise TypeError with %s" %
                               ((base, trunc_result_base),))
diff --git a/lib-python/2.7/test/test_io.py b/lib-python/2.7/test/test_io.py
--- a/lib-python/2.7/test/test_io.py
+++ b/lib-python/2.7/test/test_io.py
@@ -2561,6 +2561,31 @@
         """Check that a partial write, when it gets interrupted, properly
         invokes the signal handler, and bubbles up the exception raised
         in the latter."""
+
+        # XXX This test has three flaws that appear when objects are
+        # XXX not reference counted.
+
+        # - if wio.write() happens to trigger a garbage collection,
+        #   the signal exception may be raised when some __del__
+        #   method is running; it will not reach the assertRaises()
+        #   call.
+
+        # - more subtle, if the wio object is not destroyed at once
+        #   and survives this function, the next opened file is likely
+        #   to have the same fileno (since the file descriptor was
+        #   actively closed).  When wio.__del__ is finally called, it
+        #   will close the other's test file...  To trigger this with
+        #   CPython, try adding "global wio" in this function.
+
+        # - This happens only for streams created by the _pyio module,
+        #   because a wio.close() that fails still consider that the
+        #   file needs to be closed again.  You can try adding an
+        #   "assert wio.closed" at the end of the function.
+
+        # Fortunately, a little gc.gollect() seems to be enough to
+        # work around all these issues.
+        support.gc_collect()
+
         read_results = []
         def _read():
             s = os.read(r, 1)
diff --git a/lib-python/2.7/test/test_isinstance.py b/lib-python/2.7/test/test_isinstance.py
--- a/lib-python/2.7/test/test_isinstance.py
+++ b/lib-python/2.7/test/test_isinstance.py
@@ -260,7 +260,18 @@
     # Make sure that calling isinstance with a deeply nested tuple for its
     # argument will raise RuntimeError eventually.
     tuple_arg = (compare_to,)
-    for cnt in xrange(sys.getrecursionlimit()+5):
+
+
+    if test_support.check_impl_detail(cpython=True):
+        RECURSION_LIMIT = sys.getrecursionlimit()
+    else:
+        # on non-CPython implementations, the maximum
+        # actual recursion limit might be higher, but
+        # probably not higher than 99999
+        #
+        RECURSION_LIMIT = 99999
+
+    for cnt in xrange(RECURSION_LIMIT+5):
         tuple_arg = (tuple_arg,)
         fxn(arg, tuple_arg)
 
diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py
--- a/lib-python/2.7/test/test_itertools.py
+++ b/lib-python/2.7/test/test_itertools.py
@@ -137,6 +137,8 @@
                 self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version
                 self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_combinations_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1)
@@ -207,7 +209,10 @@
                 self.assertEqual(result, list(cwr1(values, r)))         # matches first pure python version
                 self.assertEqual(result, list(cwr2(values, r)))         # matches second pure python version
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_combinations_with_replacement_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
+        cwr = combinations_with_replacement
         self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1)
 
@@ -271,6 +276,8 @@
                     self.assertEqual(result, list(permutations(values, None))) # test r as None
                     self.assertEqual(result, list(permutations(values)))       # test default r
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_permutations_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1)
@@ -526,6 +533,9 @@
         self.assertEqual(list(izip()), zip())
         self.assertRaises(TypeError, izip, 3)
         self.assertRaises(TypeError, izip, range(3), 3)
+
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_izip_tuple_reuse(self):
         # Check tuple re-use (implementation detail)
         self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')],
                          zip('abc', 'def'))
@@ -575,6 +585,8 @@
             else:
                 self.fail('Did not raise Type in:  ' + stmt)
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_iziplongest_tuple_reuse(self):
         # Check tuple re-use (implementation detail)
         self.assertEqual([tuple(list(pair)) for pair in izip_longest('abc', 'def')],
                          zip('abc', 'def'))
@@ -683,6 +695,8 @@
             args = map(iter, args)
             self.assertEqual(len(list(product(*args))), expected_len)
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_product_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, product('abc', 'def')))), 1)
         self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1)
@@ -771,11 +785,11 @@
         self.assertRaises(ValueError, islice, xrange(10), 1, -5, -1)
         self.assertRaises(ValueError, islice, xrange(10), 1, 10, -1)
         self.assertRaises(ValueError, islice, xrange(10), 1, 10, 0)
-        self.assertRaises(ValueError, islice, xrange(10), 'a')
-        self.assertRaises(ValueError, islice, xrange(10), 'a', 1)
-        self.assertRaises(ValueError, islice, xrange(10), 1, 'a')
-        self.assertRaises(ValueError, islice, xrange(10), 'a', 1, 1)
-        self.assertRaises(ValueError, islice, xrange(10), 1, 'a', 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a')
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a', 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 1, 'a')
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a', 1, 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 1, 'a', 1)
         self.assertEqual(len(list(islice(count(), 1, 10, maxsize))), 1)
 
         # Issue #10323:  Less islice in a predictable state
@@ -855,9 +869,17 @@
         self.assertRaises(TypeError, tee, [1,2], 3, 'x')
 
         # tee object should be instantiable
-        a, b = tee('abc')
-        c = type(a)('def')
-        self.assertEqual(list(c), list('def'))
+        if test_support.check_impl_detail():
+            # XXX I (arigo) would argue that 'type(a)(iterable)' has
+            # ill-defined semantics: it always return a fresh tee object,
+            # but depending on whether 'iterable' is itself a tee object
+            # or not, it is ok or not to continue using 'iterable' after
+            # the call.  I cannot imagine why 'type(a)(non_tee_object)'
+            # would be useful, as 'iter(non_tee_obect)' is equivalent
+            # as far as I can see.
+            a, b = tee('abc')
+            c = type(a)('def')
+            self.assertEqual(list(c), list('def'))
 
         # test long-lagged and multi-way split
         a, b, c = tee(xrange(2000), 3)
@@ -895,6 +917,7 @@
         p = proxy(a)
         self.assertEqual(getattr(p, '__class__'), type(b))
         del a
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, '__class__')
 
     def test_StopIteration(self):
@@ -1317,6 +1340,7 @@
 
 class LengthTransparency(unittest.TestCase):
 
+    @test_support.impl_detail("__length_hint__() API is undocumented")
     def test_repeat(self):
         from test.test_iterlen import len
         self.assertEqual(len(repeat(None, 50)), 50)
diff --git a/lib-python/2.7/test/test_linecache.py b/lib-python/2.7/test/test_linecache.py
--- a/lib-python/2.7/test/test_linecache.py
+++ b/lib-python/2.7/test/test_linecache.py
@@ -54,13 +54,13 @@
 
         # Check whether lines correspond to those from file iteration
         for entry in TESTS:
-            filename = os.path.join(TEST_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             for index, line in enumerate(open(filename)):
                 self.assertEqual(line, getline(filename, index + 1))
 
         # Check module loading
         for entry in MODULES:
-            filename = os.path.join(MODULE_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             for index, line in enumerate(open(filename)):
                 self.assertEqual(line, getline(filename, index + 1))
 
@@ -78,7 +78,7 @@
     def test_clearcache(self):
         cached = []
         for entry in TESTS:
-            filename = os.path.join(TEST_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             cached.append(filename)
             linecache.getline(filename, 1)
 
diff --git a/lib-python/2.7/test/test_list.py b/lib-python/2.7/test/test_list.py
--- a/lib-python/2.7/test/test_list.py
+++ b/lib-python/2.7/test/test_list.py
@@ -15,6 +15,10 @@
         self.assertEqual(list(''), [])
         self.assertEqual(list('spam'), ['s', 'p', 'a', 'm'])
 
+    # the following test also works with pypy, but eats all your address
+    # space's RAM before raising and takes too long.
+    @test_support.impl_detail("eats all your RAM before working", pypy=False)
+    def test_segfault_1(self):
         if sys.maxsize == 0x7fffffff:
             # This test can currently only work on 32-bit machines.
             # XXX If/when PySequence_Length() returns a ssize_t, it should be
@@ -32,6 +36,7 @@
             #     http://sources.redhat.com/ml/newlib/2002/msg00369.html
             self.assertRaises(MemoryError, list, xrange(sys.maxint // 2))
 
+    def test_segfault_2(self):
         # This code used to segfault in Py2.4a3
         x = []
         x.extend(-y for y in x)
diff --git a/lib-python/2.7/test/test_long.py b/lib-python/2.7/test/test_long.py
--- a/lib-python/2.7/test/test_long.py
+++ b/lib-python/2.7/test/test_long.py
@@ -530,9 +530,10 @@
                 try:
                     long(TruncReturnsNonIntegral())
                 except TypeError as e:
-                    self.assertEqual(str(e),
-                                     "__trunc__ returned non-Integral"
-                                     " (type NonIntegral)")
+                    if test_support.check_impl_detail(cpython=True):
+                        self.assertEqual(str(e),
+                                          "__trunc__ returned non-Integral"
+                                          " (type NonIntegral)")
                 else:
                     self.fail("Failed to raise TypeError with %s" %
                               ((base, trunc_result_base),))
diff --git a/lib-python/2.7/test/test_marshal.py b/lib-python/2.7/test/test_marshal.py
--- a/lib-python/2.7/test/test_marshal.py
+++ b/lib-python/2.7/test/test_marshal.py
@@ -7,20 +7,31 @@
 import unittest
 import os
 
-class IntTestCase(unittest.TestCase):
+class HelperMixin:
+    def helper(self, sample, *extra, **kwargs):
+        expected = kwargs.get('expected', sample)
+        new = marshal.loads(marshal.dumps(sample, *extra))
+        self.assertEqual(expected, new)
+        self.assertEqual(type(expected), type(new))
+        try:
+            with open(test_support.TESTFN, "wb") as f:
+                marshal.dump(sample, f, *extra)
+            with open(test_support.TESTFN, "rb") as f:
+                new = marshal.load(f)
+            self.assertEqual(expected, new)
+            self.assertEqual(type(expected), type(new))
+        finally:
+            test_support.unlink(test_support.TESTFN)
+
+
+class IntTestCase(unittest.TestCase, HelperMixin):
     def test_ints(self):
         # Test the full range of Python ints.
         n = sys.maxint
         while n:
             for expected in (-n, n):
-                s = marshal.dumps(expected)
-                got = marshal.loads(s)
-                self.assertEqual(expected, got)
-                marshal.dump(expected, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(expected, got)
+                self.helper(expected)
             n = n >> 1
-        os.unlink(test_support.TESTFN)
 
     def test_int64(self):
         # Simulate int marshaling on a 64-bit box.  This is most interesting if
@@ -48,28 +59,16 @@
 
     def test_bool(self):
         for b in (True, False):
-            new = marshal.loads(marshal.dumps(b))
-            self.assertEqual(b, new)
-            self.assertEqual(type(b), type(new))
-            marshal.dump(b, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(b, new)
-            self.assertEqual(type(b), type(new))
+            self.helper(b)
 
-class FloatTestCase(unittest.TestCase):
+class FloatTestCase(unittest.TestCase, HelperMixin):
     def test_floats(self):
         # Test a few floats
         small = 1e-25
         n = sys.maxint * 3.7e250
         while n > small:
             for expected in (-n, n):
-                f = float(expected)
-                s = marshal.dumps(f)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-                marshal.dump(f, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
+                self.helper(expected)
             n /= 123.4567
 
         f = 0.0
@@ -85,59 +84,25 @@
         while n < small:
             for expected in (-n, n):
                 f = float(expected)
+                self.helper(f)
+                self.helper(f, 1)
+            n *= 123.4567
 
-                s = marshal.dumps(f)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-
-                s = marshal.dumps(f, 1)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-
-                marshal.dump(f, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
-
-                marshal.dump(f, file(test_support.TESTFN, "wb"), 1)
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
-            n *= 123.4567
-        os.unlink(test_support.TESTFN)
-
-class StringTestCase(unittest.TestCase):
+class StringTestCase(unittest.TestCase, HelperMixin):
     def test_unicode(self):
         for s in [u"", u"Andr&#65533; Previn", u"abc", u" "*10000]:
-            new = marshal.loads(marshal.dumps(s))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-            marshal.dump(s, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-        os.unlink(test_support.TESTFN)
+            self.helper(s)
 
     def test_string(self):
         for s in ["", "Andr&#65533; Previn", "abc", " "*10000]:
-            new = marshal.loads(marshal.dumps(s))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-            marshal.dump(s, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-        os.unlink(test_support.TESTFN)
+            self.helper(s)
 
     def test_buffer(self):
         for s in ["", "Andr&#65533; Previn", "abc", " "*10000]:
             with test_support.check_py3k_warnings(("buffer.. not supported",
                                                      DeprecationWarning)):
                 b = buffer(s)
-            new = marshal.loads(marshal.dumps(b))
-            self.assertEqual(s, new)
-            marshal.dump(b, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-        os.unlink(test_support.TESTFN)
+            self.helper(b, expected=s)
 
 class ExceptionTestCase(unittest.TestCase):
     def test_exceptions(self):
@@ -150,7 +115,7 @@
         new = marshal.loads(marshal.dumps(co))
         self.assertEqual(co, new)
 
-class ContainerTestCase(unittest.TestCase):
+class ContainerTestCase(unittest.TestCase, HelperMixin):
     d = {'astring': 'foo at bar.baz.spam',
          'afloat': 7283.43,
          'anint': 2**20,
@@ -161,42 +126,20 @@
          'aunicode': u"Andr&#65533; Previn"
          }
     def test_dict(self):
-        new = marshal.loads(marshal.dumps(self.d))
-        self.assertEqual(self.d, new)
-        marshal.dump(self.d, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(self.d, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(self.d)
 
     def test_list(self):
         lst = self.d.items()
-        new = marshal.loads(marshal.dumps(lst))
-        self.assertEqual(lst, new)
-        marshal.dump(lst, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(lst, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(lst)
 
     def test_tuple(self):
         t = tuple(self.d.keys())
-        new = marshal.loads(marshal.dumps(t))
-        self.assertEqual(t, new)
-        marshal.dump(t, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(t, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(t)
 
     def test_sets(self):
         for constructor in (set, frozenset):
             t = constructor(self.d.keys())
-            new = marshal.loads(marshal.dumps(t))
-            self.assertEqual(t, new)
-            self.assertTrue(isinstance(new, constructor))
-            self.assertNotEqual(id(t), id(new))
-            marshal.dump(t, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(t, new)
-            os.unlink(test_support.TESTFN)
+            self.helper(t)
 
 class BugsTestCase(unittest.TestCase):
     def test_bug_5888452(self):
@@ -226,6 +169,7 @@
         s = 'c' + ('X' * 4*4) + '{' * 2**20
         self.assertRaises(ValueError, marshal.loads, s)
 
+    @test_support.impl_detail('specific recursion check')
     def test_recursion_limit(self):
         # Create a deeply nested structure.
         head = last = []
diff --git a/lib-python/2.7/test/test_memoryio.py b/lib-python/2.7/test/test_memoryio.py
--- a/lib-python/2.7/test/test_memoryio.py
+++ b/lib-python/2.7/test/test_memoryio.py
@@ -617,7 +617,7 @@
         state = memio.__getstate__()
         self.assertEqual(len(state), 3)
         bytearray(state[0]) # Check if state[0] supports the buffer interface.
-        self.assertIsInstance(state[1], int)
+        self.assertIsInstance(state[1], (int, long))
         self.assertTrue(isinstance(state[2], dict) or state[2] is None)
         memio.close()
         self.assertRaises(ValueError, memio.__getstate__)
diff --git a/lib-python/2.7/test/test_memoryview.py b/lib-python/2.7/test/test_memoryview.py
--- a/lib-python/2.7/test/test_memoryview.py
+++ b/lib-python/2.7/test/test_memoryview.py
@@ -26,7 +26,8 @@
     def check_getitem_with_type(self, tp):
         item = self.getitem_type
         b = tp(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         self.assertEqual(m[0], item(b"a"))
         self.assertIsInstance(m[0], bytes)
@@ -43,7 +44,8 @@
         self.assertRaises(TypeError, lambda: m[0.0])
         self.assertRaises(TypeError, lambda: m["a"])
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_getitem(self):
         for tp in self._types:
@@ -65,7 +67,8 @@
         if not self.ro_type:
             return
         b = self.ro_type(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         def setitem(value):
             m[0] = value
@@ -73,14 +76,16 @@
         self.assertRaises(TypeError, setitem, 65)
         self.assertRaises(TypeError, setitem, memoryview(b"a"))
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_setitem_writable(self):
         if not self.rw_type:
             return
         tp = self.rw_type
         b = self.rw_type(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         m[0] = tp(b"0")
         self._check_contents(tp, b, b"0bcdef")
@@ -110,13 +115,14 @@
         self.assertRaises(TypeError, setitem, (0,), b"a")
         self.assertRaises(TypeError, setitem, "a", b"a")
         # Trying to resize the memory object
-        self.assertRaises(ValueError, setitem, 0, b"")
-        self.assertRaises(ValueError, setitem, 0, b"ab")
+        self.assertRaises((ValueError, TypeError), setitem, 0, b"")
+        self.assertRaises((ValueError, TypeError), setitem, 0, b"ab")
         self.assertRaises(ValueError, setitem, slice(1,1), b"a")
         self.assertRaises(ValueError, setitem, slice(0,2), b"a")
 
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_delitem(self):
         for tp in self._types:
@@ -292,6 +298,7 @@
     def _check_contents(self, tp, obj, contents):
         self.assertEqual(obj[1:7], tp(contents))
 
+    @unittest.skipUnless(hasattr(sys, 'getrefcount'), "Reference counting")
     def test_refs(self):
         for tp in self._types:
             m = memoryview(tp(self._source))
diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py
--- a/lib-python/2.7/test/test_mmap.py
+++ b/lib-python/2.7/test/test_mmap.py
@@ -119,7 +119,8 @@
     def test_access_parameter(self):
         # Test for "access" keyword parameter
         mapsize = 10
-        open(TESTFN, "wb").write("a"*mapsize)
+        with open(TESTFN, "wb") as f:
+            f.write("a"*mapsize)
         f = open(TESTFN, "rb")
         m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_READ)
         self.assertEqual(m[:], 'a'*mapsize, "Readonly memory map data incorrect.")
@@ -168,9 +169,11 @@
         else:
             self.fail("Able to resize readonly memory map")
         f.close()
+        m.close()
         del m, f
-        self.assertEqual(open(TESTFN, "rb").read(), 'a'*mapsize,
-               "Readonly memory map data file was modified")
+        with open(TESTFN, "rb") as f:
+            self.assertEqual(f.read(), 'a'*mapsize,
+                "Readonly memory map data file was modified")
 
         # Opening mmap with size too big
         import sys
@@ -220,11 +223,13 @@
         self.assertEqual(m[:], 'd' * mapsize,
                "Copy-on-write memory map data not written correctly.")
         m.flush()
-        self.assertEqual(open(TESTFN, "rb").read(), 'c'*mapsize,
-               "Copy-on-write test data file should not be modified.")
+        f.close()
+        with open(TESTFN, "rb") as f:
+            self.assertEqual(f.read(), 'c'*mapsize,
+                "Copy-on-write test data file should not be modified.")
         # Ensuring copy-on-write maps cannot be resized
         self.assertRaises(TypeError, m.resize, 2*mapsize)
-        f.close()
+        m.close()
         del m, f
 
         # Ensuring invalid access parameter raises exception
@@ -287,6 +292,7 @@
         self.assertEqual(m.find('one', 1), 8)
         self.assertEqual(m.find('one', 1, -1), 8)
         self.assertEqual(m.find('one', 1, -2), -1)
+        m.close()
 
 
     def test_rfind(self):
@@ -305,6 +311,7 @@
         self.assertEqual(m.rfind('one', 0, -2), 0)
         self.assertEqual(m.rfind('one', 1, -1), 8)
         self.assertEqual(m.rfind('one', 1, -2), -1)
+        m.close()
 
 
     def test_double_close(self):
@@ -533,7 +540,8 @@
         if not hasattr(mmap, 'PROT_READ'):
             return
         mapsize = 10
-        open(TESTFN, "wb").write("a"*mapsize)
+        with open(TESTFN, "wb") as f:
+            f.write("a"*mapsize)
         f = open(TESTFN, "rb")
         m = mmap.mmap(f.fileno(), mapsize, prot=mmap.PROT_READ)
         self.assertRaises(TypeError, m.write, "foo")
@@ -545,7 +553,8 @@
 
     def test_io_methods(self):
         data = "0123456789"
-        open(TESTFN, "wb").write("x"*len(data))
+        with open(TESTFN, "wb") as f:
+            f.write("x"*len(data))
         f = open(TESTFN, "r+b")
         m = mmap.mmap(f.fileno(), len(data))
         f.close()
@@ -574,6 +583,7 @@
         self.assertEqual(m[:], "012bar6789")
         m.seek(8)
         self.assertRaises(ValueError, m.write, "bar")
+        m.close()
 
     if os.name == 'nt':
         def test_tagname(self):
@@ -611,7 +621,8 @@
             m.close()
 
             # Should not crash (Issue 5385)
-            open(TESTFN, "wb").write("x"*10)
+            with open(TESTFN, "wb") as f:
+                f.write("x"*10)
             f = open(TESTFN, "r+b")
             m = mmap.mmap(f.fileno(), 0)
             f.close()
diff --git a/lib-python/2.7/test/test_module.py b/lib-python/2.7/test/test_module.py
--- a/lib-python/2.7/test/test_module.py
+++ b/lib-python/2.7/test/test_module.py
@@ -1,6 +1,6 @@
 # Test the module type
 import unittest
-from test.test_support import run_unittest, gc_collect
+from test.test_support import run_unittest, gc_collect, check_impl_detail
 
 import sys
 ModuleType = type(sys)
@@ -10,8 +10,10 @@
         # An uninitialized module has no __dict__ or __name__,
         # and __doc__ is None
         foo = ModuleType.__new__(ModuleType)
-        self.assertTrue(foo.__dict__ is None)
-        self.assertRaises(SystemError, dir, foo)
+        self.assertFalse(foo.__dict__)
+        if check_impl_detail():
+            self.assertTrue(foo.__dict__ is None)
+            self.assertRaises(SystemError, dir, foo)
         try:
             s = foo.__name__
             self.fail("__name__ = %s" % repr(s))
diff --git a/lib-python/2.7/test/test_multibytecodec.py b/lib-python/2.7/test/test_multibytecodec.py
--- a/lib-python/2.7/test/test_multibytecodec.py
+++ b/lib-python/2.7/test/test_multibytecodec.py
@@ -42,7 +42,7 @@
         dec = codecs.getdecoder('euc-kr')
         myreplace  = lambda exc: (u'', sys.maxint+1)
         codecs.register_error('test.cjktest', myreplace)
-        self.assertRaises(IndexError, dec,
+        self.assertRaises((IndexError, OverflowError), dec,
                           'apple\x92ham\x93spam', 'test.cjktest')
 
     def test_codingspec(self):
@@ -148,7 +148,8 @@
 class Test_StreamReader(unittest.TestCase):
     def test_bug1728403(self):
         try:
-            open(TESTFN, 'w').write('\xa1')
+            with open(TESTFN, 'w') as f:
+                f.write('\xa1')
             f = codecs.open(TESTFN, encoding='cp949')
             self.assertRaises(UnicodeDecodeError, f.read, 2)
         finally:
diff --git a/lib-python/2.7/test/test_multibytecodec_support.py b/lib-python/2.7/test/test_multibytecodec_support.py
--- a/lib-python/2.7/test/test_multibytecodec_support.py
+++ b/lib-python/2.7/test/test_multibytecodec_support.py
@@ -110,8 +110,8 @@
         def myreplace(exc):
             return (u'x', sys.maxint + 1)
         codecs.register_error("test.cjktest", myreplace)
-        self.assertRaises(IndexError, self.encode, self.unmappedunicode,
-                          'test.cjktest')
+        self.assertRaises((IndexError, OverflowError), self.encode,
+                          self.unmappedunicode, 'test.cjktest')
 
     def test_callback_None_index(self):
         def myreplace(exc):
@@ -330,7 +330,7 @@
                             repr(csetch), repr(unich), exc.reason))
 
 def load_teststring(name):
-    dir = os.path.join(os.path.dirname(__file__), 'cjkencodings')
+    dir = test_support.findfile('cjkencodings')
     with open(os.path.join(dir, name + '.txt'), 'rb') as f:
         encoded = f.read()
     with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f:
diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py
--- a/lib-python/2.7/test/test_multiprocessing.py
+++ b/lib-python/2.7/test/test_multiprocessing.py
@@ -1316,6 +1316,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1605,6 +1606,10 @@
             if len(blocks) > maxblocks:
                 i = random.randrange(maxblocks)
                 del blocks[i]
+            # XXX There should be a better way to release resources for a
+            # single block
+            if i % maxblocks == 0:
+                import gc; gc.collect()
 
         # get the heap object
         heap = multiprocessing.heap.BufferWrapper._heap
@@ -1704,6 +1709,7 @@
         a = Foo()
         util.Finalize(a, conn.send, args=('a',))
         del a           # triggers callback for a
+        test_support.gc_collect()
 
         b = Foo()
         close_b = util.Finalize(b, conn.send, args=('b',))
diff --git a/lib-python/2.7/test/test_mutants.py b/lib-python/2.7/test/test_mutants.py
--- a/lib-python/2.7/test/test_mutants.py
+++ b/lib-python/2.7/test/test_mutants.py
@@ -1,4 +1,4 @@
-from test.test_support import verbose, TESTFN
+from test.test_support import verbose, TESTFN, check_impl_detail
 import random
 import os
 
@@ -137,10 +137,16 @@
     while dict1 and len(dict1) == len(dict2):
         if verbose:
             print ".",
-        if random.random() < 0.5:
-            c = cmp(dict1, dict2)
-        else:
-            c = dict1 == dict2
+        try:
+            if random.random() < 0.5:
+                c = cmp(dict1, dict2)
+            else:
+                c = dict1 == dict2
+        except RuntimeError:
+            # CPython never raises RuntimeError here, but other implementations
+            # might, and it's fine.
+            if check_impl_detail(cpython=True):
+                raise
     if verbose:
         print
 
diff --git a/lib-python/2.7/test/test_optparse.py b/lib-python/2.7/test/test_optparse.py
--- a/lib-python/2.7/test/test_optparse.py
+++ b/lib-python/2.7/test/test_optparse.py
@@ -383,6 +383,7 @@
         self.assertRaises(self.parser.remove_option, ('foo',), None,
                           ValueError, "no such option 'foo'")
 
+    @test_support.impl_detail("sys.getrefcount")
     def test_refleak(self):
         # If an OptionParser is carrying around a reference to a large
         # object, various cycles can prevent it from being GC'd in
diff --git a/lib-python/2.7/test/test_peepholer.py b/lib-python/2.7/test/test_peepholer.py
--- a/lib-python/2.7/test/test_peepholer.py
+++ b/lib-python/2.7/test/test_peepholer.py
@@ -41,7 +41,7 @@
     def test_none_as_constant(self):
         # LOAD_GLOBAL None  -->  LOAD_CONST None
         def f(x):
-            None
+            y = None
             return x
         asm = disassemble(f)
         for elem in ('LOAD_GLOBAL',):
@@ -67,10 +67,13 @@
             self.assertIn(elem, asm)
 
     def test_pack_unpack(self):
+        # On PyPy, "a, b = ..." is even more optimized, by removing
+        # the ROT_TWO.  But the ROT_TWO is not removed if assigning
+        # to more complex expressions, so check that.
         for line, elem in (
             ('a, = a,', 'LOAD_CONST',),
-            ('a, b = a, b', 'ROT_TWO',),
-            ('a, b, c = a, b, c', 'ROT_THREE',),
+            ('a[1], b = a, b', 'ROT_TWO',),
+            ('a, b[2], c = a, b, c', 'ROT_THREE',),
             ):
             asm = dis_single(line)
             self.assertIn(elem, asm)
@@ -78,6 +81,8 @@
             self.assertNotIn('UNPACK_TUPLE', asm)
 
     def test_folding_of_tuples_of_constants(self):
+        # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>"
+        # but on PyPy, it turns into "a=1;b=2;c=3".
         for line, elem in (
             ('a = 1,2,3', '((1, 2, 3))'),
             ('("a","b","c")', "(('a', 'b', 'c'))"),
@@ -86,7 +91,8 @@
             ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'),
             ):
             asm = dis_single(line)
-            self.assertIn(elem, asm)
+            self.assert_(elem in asm or (
+                line == 'a,b,c = 1,2,3' and 'UNPACK_TUPLE' not in asm))
             self.assertNotIn('BUILD_TUPLE', asm)
 
         # Bug 1053819:  Tuple of constants misidentified when presented with:
diff --git a/lib-python/2.7/test/test_pprint.py b/lib-python/2.7/test/test_pprint.py
--- a/lib-python/2.7/test/test_pprint.py
+++ b/lib-python/2.7/test/test_pprint.py
@@ -233,7 +233,16 @@
                                   frozenset([0, 2]),
                                   frozenset([0, 1])])}"""
         cube = test.test_set.cube(3)
-        self.assertEqual(pprint.pformat(cube), cube_repr_tgt)
+        # XXX issues of dictionary order, and for the case below,
+        # order of items in the frozenset([...]) representation.
+        # Whether we get precisely cube_repr_tgt or not is open
+        # to implementation-dependent choices (this test probably
+        # fails horribly in CPython if we tweak the dict order too).
+        got = pprint.pformat(cube)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertEqual(got, cube_repr_tgt)
+        else:
+            self.assertEqual(eval(got), cube)
         cubo_repr_tgt = """\
 {frozenset([frozenset([0, 2]), frozenset([0])]): frozenset([frozenset([frozenset([0,
                                                                                   2]),
@@ -393,7 +402,11 @@
                                                                                         2])])])}"""
 
         cubo = test.test_set.linegraph(cube)
-        self.assertEqual(pprint.pformat(cubo), cubo_repr_tgt)
+        got = pprint.pformat(cubo)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertEqual(got, cubo_repr_tgt)
+        else:
+            self.assertEqual(eval(got), cubo)
 
     def test_depth(self):
         nested_tuple = (1, (2, (3, (4, (5, 6)))))
diff --git a/lib-python/2.7/test/test_pydoc.py b/lib-python/2.7/test/test_pydoc.py
--- a/lib-python/2.7/test/test_pydoc.py
+++ b/lib-python/2.7/test/test_pydoc.py
@@ -267,8 +267,8 @@
         testpairs = (
             ('i_am_not_here', 'i_am_not_here'),
             ('test.i_am_not_here_either', 'i_am_not_here_either'),
-            ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
-            ('i_am_not_here.{}'.format(modname), 'i_am_not_here.{}'.format(modname)),
+            ('test.i_am_not_here.neither_am_i', 'i_am_not_here'),
+            ('i_am_not_here.{}'.format(modname), 'i_am_not_here'),
             ('test.{}'.format(modname), modname),
             )
 
@@ -292,8 +292,8 @@
                     result = run_pydoc(modname)
                 finally:
                     forget(modname)
-                expected = badimport_pattern % (modname, expectedinmsg)
-                self.assertEqual(expected, result)
+                expected = badimport_pattern % (modname, '(.+\\.)?' + expectedinmsg + '(\\..+)?$')
+                self.assertTrue(re.match(expected, result))
 
     def test_input_strip(self):
         missing_module = " test.i_am_not_here "
diff --git a/lib-python/2.7/test/test_pyexpat.py b/lib-python/2.7/test/test_pyexpat.py
--- a/lib-python/2.7/test/test_pyexpat.py
+++ b/lib-python/2.7/test/test_pyexpat.py
@@ -570,6 +570,9 @@
         self.assertEqual(self.n, 4)
 
 class MalformedInputText(unittest.TestCase):
+    # CPython seems to ship its own version of expat, they fixed it on this commit : 
+    # http://svn.python.org/view?revision=74429&view=revision
+    @unittest.skipIf(sys.platform == "darwin", "Expat is broken on Mac OS X 10.6.6")
     def test1(self):
         xml = "\0\r\n"
         parser = expat.ParserCreate()
@@ -579,6 +582,7 @@
         except expat.ExpatError as e:
             self.assertEqual(str(e), 'unclosed token: line 2, column 0')
 
+    @unittest.skipIf(sys.platform == "darwin", "Expat is broken on Mac OS X 10.6.6")
     def test2(self):
         xml = "<?xml version\xc2\x85='1.0'?>\r\n"
         parser = expat.ParserCreate()
diff --git a/lib-python/2.7/test/test_repr.py b/lib-python/2.7/test/test_repr.py
--- a/lib-python/2.7/test/test_repr.py
+++ b/lib-python/2.7/test/test_repr.py
@@ -9,6 +9,7 @@
 import unittest
 
 from test.test_support import run_unittest, check_py3k_warnings
+from test.test_support import check_impl_detail
 from repr import repr as r # Don't shadow builtin repr
 from repr import Repr
 
@@ -145,8 +146,11 @@
         # Functions
         eq(repr(hash), '<built-in function hash>')
         # Methods
-        self.assertTrue(repr(''.split).startswith(
-            '<built-in method split of str object at 0x'))
+        if check_impl_detail(cpython=True):
+            self.assertTrue(repr(''.split).startswith(
+                '<built-in method split of str object at 0x'))
+        elif check_impl_detail(pypy=True):
+            eq(repr(''.split), "<bound method str.split of ''>")
 
     def test_xrange(self):
         eq = self.assertEqual
@@ -185,7 +189,10 @@
     def test_descriptors(self):
         eq = self.assertEqual
         # method descriptors
-        eq(repr(dict.items), "<method 'items' of 'dict' objects>")
+        if check_impl_detail(cpython=True):
+            eq(repr(dict.items), "<method 'items' of 'dict' objects>")
+        elif check_impl_detail(pypy=True):
+            eq(repr(dict.items), "<unbound method dict.items>")
         # XXX member descriptors
         # XXX attribute descriptors
         # XXX slot descriptors
@@ -247,8 +254,14 @@
         eq = self.assertEqual
         touch(os.path.join(self.subpkgname, self.pkgname + os.extsep + 'py'))
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import areallylongpackageandmodulenametotestreprtruncation
-        eq(repr(areallylongpackageandmodulenametotestreprtruncation),
-           "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+        # On PyPy, we use %r to format the file name; on CPython it is done
+        # with '%s'.  It seems to me that %r is safer <arigo>.
+        if '__pypy__' in sys.builtin_module_names:
+            eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+               "<module %r from %r>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+        else:
+            eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+               "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
         eq(repr(sys), "<module 'sys' (built-in)>")
 
     def test_type(self):
diff --git a/lib-python/2.7/test/test_runpy.py b/lib-python/2.7/test/test_runpy.py
--- a/lib-python/2.7/test/test_runpy.py
+++ b/lib-python/2.7/test/test_runpy.py
@@ -5,10 +5,15 @@
 import sys
 import re
 import tempfile
-from test.test_support import verbose, run_unittest, forget
+from test.test_support import verbose, run_unittest, forget, check_impl_detail
 from test.script_helper import (temp_dir, make_script, compile_script,
                                 make_pkg, make_zip_script, make_zip_pkg)
 
+if check_impl_detail(pypy=True):
+    no_lone_pyc_file = True
+else:
+    no_lone_pyc_file = False
+
 
 from runpy import _run_code, _run_module_code, run_module, run_path
 # Note: This module can't safely test _run_module_as_main as it
@@ -168,13 +173,14 @@
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", mod_name
-            d2 = run_module(mod_name) # Read from bytecode
-            self.assertIn("x", d2)
-            self.assertTrue(d2["x"] == 1)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", mod_name
+                d2 = run_module(mod_name) # Read from bytecode
+                self.assertIn("x", d2)
+                self.assertTrue(d2["x"] == 1)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose: print "Module executed successfully"
@@ -190,13 +196,14 @@
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", pkg_name
-            d2 = run_module(pkg_name) # Read from bytecode
-            self.assertIn("x", d2)
-            self.assertTrue(d2["x"] == 1)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", pkg_name
+                d2 = run_module(pkg_name) # Read from bytecode
+                self.assertIn("x", d2)
+                self.assertTrue(d2["x"] == 1)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, pkg_name)
         if verbose: print "Package executed successfully"
@@ -244,15 +251,17 @@
             self.assertIn("sibling", d1)
             self.assertIn("nephew", d1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", mod_name
-            d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
-            self.assertIn("__package__", d2)
-            self.assertTrue(d2["__package__"] == pkg_name)
-            self.assertIn("sibling", d2)
-            self.assertIn("nephew", d2)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", mod_name
+                # Read from bytecode
+                d2 = run_module(mod_name, run_name=run_name)
+                self.assertIn("__package__", d2)
+                self.assertTrue(d2["__package__"] == pkg_name)
+                self.assertIn("sibling", d2)
+                self.assertIn("nephew", d2)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose: print "Module executed successfully"
@@ -345,6 +354,8 @@
                                script_dir, '')
 
     def test_directory_compiled(self):
+        if no_lone_pyc_file:
+            return
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
diff --git a/lib-python/2.7/test/test_scope.py b/lib-python/2.7/test/test_scope.py
--- a/lib-python/2.7/test/test_scope.py
+++ b/lib-python/2.7/test/test_scope.py
@@ -1,6 +1,6 @@
 import unittest
 from test.test_support import check_syntax_error, check_py3k_warnings, \
-                              check_warnings, run_unittest
+                              check_warnings, run_unittest, gc_collect
 
 
 class ScopeTests(unittest.TestCase):
@@ -432,6 +432,7 @@
 
         for i in range(100):
             f1()
+        gc_collect()
 
         self.assertEqual(Foo.count, 0)
 
diff --git a/lib-python/2.7/test/test_set.py b/lib-python/2.7/test/test_set.py
--- a/lib-python/2.7/test/test_set.py
+++ b/lib-python/2.7/test/test_set.py
@@ -309,6 +309,7 @@
             fo.close()
             test_support.unlink(test_support.TESTFN)
 
+    @test_support.impl_detail(pypy=False)
     def test_do_not_rehash_dict_keys(self):
         n = 10
         d = dict.fromkeys(map(HashCountingInt, xrange(n)))
@@ -559,6 +560,7 @@
         p = weakref.proxy(s)
         self.assertEqual(str(p), str(s))
         s = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, str, p)
 
     # C API test only available in a debug build
@@ -590,6 +592,7 @@
         s.__init__(self.otherword)
         self.assertEqual(s, set(self.word))
 
+    @test_support.impl_detail()
     def test_singleton_empty_frozenset(self):
         f = frozenset()
         efs = [frozenset(), frozenset([]), frozenset(()), frozenset(''),
@@ -770,9 +773,10 @@
         for v in self.set:
             self.assertIn(v, self.values)
         setiter = iter(self.set)
-        # note: __length_hint__ is an internal undocumented API,
-        # don't rely on it in your own programs
-        self.assertEqual(setiter.__length_hint__(), len(self.set))
+        if test_support.check_impl_detail():
+            # note: __length_hint__ is an internal undocumented API,
+            # don't rely on it in your own programs
+            self.assertEqual(setiter.__length_hint__(), len(self.set))
 
     def test_pickling(self):
         p = pickle.dumps(self.set)
@@ -1564,7 +1568,7 @@
             for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint):
                 for g in (G, I, Ig, L, R):
                     expected = meth(data)
-                    actual = meth(G(data))
+                    actual = meth(g(data))
                     if isinstance(expected, bool):
                         self.assertEqual(actual, expected)
                     else:
diff --git a/lib-python/2.7/test/test_sets.py b/lib-python/2.7/test/test_sets.py
--- a/lib-python/2.7/test/test_sets.py
+++ b/lib-python/2.7/test/test_sets.py
@@ -686,7 +686,9 @@
         set_list = sorted(self.set)
         self.assertEqual(len(dup_list), len(set_list))
         for i, el in enumerate(dup_list):
-            self.assertIs(el, set_list[i])
+            # Object identity is not guarnteed for immutable objects, so we
+            # can't use assertIs here.
+            self.assertEqual(el, set_list[i])
 
     def test_deep_copy(self):
         dup = copy.deepcopy(self.set)
diff --git a/lib-python/2.7/test/test_site.py b/lib-python/2.7/test/test_site.py
--- a/lib-python/2.7/test/test_site.py
+++ b/lib-python/2.7/test/test_site.py
@@ -226,6 +226,10 @@
             self.assertEqual(len(dirs), 1)
             wanted = os.path.join('xoxo', 'Lib', 'site-packages')
             self.assertEqual(dirs[0], wanted)
+        elif '__pypy__' in sys.builtin_module_names:
+            self.assertEquals(len(dirs), 1)
+            wanted = os.path.join('xoxo', 'site-packages')
+            self.assertEquals(dirs[0], wanted)
         elif os.sep == '/':
             self.assertEqual(len(dirs), 2)
             wanted = os.path.join('xoxo', 'lib', 'python' + sys.version[:3],
diff --git a/lib-python/2.7/test/test_socket.py b/lib-python/2.7/test/test_socket.py
--- a/lib-python/2.7/test/test_socket.py
+++ b/lib-python/2.7/test/test_socket.py
@@ -252,6 +252,7 @@
         self.assertEqual(p.fileno(), s.fileno())
         s.close()
         s = None
+        test_support.gc_collect()
         try:
             p.fileno()
         except ReferenceError:
@@ -285,32 +286,34 @@
             s.sendto(u'\u2620', sockname)
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, sockname)
-        self.assertIn('not complex', str(cm.exception))
+        self.assertIn('complex', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', None)
-        self.assertIn('not NoneType', str(cm.exception))
+        self.assertIn('NoneType', str(cm.exception))
         # 3 args
         with self.assertRaises(UnicodeEncodeError):
             s.sendto(u'\u2620', 0, sockname)
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, 0, sockname)
-        self.assertIn('not complex', str(cm.exception))
+        self.assertIn('complex', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 0, None)
-        self.assertIn('not NoneType', str(cm.exception))
+        if test_support.check_impl_detail():
+            self.assertIn('not NoneType', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 'bar', sockname)
-        self.assertIn('an integer is required', str(cm.exception))
+        self.assertIn('integer', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', None, None)
-        self.assertIn('an integer is required', str(cm.exception))
+        if test_support.check_impl_detail():
+            self.assertIn('an integer is required', str(cm.exception))
         # wrong number of args
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo')
-        self.assertIn('(1 given)', str(cm.exception))
+        self.assertIn(' given)', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 0, sockname, 4)
-        self.assertIn('(4 given)', str(cm.exception))
+        self.assertIn(' given)', str(cm.exception))
 
 
     def testCrucialConstants(self):
@@ -385,10 +388,10 @@
             socket.htonl(k)
             socket.htons(k)
         for k in bad_values:
-            self.assertRaises(OverflowError, socket.ntohl, k)
-            self.assertRaises(OverflowError, socket.ntohs, k)
-            self.assertRaises(OverflowError, socket.htonl, k)
-            self.assertRaises(OverflowError, socket.htons, k)
+            self.assertRaises((OverflowError, ValueError), socket.ntohl, k)
+            self.assertRaises((OverflowError, ValueError), socket.ntohs, k)
+            self.assertRaises((OverflowError, ValueError), socket.htonl, k)
+            self.assertRaises((OverflowError, ValueError), socket.htons, k)
 
     def testGetServBy(self):
         eq = self.assertEqual
@@ -428,8 +431,8 @@
         if udpport is not None:
             eq(socket.getservbyport(udpport, 'udp'), service)
         # Make sure getservbyport does not accept out of range ports.
-        self.assertRaises(OverflowError, socket.getservbyport, -1)
-        self.assertRaises(OverflowError, socket.getservbyport, 65536)
+        self.assertRaises((OverflowError, ValueError), socket.getservbyport, -1)
+        self.assertRaises((OverflowError, ValueError), socket.getservbyport, 65536)
 
     def testDefaultTimeout(self):
         # Testing default timeout
@@ -608,8 +611,8 @@
         neg_port = port - 65536
         sock = socket.socket()
         try:
-            self.assertRaises(OverflowError, sock.bind, (host, big_port))
-            self.assertRaises(OverflowError, sock.bind, (host, neg_port))
+            self.assertRaises((OverflowError, ValueError), sock.bind, (host, big_port))
+            self.assertRaises((OverflowError, ValueError), sock.bind, (host, neg_port))
             sock.bind((host, port))
         finally:
             sock.close()
@@ -1309,6 +1312,7 @@
             closed = False
             def flush(self): pass
             def close(self): self.closed = True
+            def _decref_socketios(self): pass
 
         # must not close unless we request it: the original use of _fileobject
         # by module socket requires that the underlying socket not be closed until
diff --git a/lib-python/2.7/test/test_sort.py b/lib-python/2.7/test/test_sort.py
--- a/lib-python/2.7/test/test_sort.py
+++ b/lib-python/2.7/test/test_sort.py
@@ -140,7 +140,10 @@
                 return random.random() < 0.5
 
         L = [C() for i in range(50)]
-        self.assertRaises(ValueError, L.sort)
+        try:
+            L.sort()
+        except ValueError:
+            pass
 
     def test_cmpNone(self):
         # Testing None as a comparison function.
@@ -150,8 +153,10 @@
         L.sort(None)
         self.assertEqual(L, range(50))
 
+    @test_support.impl_detail(pypy=False)
     def test_undetected_mutation(self):
         # Python 2.4a1 did not always detect mutation
+        # So does pypy...
         memorywaster = []
         for i in range(20):
             def mutating_cmp(x, y):
@@ -226,7 +231,10 @@
             def __del__(self):
                 del data[:]
                 data[:] = range(20)
-        self.assertRaises(ValueError, data.sort, key=SortKiller)
+        try:
+            data.sort(key=SortKiller)
+        except ValueError:
+            pass
 
     def test_key_with_mutating_del_and_exception(self):
         data = range(10)
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -881,6 +881,8 @@
                 c = socket.socket()
                 c.connect((HOST, port))
                 listener_gone.wait()
+                # XXX why is it necessary?
+                test_support.gc_collect()
                 try:
                     ssl_sock = ssl.wrap_socket(c)
                 except IOError:
@@ -1330,10 +1332,8 @@
 
 def test_main(verbose=False):
     global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT
-    CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir,
-                            "keycert.pem")
-    SVN_PYTHON_ORG_ROOT_CERT = os.path.join(
-        os.path.dirname(__file__) or os.curdir,
+    CERTFILE = test_support.findfile("keycert.pem")
+    SVN_PYTHON_ORG_ROOT_CERT = test_support.findfile(
         "https_svn_python_org_root.pem")
 
     if (not os.path.exists(CERTFILE) or
diff --git a/lib-python/2.7/test/test_str.py b/lib-python/2.7/test/test_str.py
--- a/lib-python/2.7/test/test_str.py
+++ b/lib-python/2.7/test/test_str.py
@@ -422,10 +422,11 @@
         for meth in ('foo'.startswith, 'foo'.endswith):
             with self.assertRaises(TypeError) as cm:
                 meth(['f'])
-            exc = str(cm.exception)
-            self.assertIn('unicode', exc)
-            self.assertIn('str', exc)
-            self.assertIn('tuple', exc)
+            if test_support.check_impl_detail():
+                exc = str(cm.exception)
+                self.assertIn('unicode', exc)
+                self.assertIn('str', exc)
+                self.assertIn('tuple', exc)
 
 def test_main():
     test_support.run_unittest(StrTest)
diff --git a/lib-python/2.7/test/test_struct.py b/lib-python/2.7/test/test_struct.py
--- a/lib-python/2.7/test/test_struct.py
+++ b/lib-python/2.7/test/test_struct.py
@@ -535,7 +535,8 @@
 
     @unittest.skipUnless(IS32BIT, "Specific to 32bit machines")
     def test_crasher(self):
-        self.assertRaises(MemoryError, struct.pack, "357913941c", "a")
+        self.assertRaises((MemoryError, struct.error), struct.pack,
+                          "357913941c", "a")
 
     def test_count_overflow(self):
         hugecount = '{}b'.format(sys.maxsize+1)
diff --git a/lib-python/2.7/test/test_subprocess.py b/lib-python/2.7/test/test_subprocess.py
--- a/lib-python/2.7/test/test_subprocess.py
+++ b/lib-python/2.7/test/test_subprocess.py
@@ -16,11 +16,11 @@
 # Depends on the following external programs: Python
 #
 
-if mswindows:
-    SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
-                                                'os.O_BINARY);')
-else:
-    SETBINARY = ''
+#if mswindows:
+#    SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
+#                                                'os.O_BINARY);')
+#else:
+#    SETBINARY = ''
 
 
 try:
@@ -420,8 +420,9 @@
         self.assertStderrEqual(stderr, "")
 
     def test_universal_newlines(self):
-        p = subprocess.Popen([sys.executable, "-c",
-                          'import sys,os;' + SETBINARY +
+        # NB. replaced SETBINARY with the -u flag
+        p = subprocess.Popen([sys.executable, "-u", "-c",
+                          'import sys,os;' + #SETBINARY +
                           'sys.stdout.write("line1\\n");'
                           'sys.stdout.flush();'
                           'sys.stdout.write("line2\\r");'
@@ -448,8 +449,9 @@
 
     def test_universal_newlines_communicate(self):
         # universal newlines through communicate()
-        p = subprocess.Popen([sys.executable, "-c",
-                          'import sys,os;' + SETBINARY +
+        # NB. replaced SETBINARY with the -u flag
+        p = subprocess.Popen([sys.executable, "-u", "-c",
+                          'import sys,os;' + #SETBINARY +
                           'sys.stdout.write("line1\\n");'
                           'sys.stdout.flush();'
                           'sys.stdout.write("line2\\r");'
diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py
--- a/lib-python/2.7/test/test_support.py
+++ b/lib-python/2.7/test/test_support.py
@@ -431,16 +431,20 @@
             rmtree(name)
 
 
-def findfile(file, here=__file__, subdir=None):
+def findfile(file, here=None, subdir=None):
     """Try to find a file on sys.path and the working directory.  If it is not
     found the argument passed to the function is returned (this does not
     necessarily signal failure; could still be the legitimate path)."""
+    import test
     if os.path.isabs(file):
         return file
     if subdir is not None:
         file = os.path.join(subdir, file)
     path = sys.path
-    path = [os.path.dirname(here)] + path
+    if here is None:
+        path = test.__path__ + path
+    else:
+        path = [os.path.dirname(here)] + path
     for dn in path:
         fn = os.path.join(dn, file)
         if os.path.exists(fn): return fn
@@ -1050,15 +1054,33 @@
     guards, default = _parse_guards(guards)
     return guards.get(platform.python_implementation().lower(), default)
 
+# ----------------------------------
+# PyPy extension: you can run::
+#     python ..../test_foo.py --pdb
+# to get a pdb prompt in case of exceptions
 
+ResultClass = unittest.TextTestRunner.resultclass
+
+class TestResultWithPdb(ResultClass):
+
+    def addError(self, testcase, exc_info):
+        ResultClass.addError(self, testcase, exc_info)
+        if '--pdb' in sys.argv:
+            import pdb, traceback
+            traceback.print_tb(exc_info[2])
+            pdb.post_mortem(exc_info[2])
+
+# ----------------------------------
 
 def _run_suite(suite):
     """Run tests from a unittest.TestSuite-derived class."""
     if verbose:
-        runner = unittest.TextTestRunner(sys.stdout, verbosity=2)
+        runner = unittest.TextTestRunner(sys.stdout, verbosity=2,
+                                         resultclass=TestResultWithPdb)
     else:
         runner = BasicTestRunner()
 
+
     result = runner.run(suite)
     if not result.wasSuccessful():
         if len(result.errors) == 1 and not result.failures:
@@ -1071,6 +1093,34 @@
                 err += "; run in verbose mode for details"
         raise TestFailed(err)
 
+# ----------------------------------
+# PyPy extension: you can run::
+#     python ..../test_foo.py --filter bar
+# to run only the test cases whose name contains bar
+
+def filter_maybe(suite):
+    try:
+        i = sys.argv.index('--filter')
+        filter = sys.argv[i+1]
+    except (ValueError, IndexError):
+        return suite
+    tests = []
+    for test in linearize_suite(suite):
+        if filter in test._testMethodName:
+            tests.append(test)
+    return unittest.TestSuite(tests)
+
+def linearize_suite(suite_or_test):
+    try:
+        it = iter(suite_or_test)
+    except TypeError:
+        yield suite_or_test
+        return
+    for subsuite in it:
+        for item in linearize_suite(subsuite):
+            yield item
+
+# ----------------------------------
 
 def run_unittest(*classes):
     """Run tests from unittest.TestCase-derived classes."""
@@ -1086,6 +1136,7 @@
             suite.addTest(cls)
         else:
             suite.addTest(unittest.makeSuite(cls))
+    suite = filter_maybe(suite)
     _run_suite(suite)
 
 
diff --git a/lib-python/2.7/test/test_syntax.py b/lib-python/2.7/test/test_syntax.py
--- a/lib-python/2.7/test/test_syntax.py
+++ b/lib-python/2.7/test/test_syntax.py
@@ -5,7 +5,8 @@
 >>> def f(x):
 ...     global x
 Traceback (most recent call last):
-SyntaxError: name 'x' is local and global (<doctest test.test_syntax[0]>, line 1)
+  File "<doctest test.test_syntax[0]>", line 1
+SyntaxError: name 'x' is local and global
 
 The tests are all raise SyntaxErrors.  They were created by checking
 each C call that raises SyntaxError.  There are several modules that
@@ -375,7 +376,7 @@
 In 2.5 there was a missing exception and an assert was triggered in a debug
 build.  The number of blocks must be greater than CO_MAXBLOCKS.  SF #1565514
 
-   >>> while 1:
+   >>> while 1: # doctest:+SKIP
    ...  while 2:
    ...   while 3:
    ...    while 4:
diff --git a/lib-python/2.7/test/test_sys.py b/lib-python/2.7/test/test_sys.py
--- a/lib-python/2.7/test/test_sys.py
+++ b/lib-python/2.7/test/test_sys.py
@@ -264,6 +264,7 @@
             self.assertEqual(sys.getdlopenflags(), oldflags+1)
             sys.setdlopenflags(oldflags)
 
+    @test.test_support.impl_detail("reference counting")
     def test_refcount(self):
         # n here must be a global in order for this test to pass while
         # tracing with a python function.  Tracing calls PyFrame_FastToLocals
@@ -287,7 +288,7 @@
             is sys._getframe().f_code
         )
 
-    # sys._current_frames() is a CPython-only gimmick.
+    @test.test_support.impl_detail("current_frames")
     def test_current_frames(self):
         have_threads = True
         try:
@@ -383,7 +384,10 @@
         self.assertEqual(len(sys.float_info), 11)
         self.assertEqual(sys.float_info.radix, 2)
         self.assertEqual(len(sys.long_info), 2)
-        self.assertTrue(sys.long_info.bits_per_digit % 5 == 0)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertTrue(sys.long_info.bits_per_digit % 5 == 0)
+        else:
+            self.assertTrue(sys.long_info.bits_per_digit >= 1)
         self.assertTrue(sys.long_info.sizeof_digit >= 1)
         self.assertEqual(type(sys.long_info.bits_per_digit), int)
         self.assertEqual(type(sys.long_info.sizeof_digit), int)
@@ -432,6 +436,7 @@
             self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
         self.assertTrue(repr(sys.flags))
 
+    @test.test_support.impl_detail("sys._clear_type_cache")
     def test_clear_type_cache(self):
         sys._clear_type_cache()
 
@@ -473,6 +478,7 @@
         p.wait()
         self.assertIn(executable, ["''", repr(sys.executable)])
 
+ at unittest.skipUnless(test.test_support.check_impl_detail(), "sys.getsizeof()")
 class SizeofTest(unittest.TestCase):
 
     TPFLAGS_HAVE_GC = 1<<14
diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py
--- a/lib-python/2.7/test/test_sys_settrace.py
+++ b/lib-python/2.7/test/test_sys_settrace.py
@@ -213,12 +213,16 @@
         "finally"
 def generator_example():
     # any() will leave the generator before its end
-    x = any(generator_function())
+    x = any(generator_function()); gc.collect()
 
     # the following lines were not traced
     for x in range(10):
         y = x
 
+# On CPython, when the generator is decref'ed to zero, we see the trace
+# for the "finally:" portion.  On PyPy, we don't see it before the next
+# garbage collection.  That's why we put gc.collect() on the same line above.
+
 generator_example.events = ([(0, 'call'),
                              (2, 'line'),
                              (-6, 'call'),
@@ -282,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
@@ -323,17 +327,24 @@
         self.run_test(tighterloop_example)
 
     def test_13_genexp(self):
-        self.run_test(generator_example)
-        # issue1265: if the trace function contains a generator,
-        # and if the traced function contains another generator
-        # that is not completely exhausted, the trace stopped.
-        # Worse: the 'finally' clause was not invoked.
-        tracer = Tracer()
-        sys.settrace(tracer.traceWithGenexp)
-        generator_example()
-        sys.settrace(None)
-        self.compare_events(generator_example.__code__.co_firstlineno,
-                            tracer.events, generator_example.events)
+        if self.using_gc:
+            test_support.gc_collect()
+            gc.enable()
+        try:
+            self.run_test(generator_example)
+            # issue1265: if the trace function contains a generator,
+            # and if the traced function contains another generator
+            # that is not completely exhausted, the trace stopped.
+            # Worse: the 'finally' clause was not invoked.
+            tracer = Tracer()
+            sys.settrace(tracer.traceWithGenexp)
+            generator_example()
+            sys.settrace(None)
+            self.compare_events(generator_example.__code__.co_firstlineno,
+                                tracer.events, generator_example.events)
+        finally:
+            if self.using_gc:
+                gc.disable()
 
     def test_14_onliner_if(self):
         def onliners():
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -209,13 +209,22 @@
 
         self.assertEqual(get_platform(), 'macosx-10.4-fat64')
 
-        for arch in ('ppc', 'i386', 'x86_64', 'ppc64'):
+        for arch in ('ppc', 'i386', 'ppc64', 'x86_64'):
             get_config_vars()['CFLAGS'] = ('-arch %s -isysroot '
                                            '/Developer/SDKs/MacOSX10.4u.sdk  '
                                            '-fno-strict-aliasing -fno-common '
                                            '-dynamic -DNDEBUG -g -O3'%(arch,))
 
             self.assertEqual(get_platform(), 'macosx-10.4-%s'%(arch,))
+        
+        # macosx with ARCHFLAGS set and empty _CONFIG_VARS
+        os.environ['ARCHFLAGS'] = '-arch i386'
+        sysconfig._CONFIG_VARS = None
+        
+        # this will attempt to recreate the _CONFIG_VARS based on environment 
+        # variables; used to check a problem with the PyPy's _init_posix
+        # implementation; see: issue 705
+        get_config_vars() 
 
         # linux debian sarge
         os.name = 'posix'
@@ -235,7 +244,7 @@
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
-                  'posix_home', 'posix_prefix', 'posix_user')
+                  'posix_home', 'posix_prefix', 'posix_user', 'pypy')
         self.assertEqual(get_scheme_names(), wanted)
 
     def test_symlink(self):
diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/2.7/test/test_tarfile.py
--- a/lib-python/2.7/test/test_tarfile.py
+++ b/lib-python/2.7/test/test_tarfile.py
@@ -169,6 +169,7 @@
         except tarfile.ReadError:
             self.fail("tarfile.open() failed on empty archive")
         self.assertListEqual(tar.getmembers(), [])
+        tar.close()
 
     def test_null_tarfile(self):
         # Test for issue6123: Allow opening empty archives.
@@ -207,16 +208,21 @@
         fobj = open(self.tarname, "rb")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, os.path.abspath(fobj.name))
+        tar.close()
 
     def test_no_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self.assertRaises(AttributeError, getattr, fobj, "name")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, None)
 
     def test_empty_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         fobj.name = ""
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
@@ -515,6 +521,7 @@
         self.tar = tarfile.open(self.tarname, mode=self.mode, encoding="iso8859-1")
         tarinfo = self.tar.getmember("pax/umlauts-&#65533;&#65533;&#65533;&#65533;&#65533;&#65533;&#65533;")
         self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self.tar.close()
 
 
 class LongnameTest(ReadTest):
@@ -675,6 +682,7 @@
             tar = tarfile.open(tmpname, self.mode)
             tarinfo = tar.gettarinfo(path)
             self.assertEqual(tarinfo.size, 0)
+            tar.close()
         finally:
             os.rmdir(path)
 
@@ -692,6 +700,7 @@
                 tar.gettarinfo(target)
                 tarinfo = tar.gettarinfo(link)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(target)
                 os.remove(link)
@@ -704,6 +713,7 @@
                 tar = tarfile.open(tmpname, self.mode)
                 tarinfo = tar.gettarinfo(path)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(path)
 
@@ -722,6 +732,7 @@
         tar.add(dstname)
         os.chdir(cwd)
         self.assertTrue(tar.getnames() == [], "added the archive to itself")
+        tar.close()
 
     def test_exclude(self):
         tempdir = os.path.join(TEMPDIR, "exclude")
@@ -742,6 +753,7 @@
             tar = tarfile.open(tmpname, "r")
             self.assertEqual(len(tar.getmembers()), 1)
             self.assertEqual(tar.getnames()[0], "empty_dir")
+            tar.close()
         finally:
             shutil.rmtree(tempdir)
 
@@ -947,7 +959,9 @@
             fobj.close()
         elif self.mode.endswith("bz2"):
             dec = bz2.BZ2Decompressor()
-            data = open(tmpname, "rb").read()
+            f = open(tmpname, "rb")
+            data = f.read()
+            f.close()
             data = dec.decompress(data)
             self.assertTrue(len(dec.unused_data) == 0,
                     "found trailing data")
@@ -1026,6 +1040,7 @@
                 "unable to read longname member")
         self.assertEqual(tarinfo.linkname, member.linkname,
                 "unable to read longname member")
+        tar.close()
 
     def test_longname_1023(self):
         self._test(("longnam/" * 127) + "longnam")
@@ -1118,6 +1133,7 @@
         else:
             n = tar.getmembers()[0].name
             self.assertTrue(name == n, "PAX longname creation failed")
+        tar.close()
 
     def test_pax_global_header(self):
         pax_headers = {
@@ -1146,6 +1162,7 @@
                     tarfile.PAX_NUMBER_FIELDS[key](val)
                 except (TypeError, ValueError):
                     self.fail("unable to convert pax header field")
+        tar.close()
 
     def test_pax_extended_header(self):
         # The fields from the pax header have priority over the
@@ -1165,6 +1182,7 @@
         self.assertEqual(t.pax_headers, pax_headers)
         self.assertEqual(t.name, "foo")
         self.assertEqual(t.uid, 123)
+        tar.close()
 
 
 class UstarUnicodeTest(unittest.TestCase):
@@ -1208,6 +1226,7 @@
         tarinfo.name = "foo"
         tarinfo.uname = u"&#65533;&#65533;&#65533;"
         self.assertRaises(UnicodeError, tar.addfile, tarinfo)
+        tar.close()
 
     def test_unicode_argument(self):
         tar = tarfile.open(tarname, "r", encoding="iso8859-1", errors="strict")
@@ -1262,6 +1281,7 @@
             tar = tarfile.open(tmpname, format=self.format, encoding="ascii",
                     errors=handler)
             self.assertEqual(tar.getnames()[0], name)
+            tar.close()
 
         self.assertRaises(UnicodeError, tarfile.open, tmpname,
                 encoding="ascii", errors="strict")
@@ -1274,6 +1294,7 @@
         tar = tarfile.open(tmpname, format=self.format, encoding="iso8859-1",
                 errors="utf-8")
         self.assertEqual(tar.getnames()[0], "&#65533;&#65533;&#65533;/" + u"&#65533;".encode("utf8"))
+        tar.close()
 
 
 class AppendTest(unittest.TestCase):
@@ -1301,6 +1322,7 @@
     def _test(self, names=["bar"], fileobj=None):
         tar = tarfile.open(self.tarname, fileobj=fileobj)
         self.assertEqual(tar.getnames(), names)
+        tar.close()
 
     def test_non_existing(self):
         self._add_testfile()
@@ -1319,7 +1341,9 @@
 
     def test_fileobj(self):
         self._create_testtar()
-        data = open(self.tarname).read()
+        f = open(self.tarname)
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self._add_testfile(fobj)
         fobj.seek(0)
@@ -1345,7 +1369,9 @@
     # Append mode is supposed to fail if the tarfile to append to
     # does not end with a zero block.
     def _test_error(self, data):
-        open(self.tarname, "wb").write(data)
+        f = open(self.tarname, "wb")
+        f.write(data)
+        f.close()
         self.assertRaises(tarfile.ReadError, self._add_testfile)
 
     def test_null(self):
diff --git a/lib-python/2.7/test/test_tempfile.py b/lib-python/2.7/test/test_tempfile.py
--- a/lib-python/2.7/test/test_tempfile.py
+++ b/lib-python/2.7/test/test_tempfile.py
@@ -23,8 +23,8 @@
 
 # TEST_FILES may need to be tweaked for systems depending on the maximum
 # number of files that can be opened at one time (see ulimit -n)
-if sys.platform in ('openbsd3', 'openbsd4'):
-    TEST_FILES = 48
+if sys.platform.startswith("openbsd"):
+    TEST_FILES = 64 # ulimit -n defaults to 128 for normal users
 else:
     TEST_FILES = 100
 
@@ -244,6 +244,7 @@
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir).write("blat")
+            test_support.gc_collect()
         finally:
             os.rmdir(dir)
 
@@ -528,12 +529,15 @@
         self.do_create(suf="b")
         self.do_create(pre="a", suf="b")
         self.do_create(pre="aa", suf=".txt")
+        test_support.gc_collect()
 
     def test_many(self):
         # mktemp can choose many usable file names (stochastic)
         extant = range(TEST_FILES)
         for i in extant:
             extant[i] = self.do_create(pre="aa")
+        del extant
+        test_support.gc_collect()
 
 ##     def test_warning(self):
 ##         # mktemp issues a warning when used
diff --git a/lib-python/2.7/test/test_thread.py b/lib-python/2.7/test/test_thread.py
--- a/lib-python/2.7/test/test_thread.py
+++ b/lib-python/2.7/test/test_thread.py
@@ -128,6 +128,7 @@
         del task
         while not done:
             time.sleep(0.01)
+            test_support.gc_collect()
         self.assertEqual(thread._count(), orig)
 

From noreply at buildbot.pypy.org  Sat Apr 14 12:05:01 2012
From: noreply at buildbot.pypy.org (mikefc)
Date: Sat, 14 Apr 2012 12:05:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: swapaxes for numpypy
Message-ID: <20120414100501.62A1882F4E@wyvern.cs.uni-duesseldorf.de>

Author: Michael Cheng <coolbutuseless at gmail.com>
Branch: 
Changeset: r54360:886d352cf776
Date: 2012-04-14 20:04 +1000
http://bitbucket.org/pypy/pypy/changeset/886d352cf776/

Log:	swapaxes for numpypy

diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -513,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1435,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1410,6 +1410,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()

From noreply at buildbot.pypy.org  Sat Apr 14 14:32:19 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 14:32:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: escape slashes in udir basenames with --
Message-ID: <20120414123219.E345082F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54361:4b487dad7abe
Date: 2012-04-14 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/4b487dad7abe/

Log:	escape slashes in udir basenames with --

diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):

From noreply at buildbot.pypy.org  Sat Apr 14 14:32:21 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 14:32:21 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification: merge from default
Message-ID: <20120414123221.39B0C82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification
Changeset: r54362:ccfea861d0e3
Date: 2012-04-14 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/ccfea861d0e3/

Log:	merge from default

diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:

From noreply at buildbot.pypy.org  Sat Apr 14 14:32:22 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 14:32:22 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification/py3k: merge from
	stdlib-unification
Message-ID: <20120414123222.8ECC582F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification/py3k
Changeset: r54363:4e795faa4933
Date: 2012-04-14 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/4e795faa4933/

Log:	merge from stdlib-unification

diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:

From noreply at buildbot.pypy.org  Sat Apr 14 14:32:23 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 14 Apr 2012 14:32:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120414123223.D14B082F4E@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54364:ee5f8c2f4686
Date: 2012-04-14 14:31 +0200
http://bitbucket.org/pypy/pypy/changeset/ee5f8c2f4686/

Log:	merge

diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -513,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1435,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1410,6 +1410,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()

From noreply at buildbot.pypy.org  Sat Apr 14 18:37:58 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 14 Apr 2012 18:37:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: speed up jitted lsprof on builtin calls
	significantly
Message-ID: <20120414163758.67C4382F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54365:822d774c2560
Date: 2012-04-14 18:23 +0200
http://bitbucket.org/pypy/pypy/changeset/822d774c2560/

Log:	speed up jitted lsprof on builtin calls significantly

diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function

From noreply at buildbot.pypy.org  Sat Apr 14 18:38:19 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 14 Apr 2012 18:38:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal,
 arigo) heuristic for unrolling loops, unroll a bit more
Message-ID: <20120414163819.80D0482F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54366:c85c18524c4f
Date: 2012-04-14 18:37 +0200
http://bitbucket.org/pypy/pypy/changeset/c85c18524c4f/

Log:	(fijal, arigo) heuristic for unrolling loops, unroll a bit more

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -204,6 +204,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.argtype(0)
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:

From noreply at buildbot.pypy.org  Sat Apr 14 18:40:49 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 14 Apr 2012 18:40:49 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: a direct test for the return
Message-ID: <20120414164049.9BC4082F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54367:5a93314d513e
Date: 2012-04-14 18:40 +0200
http://bitbucket.org/pypy/pypy/changeset/5a93314d513e/

Log:	a direct test for the return

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -333,7 +333,8 @@
             # external factors, like trackgcroot or the handling of the write
             # barrier.  Implemented by still using 'minsize' for the nursery
             # size (needed to handle mallocs just below 'large_objects') but
-            # hacking at the current nursery position in collect_and_reserve().
+            # hacking at the current nursery position in
+            # maybe_collect_and_reserve().
             if newsize <= 0:
                 newsize = 4*1024*1024   # fixed to 4MB by default
                 #        (it was env.estimate_best_nursery_size())
@@ -502,7 +503,7 @@
             result = self.nursery_free
             self.nursery_free = result + totalsize
             if self.nursery_free > self.nursery_top:
-                result = self.collect_and_reserve(totalsize)
+                result = self.maybe_collect_and_reserve(totalsize)
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
@@ -561,7 +562,7 @@
             result = self.nursery_free
             self.nursery_free = result + totalsize
             if self.nursery_free > self.nursery_top:
-                result = self.collect_and_reserve(totalsize)
+                result = self.maybe_collect_and_reserve(totalsize)
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
@@ -580,7 +581,7 @@
         if gen > 0:
             self.major_collection()
 
-    def collect_and_reserve(self, totalsize):
+    def maybe_collect_and_reserve(self, totalsize):
         """To call when nursery_free overflows nursery_top.
         Do a minor collection, and possibly also a major collection,
         and finally reserve 'totalsize' bytes at the start of the
@@ -595,7 +596,6 @@
                 self.nursery_free = self.nursery_free + cur_obj_size
                 self.nursery_top = self.nursery_barriers.popleft()
             if self.nursery_free + totalsize <= self.nursery_top:
-                llarena.arena_reserve(self.nursery_free, totalsize)
                 res = self.nursery_free
                 self.nursery_free = res + totalsize
                 return res
@@ -622,7 +622,7 @@
                 self.nursery_free = self.nursery_top - self.debug_tiny_nursery
         #
         return result
-    collect_and_reserve._dont_inline_ = True
+    maybe_collect_and_reserve._dont_inline_ = True
 
 
     def external_malloc(self, typeid, length, can_make_young=True):
@@ -789,10 +789,12 @@
                 not self.header(obj).tid & GCFLAG_PINNED)
 
     def pin(self, obj):
-        self.header(obj).tid |= GCFLAG_PINNED
+        if self.is_in_nursery(obj):
+            self.header(obj).tid |= GCFLAG_PINNED
 
     def unpin(self, obj):
-        self.header(obj).tid &= ~GCFLAG_PINNED
+        if self.is_in_nursery(obj):
+            self.header(obj).tid &= ~GCFLAG_PINNED
 
     def shrink_array(self, obj, smallerlength):
         #
diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -522,6 +522,19 @@
             self.stackroots.pop()
     test_card_marker.GC_PARAMS = {"card_page_indices": 4}
 
+    def test_pin_1(self):
+        self.malloc(S)
+        s2 = self.malloc(S)
+        self.gc.pin(llmemory.cast_ptr_to_adr(s2))
+        self.stackroots.append(s2)
+        self.gc.minor_collection()
+        self.stackroots.pop()
+        self.malloc(S)
+        one = self.gc.nursery_free
+        self.malloc(S)
+        two = self.gc.nursery_free
+        assert one != two
+
     def test_writebarrier_before_copy(self):
         from pypy.rpython.memory.gc import minimark
         largeobj_size =  self.gc.nonlarge_max + 1

From noreply at buildbot.pypy.org  Sat Apr 14 18:51:15 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 14 Apr 2012 18:51:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: hopefully a better specialization
Message-ID: <20120414165115.1A89182F4E@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54368:57f6dff7fb22
Date: 2012-04-14 18:50 +0200
http://bitbucket.org/pypy/pypy/changeset/57f6dff7fb22/

Log:	hopefully a better specialization

diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -206,7 +206,7 @@
 
 LIST_CUTOFF = 2
 
- at specialize.argtype(0)
+ at specialize.call_location()
 def loop_unrolling_heuristic(lst, size):
     """ In which cases iterating over items of lst can be unrolled
     """

From noreply at buildbot.pypy.org  Sat Apr 14 19:15:39 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 14 Apr 2012 19:15:39 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: - unwrite the specific details
 about nested transactions, to be
Message-ID: <20120414171539.39AAA82F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4192:bb8a0daf30d6
Date: 2012-04-14 19:15 +0200
http://bitbucket.org/pypy/extradoc/changeset/bb8a0daf30d6/

Log:	- unwrite the specific details about nested transactions, to be
	considered later

	- found out a more flexible way to ensure ordering when needed

diff --git a/planning/stm.txt b/planning/stm.txt
--- a/planning/stm.txt
+++ b/planning/stm.txt
@@ -11,28 +11,17 @@
 Python Interface
 ----------------
 
-Planned interface refactoring: integrate the states of "running
-transactionally" and "not running transactionally".
+Planned interface refactorings:
 
-Running normally the program is done in one big top-level transaction.
-The "transaction" module should be provisionally renamed to "ame".  At
-any point (including in sub-transactions) we can create an instance of
-'ame.RandomOrder()', and call methods on it:
+* inspired by "concurrent.futures" from Python 3.2: have
+  transaction.add() return a Future instance with a method result().
+  If called inside another transaction, it will suspend it until the
+  result is available, i.e. until the first transaction commits.  Can
+  be used even if the result is not used, just to ensure some ordering.
 
-  - 'add(callable, *args, **kwds)'
-  - 'run()'
-
-When run() is called the current transaction is suspended and all
-added transactions are run (including the ones added by them).  These
-transactions commit into the current transaction.  When run() returns,
-the parent transaction continues running.  If at any point any
-transaction becomes inevitable, its parent transaction does the same,
-and so on.
-
-This should give a model that composes more naturally than the current
-one.  Also, it opens the door to other 'ame' objects; e.g. an
-'ame.Sequential()' would run the subtransactions in the same order as
-they have been added.
+* (later, maybe) allow nested transactions: either by calling
+  transaction.run() inside transactions too, or with actual objects
+  that store collections of transactions.
 
 
 Overview of the GC

From noreply at buildbot.pypy.org  Sat Apr 14 19:21:11 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 14 Apr 2012 19:21:11 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Grr
Message-ID: <20120414172111.67AC182F4E@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4193:80057c82b30f
Date: 2012-04-14 19:20 +0200
http://bitbucket.org/pypy/extradoc/changeset/80057c82b30f/

Log:	Grr

diff --git a/planning/stm.txt b/planning/stm.txt
--- a/planning/stm.txt
+++ b/planning/stm.txt
@@ -18,6 +18,7 @@
   If called inside another transaction, it will suspend it until the
   result is available, i.e. until the first transaction commits.  Can
   be used even if the result is not used, just to ensure some ordering.
+  XXX but can that be emulated in transaction.py???
 
 * (later, maybe) allow nested transactions: either by calling
   transaction.run() inside transactions too, or with actual objects

From noreply at buildbot.pypy.org  Sun Apr 15 15:09:58 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 15 Apr 2012 15:09:58 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Progress
Message-ID: <20120415130958.632B1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54369:ddb7a8bdd471
Date: 2012-04-15 15:09 +0200
http://bitbucket.org/pypy/pypy/changeset/ddb7a8bdd471/

Log:	Progress

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -44,6 +44,7 @@
 GCFLAG_FIXED_HASH = first_gcflag << 3
 GCFLAG_WEAKREF    = first_gcflag << 4
 GCFLAG_VISITED    = first_gcflag << 5
+GCFLAG_MOVED      = first_gcflag << 6
 
 
 def always_inline(fn):
diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -27,27 +27,22 @@
         self.chained_list = NULL
         self.special_stack = self.gc.AddressStack()
 
-    def delete(self):
-        self.special_stack.delete()
-        free_non_gc_object(self)
+    def malloc_object(self, totalsize):
+        """Malloc.  You must also call add_regular() or add_special() later."""
+        adr1 = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize), 0)
+        llarena.arena_reserve(adr1, totalsize)
+        return adr1 + self.gc.gcheaderbuilder.size_gc_header
 
-    def malloc_regular(self, size):
-        """Malloc for an object where the 'version' field can be used
-        internally for a chained list."""
-        adr1 = llarena.arena_malloc(size, 0)
-        adr2 = adr1 + self.gc.gcheaderbuilder
-        hdr = llmemory.cast_adr_to_ptr(adr1, lltype.Ptr(self.gc.HDR))
+    def add_regular(self, obj):
+        """After malloc_object(), register the object in the internal chained
+        list.  For objects whose 'version' field is not otherwise needed."""
+        hdr = self.gc.header(obj)
         hdr.version = self.chained_list
-        self.chained_list = adr2
-        return adr2
+        self.chained_list = obj
 
-    def malloc_special(self, size):
-        """Malloc for an object where the 'version' field cannot be
-        used internally.  It's the rare case here."""
-        adr1 = llarena.arena_malloc(size, 0)
-        adr2 = adr1 + self.gc.gcheaderbuilder.size_gc_header
-        self.special_stack.append(adr2)
-        return adr2
+    def add_special(self, obj):
+        """After malloc_object(), register the object in a separate stack."""
+        self.special_stack.append(obj)
 
     def free_object(self, adr2):
         adr1 = adr2 - self.gc.gcheaderbuilder.size_gc_header
@@ -56,3 +51,18 @@
     def replace_special_stack(self, new_special_stack):
         self.special_stack.delete()
         self.special_stack = new_special_stack
+
+    def clear(self):
+        obj = self.chained_list
+        self.chained_list = NULL
+        while obj:
+            next = self.gc.header(obj).version
+            self.free_object(obj)
+            obj = next
+        s = self.special_stack
+        while s.non_empty():
+            self.free_object(s.pop())
+
+    def delete(self):
+        self.special_stack.delete()
+        free_non_gc_object(self)
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -8,6 +8,7 @@
 from pypy.rpython.memory.gc.stmgc import WORD, NULL
 from pypy.rpython.memory.gc.stmgc import always_inline, dont_inline
 from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED
+from pypy.rpython.memory.gc.stmgc import GCFLAG_MOVED
 
 
 class StmGCTLS(object):
@@ -19,7 +20,6 @@
     nontranslated_dict = {}
 
     def __init__(self, gc, in_main_thread):
-        from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
         self.gc = gc
         self.in_main_thread = in_main_thread
         self.stm_operations = self.gc.stm_operations
@@ -46,7 +46,7 @@
         #self.old_objects_with_weakrefs = NULL
         #
         # --- a thread-local allocator for the shared area
-        self.sharedarea_tls = StmGCThreadLocalAllocator(gc.sharedarea)
+        self.sharedarea_tls = None
         #
         self._register_with_C_code()
 
@@ -153,37 +153,41 @@
         #
         debug_start("gc-local")
         #
-        # Linked list of LOCAL objects pending a visit.  Note that no
-        # GLOBAL object can at any point contain a reference to a LOCAL
-        # object.
-        self.pending_list = NULL
+        # Move away the previous sharedarea_tls and start a new one.
+        from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
+        previous_sharedarea_tls = self.sharedarea_tls    # may be None
+        self.sharedarea_tls = StmGCThreadLocalAllocator(self.gc.sharedarea)
+        #
+        # List of LOCAL objects pending a visit.  Note that no GLOBAL
+        # object can at any point contain a reference to a LOCAL object.
+        self.pending = self.AddressStack()
         #
         # First, find the roots that point to LOCAL objects.  All YOUNG
         # (i.e. nursery) objects found are copied out of the nursery.
-        # All OLD objects found are flagged with GCFLAG_VISITED.  At this
-        # point, the content of the objects is not modified; the objects
-        # are merely added to the chained list 'pending_list'.
-        self.collect_roots_in_nursery()
+        # All OLD objects found are flagged with GCFLAG_VISITED.
+        # At this point, the content of the objects is not modified;
+        # they are simply added to 'pending'.
+        self.collect_roots_from_stack()
         #
         # Also find the roots that are the local copy of GCFLAG_WAS_COPIED
         # objects.
         self.collect_roots_from_tldict()
         #
-        # Now repeat following objects until 'pending_list' is empty.
-        self.collect_oldrefs_to_nursery()
+        # Now repeatedly follow objects until 'pending' is empty.
+        self.collect_flush_pending()
         #
         # Walk the list of LOCAL raw-malloced objects, and free them if
         # necessary.
         #self.free_local_rawmalloced_objects()
         #
-        # Ask the ArenaCollection to visit all objects.  Free the ones
-        # that have not been visited above, and reset GCFLAG_VISITED on
-        # the others.
-        self.ac.mass_free(self._free_if_unvisited)
+        # Visit all previous OLD objects.  Free the ones that have not been
+        # visited above, and reset GCFLAG_VISITED on the others.
+        if previous_sharedarea_tls is not None:
+            self.mass_free_old_local(previous_sharedarea_tls)
         #
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
-        llarena.arena_reset(self.nursery, self.nursery_size, 2)
+        llarena.arena_reset(self.nursery_start, self.nursery_size, 2)
         self.nursery_free = self.nursery_start
         #
         debug_stop("gc-local")
@@ -215,7 +219,7 @@
     def allocate_object_of_size(self, size):
         if not self.nursery_free:
             fatalerror("malloc in a non-main thread but outside a transaction")
-        if size > self.nursery_size // 8 * 7:
+        if llmemory.raw_malloc_usage(size) > self.nursery_size // 8 * 7:
             fatalerror("object too large to ever fit in the nursery")
         while True:
             self.local_collection()
@@ -225,6 +229,11 @@
                 continue         # try again
             return free
 
+    def is_in_nursery(self, addr):
+        ll_assert(llmemory.cast_adr_to_int(addr) & 1 == 0,
+                  "odd-valued (i.e. tagged) pointer unexpected here")
+        return self.nursery_start <= addr < self.nursery_top
+
     # ------------------------------------------------------------
 
     def _promote_locals_to_globals(self):
@@ -245,5 +254,102 @@
 ##            obj = hdr.version
 
     def _cleanup_state(self):
-        if self.rawmalloced_objects:
-            xxx     # free the rawmalloced_objects still around
+        #if self.rawmalloced_objects:
+        #    xxx     # free the rawmalloced_objects still around
+
+        # if we still have a StmGCThreadLocalAllocator, free the old unused
+        # local objects it still contains
+        if self.sharedarea_tls is not None:
+            self.sharedarea_tls.clear()
+            self.sharedarea_tls.delete()
+            self.sharedarea_tls = None
+
+
+    def collect_roots_from_stack(self):
+        self.gc.root_walker.walk_roots(
+            StmGCTLS._trace_drag_out1,  # stack roots of the current thread
+            None,                       # static in prebuilt non-gc
+            None,                       # static in prebuilt gc
+            self)
+
+    def _trace_drag_out1(self, root):
+        self._trace_drag_out(root, None)
+
+    def _trace_drag_out(self, root, ignored):
+        """Trace callback: 'root' is the address of some pointer.  If that
+        pointer points to a YOUNG object, allocate an OLD copy of it and
+        fix the pointer.  Also, add the object to 'pending_list', if it was
+        not done so far.
+        """
+        obj = root.address[0]
+        hdr = self.gc.header(obj)
+        #
+        # If 'obj' is not in the nursery, we set GCFLAG_VISITED
+        if not self.is_in_nursery(obj):
+            if hdr.tid & GCFLAG_VISITED == 0:
+                hdr.tid |= GCFLAG_VISITED
+                self.pending.append(obj)
+            return
+        #
+        # If 'obj' was already forwarded, change it to its forwarding address.
+        if hdr.tid & GCFLAG_MOVED:
+            root.address[0] = hdr.version
+            return
+        #
+        # First visit to 'obj': we must move this YOUNG obj out of the nursery.
+        size_gc_header = self.gc.gcheaderbuilder.size_gc_header
+        size = self.gc.get_size(obj)
+        totalsize = size_gc_header + size
+        #
+        # Common case: allocate a new nonmovable location for it.
+        newobj = self._malloc_out_of_nursery(totalsize)
+        #
+        # Copy it.  Note that references to other objects in the
+        # nursery are kept unchanged in this step.
+        llmemory.raw_memcopy(obj - size_gc_header,
+                             newobj - size_gc_header,
+                             totalsize)
+        #
+        # Set the YOUNG copy's GCFLAG_MOVED and set its version to
+        # point to the OLD copy.
+        hdr.tid |= GCFLAG_MOVED
+        hdr.version = newobj
+        #
+        # Change the original pointer to this object.
+        root.address[0] = newobj
+        #
+        # Add the newobj to the list 'pending', because it can contain
+        # further pointers to other young objects.  We will fix such
+        # references to point to the copy of the young objects when we
+        # walk 'pending_list'.
+        self.pending.append(newobj)
+
+    def _malloc_out_of_nursery(self, totalsize):
+        obj = self.sharedarea_tls.malloc_object(totalsize)
+        self.sharedarea_tls.add_regular(obj)
+        return obj
+
+    def collect_roots_from_tldict(self):
+        pass  # XXX
+
+    def collect_flush_pending(self):
+        # Follow the objects in the 'pending' stack and move the
+        # young objects they point to out of the nursery.
+        while self.pending.non_empty():
+            obj = self.pending.pop()
+            self.gc.trace(obj, self._trace_drag_out, None)
+
+    def mass_free_old_local(self, previous_sharedarea_tls):
+        obj = previous_sharedarea_tls.chained_list
+        previous_sharedarea_tls.delete()
+        while obj != NULL:
+            hdr = self.gc.header(obj)
+            next = hdr.version
+            if hdr.tid & GCFLAG_VISITED:
+                # survives: relink in the new sharedarea_tls
+                self.sharedarea_tls.add_regular(obj)
+            else:
+                # dies
+                self.sharedarea_tls.free_object(obj)
+            #
+            obj = next
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -1,10 +1,12 @@
 import py
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.memory.gc.stmtls import StmGCTLS, WORD
 from pypy.rpython.memory.gc.test.test_stmgc import StmGCTests
 from pypy.rpython.memory.support import get_address_stack, get_address_deque
+from pypy.rpython.memory.gcheader import GCHeaderBuilder
 
 
+NULL = llmemory.NULL
 S = lltype.GcStruct('S', ('a', lltype.Signed), ('b', lltype.Signed),
                          ('c', lltype.Signed))
 
@@ -18,6 +20,24 @@
 class FakeSharedArea:
     pass
 
+class FakeRootWalker:
+    def walk_roots(self, f1, f2, f3, arg):
+        if f1 is not None:
+            A = lltype.Array(llmemory.Address)
+            roots = lltype.malloc(A, len(self.current_stack), flavor='raw')
+            for i in range(len(self.current_stack)):
+                roots[i] = llmemory.cast_ptr_to_adr(self.current_stack[i])
+            for i in range(len(self.current_stack)):
+                root = lltype.direct_ptradd(lltype.direct_arrayitems(roots), i)
+                root = llmemory.cast_ptr_to_adr(root)
+                f1(arg, root)
+            for i in range(len(self.current_stack)):
+                P = lltype.typeOf(self.current_stack[i])
+                self.current_stack[i] = llmemory.cast_adr_to_ptr(roots[i], P)
+            lltype.free(roots, flavor='raw')
+        assert f2 is None
+        assert f3 is None
+
 class FakeGC:
     from pypy.rpython.memory.support import AddressDict, null_address_dict
     AddressStack = get_address_stack()
@@ -25,6 +45,28 @@
     nursery_size = 128
     stm_operations = FakeStmOperations()
     sharedarea = FakeSharedArea()
+    root_walker = FakeRootWalker()
+    HDR = lltype.Struct('header', ('tid', lltype.Signed),
+                                  ('version', llmemory.Address))
+    gcheaderbuilder = GCHeaderBuilder(HDR)
+
+    def header(self, addr):
+        addr -= self.gcheaderbuilder.size_gc_header
+        return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+
+    def get_size(self, addr):
+        return llmemory.sizeof(lltype.typeOf(addr.ptr).TO)
+
+    def trace(self, obj, callback, arg):
+        TYPE = obj.ptr._TYPE.TO
+        if TYPE == S:
+            ofslist = []     # no pointers in S
+        else:
+            assert 0
+        for ofs in ofslist:
+            addr = obj + ofs
+            if addr.address[0]:
+                callback(addr, arg)
 
 
 class TestStmGCTLS(object):
@@ -36,6 +78,8 @@
         self.gctls_main = StmGCTLS(self.gc, in_main_thread=True)
         self.gctls_thrd = StmGCTLS(self.gc, in_main_thread=False)
         self.gc.main_thread_tls = self.gctls_main
+        self.gctls_main.start_transaction()
+        self.gc.root_walker.current_stack = self.current_stack
 
     def stack_add(self, p):
         self.current_stack.append(p)
@@ -43,6 +87,20 @@
     def stack_pop(self):
         return self.current_stack.pop()
 
+    def malloc(self, STRUCT):
+        size = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT))
+        size_gc_header = self.gc.gcheaderbuilder.size_gc_header
+        totalsize = size_gc_header + size
+        tls = self.gc.main_thread_tls
+        adr = tls.allocate_bump_pointer(totalsize)
+        #
+        llarena.arena_reserve(adr, totalsize)
+        obj = adr + size_gc_header
+        hdr = self.gc.header(obj)
+        hdr.tid = 0
+        hdr.version = NULL
+        return llmemory.cast_adr_to_ptr(obj, lltype.Ptr(STRUCT))
+
     # ----------
 
     def test_creation_works(self):
@@ -59,16 +117,19 @@
         assert a6 - a5 == 5
 
     def test_local_collection(self):
-        s1, _ = self.malloc(S); s1.a = 111
-        s2, _ = self.malloc(S); s2.a = 222
+        s1 = self.malloc(S); s1.a = 111
+        s2 = self.malloc(S); s2.a = 222
         self.stack_add(s2)
         self.gc.main_thread_tls.local_collection()
         s3 = self.stack_pop()
         assert s3.a == 222
-        xxxx # raises...
-        s1.a
-        s2.a
+        py.test.raises(RuntimeError, "s1.a")
+        py.test.raises(RuntimeError, "s2.a")
 
-    def test_alloc_a_lot(self):
-        for i in range(1000):
-            sr1, sr1_adr = self.malloc(SR)
+    def test_alloc_a_lot_nonkept(self):
+        for i in range(100):
+            self.malloc(S)
+
+    def test_alloc_a_lot_kept(self):
+        for i in range(100):
+            self.stack_add(self.malloc(S))
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -1374,16 +1374,19 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   arg=None):
         gcdata = self.gcdata
         gc = self.gc
+        if arg is None:
+            arg = gc
         if collect_static_in_prebuilt_nongc:
             addr = gcdata.static_root_start
             end = gcdata.static_root_nongcend
             while addr != end:
                 result = addr.address[0]
                 if gc.points_to_valid_gc_object(result):
-                    collect_static_in_prebuilt_nongc(gc, result)
+                    collect_static_in_prebuilt_nongc(arg, result)
                 addr += sizeofaddr
         if collect_static_in_prebuilt_gc:
             addr = gcdata.static_root_nongcend
@@ -1391,10 +1394,10 @@
             while addr != end:
                 result = addr.address[0]
                 if gc.points_to_valid_gc_object(result):
-                    collect_static_in_prebuilt_gc(gc, result)
+                    collect_static_in_prebuilt_gc(arg, result)
                 addr += sizeofaddr
         if collect_stack_root:
-            self.walk_stack_roots(collect_stack_root)     # abstract
+            self.walk_stack_roots(collect_stack_root, arg)     # abstract
 
     def need_stacklet_support(self):
         raise Exception("%s does not support stacklets" % (
diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py
--- a/pypy/rpython/memory/gctransform/shadowstack.py
+++ b/pypy/rpython/memory/gctransform/shadowstack.py
@@ -28,7 +28,7 @@
         self.decr_stack = decr_stack
 
         root_iterator = get_root_iterator(gctransformer)
-        def walk_stack_root(callback, start, end):
+        def walk_stack_root(callback, arg, start, end):
             root_iterator.setcontext(NonConstant(llmemory.NULL))
             gc = self.gc
             addr = end
@@ -36,7 +36,7 @@
                 addr = root_iterator.nextleft(gc, start, addr)
                 if addr == llmemory.NULL:
                     return
-                callback(gc, addr)
+                callback(arg, addr)
         self.rootstackhook = walk_stack_root
 
         self.shadow_stack_pool = ShadowStackPool(gcdata)
@@ -56,9 +56,9 @@
         self.shadow_stack_pool.initial_setup()
         BaseRootWalker.setup_root_walker(self)
 
-    def walk_stack_roots(self, collect_stack_root):
+    def walk_stack_roots(self, collect_stack_root, arg):
         gcdata = self.gcdata
-        self.rootstackhook(collect_stack_root,
+        self.rootstackhook(collect_stack_root, arg,
                            gcdata.root_stack_base, gcdata.root_stack_top)
 
     def need_thread_support(self, gctransformer, getfn):

From noreply at buildbot.pypy.org  Sun Apr 15 15:14:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 15 Apr 2012 15:14:41 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Added a test.
Message-ID: <20120415131441.813DA820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54370:b4693feb6f55
Date: 2012-04-15 15:14 +0200
http://bitbucket.org/pypy/pypy/changeset/b4693feb6f55/

Log:	Added a test.

diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -9,6 +9,10 @@
 NULL = llmemory.NULL
 S = lltype.GcStruct('S', ('a', lltype.Signed), ('b', lltype.Signed),
                          ('c', lltype.Signed))
+SR = lltype.GcForwardReference()
+SR.become(lltype.GcStruct('SR', ('s1', lltype.Ptr(S)),
+                                ('sr2', lltype.Ptr(SR)),
+                                ('sr3', lltype.Ptr(SR))))
 
 
 class FakeStmOperations:
@@ -61,6 +65,10 @@
         TYPE = obj.ptr._TYPE.TO
         if TYPE == S:
             ofslist = []     # no pointers in S
+        elif TYPE == SR:
+            ofslist = [llmemory.offsetof(SR, 's1'),
+                       llmemory.offsetof(SR, 'sr2'),
+                       llmemory.offsetof(SR, 'sr3')]
         else:
             assert 0
         for ofs in ofslist:
@@ -132,4 +140,22 @@
 
     def test_alloc_a_lot_kept(self):
         for i in range(100):
-            self.stack_add(self.malloc(S))
+            s1 = self.malloc(S)
+            s1.a = i
+            self.stack_add(s1)
+        for i in range(100)[::-1]:
+            s2 = self.stack_pop()
+            assert s2.a == i
+
+    def test_alloc_chain(self):
+        srlist = lltype.nullptr(SR)
+        for i in range(100):
+            self.stack_add(srlist)
+            sr1 = self.malloc(SR)
+            srlist = self.stack_pop()
+            sr1.sr2 = srlist
+            srlist = sr1
+        for i in range(100)[::-1]:
+            assert srlist
+            srlist = srlist.sr2
+        assert not srlist

From noreply at buildbot.pypy.org  Sun Apr 15 19:01:14 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 19:01:14 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: implement malloc_and_pin
	operation
Message-ID: <20120415170114.7332A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54371:7deff2c04f6f
Date: 2012-04-15 14:44 +0200
http://bitbucket.org/pypy/pypy/changeset/7deff2c04f6f/

Log:	implement malloc_and_pin operation

diff --git a/pypy/jit/tl/jittest.py b/pypy/jit/tl/jittest.py
--- a/pypy/jit/tl/jittest.py
+++ b/pypy/jit/tl/jittest.py
@@ -5,7 +5,6 @@
 """
 
 from pypy.conftest import option
-from pypy.rpython.lltypesystem import lltype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llstr
 from pypy.jit.metainterp import warmspot
@@ -19,10 +18,6 @@
     graph = driver.translator.graphs[0]
     interp = LLInterpreter(driver.translator.rtyper, malloc_check=False)
 
-    def returns_null(T, *args, **kwds):
-        return lltype.nullptr(T)
-    interp.heap.malloc_nonmovable = returns_null     # XXX
-
     get_policy = driver.extra['jitpolicy']
     jitpolicy = get_policy(driver)
 
diff --git a/pypy/jit/tl/pypyjit_child.py b/pypy/jit/tl/pypyjit_child.py
--- a/pypy/jit/tl/pypyjit_child.py
+++ b/pypy/jit/tl/pypyjit_child.py
@@ -1,5 +1,4 @@
 from pypy.conftest import option
-from pypy.rpython.lltypesystem import lltype
 from pypy.jit.metainterp import warmspot
 from pypy.module.pypyjit.policy import PyPyJitPolicy
 
@@ -10,10 +9,6 @@
     graph = loc['graph']
     interp.malloc_check = False
 
-    def returns_null(T, *args, **kwds):
-        return lltype.nullptr(T)
-    interp.heap.malloc_nonmovable = returns_null     # XXX
-
     from pypy.jit.backend.llgraph.runner import LLtypeCPU
     #LLtypeCPU.supports_floats = False    # for now
     apply_jit(interp, graph, LLtypeCPU)
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -116,19 +116,18 @@
         return annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP))
 
     def specialize_call(self, hop):
-        from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP
         hop.exception_is_here()
         return hop.genop('gc_heap_stats', [], resulttype=hop.r_result)
 
-def malloc_nonmovable(TP, n=None, zero=False):
-    """ Allocate a non-moving buffer or return nullptr.
-    When running directly, will pretend that gc is always
-    moving (might be configurable in a future)
+def malloc_and_pin(TP, n=None, zero=False):
+    """ Allocate a buffer and immediately pin it. If the GC does not support
+    pinning or cannot fulfill the request for some other reason, null ptr
+    is returned. Make sure you unpin this thing when you're done
     """
-    return lltype.nullptr(TP)
+    return lltype.malloc(TP, n, zero=zero)
 
-class MallocNonMovingEntry(ExtRegistryEntry):
-    _about_ = malloc_nonmovable
+class MallocAndPinEntry(ExtRegistryEntry):
+    _about_ = malloc_and_pin
 
     def compute_result_annotation(self, s_TP, s_n=None, s_zero=None):
         # basically return the same as malloc
@@ -136,10 +135,9 @@
         return malloc(s_TP, s_n, s_zero=s_zero)
 
     def specialize_call(self, hop, i_zero=None):
-        # XXX assume flavor and zero to be None by now
         assert hop.args_s[0].is_constant()
         vlist = [hop.inputarg(lltype.Void, arg=0)]
-        opname = 'malloc_nonmovable'
+        opname = 'malloc_and_pin'
         flags = {'flavor': 'gc'}
         if i_zero is not None:
             flags['zero'] = hop.args_s[i_zero].const
@@ -150,7 +148,7 @@
 
         if nb_args == 2:
             vlist.append(hop.inputarg(lltype.Signed, arg=1))
-            opname += '_varsize'
+            opname = 'malloc_varsize_and_pin'
 
         hop.exception_cannot_occur()
         return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype)
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -702,17 +702,17 @@
         except MemoryError:
             self.make_llexception()
 
-    def op_malloc_nonmovable(self, TYPE, flags):
+    def op_malloc_and_pin(self, TYPE, flags):
         flavor = flags['flavor']
         assert flavor == 'gc'
         zero = flags.get('zero', False)
-        return self.heap.malloc_nonmovable(TYPE, zero=zero)
+        return self.heap.malloc_and_pin(TYPE, zero=zero)
 
-    def op_malloc_nonmovable_varsize(self, TYPE, flags, size):
+    def op_malloc_varsize_and_pin(self, TYPE, flags, size):
         flavor = flags['flavor']
         assert flavor == 'gc'
         zero = flags.get('zero', False)
-        return self.heap.malloc_nonmovable(TYPE, size, zero=zero)
+        return self.heap.malloc_varsize_and_pin(TYPE, size, zero=zero)
 
     def op_free(self, obj, flags):
         assert flags['flavor'] == 'raw'
diff --git a/pypy/rpython/lltypesystem/llheap.py b/pypy/rpython/lltypesystem/llheap.py
--- a/pypy/rpython/lltypesystem/llheap.py
+++ b/pypy/rpython/lltypesystem/llheap.py
@@ -18,7 +18,7 @@
 def weakref_create_getlazy(objgetter):
     return weakref_create(objgetter())
 
-malloc_nonmovable = malloc
+malloc_and_pin = malloc
 
 def shrink_array(p, smallersize):
     return False
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -357,8 +357,8 @@
 
     'malloc':               LLOp(canmallocgc=True),
     'malloc_varsize':       LLOp(canmallocgc=True),
-    'malloc_nonmovable':    LLOp(canmallocgc=True),
-    'malloc_nonmovable_varsize':LLOp(canmallocgc=True),
+    'malloc_and_pin':    LLOp(canmallocgc=True),
+    'malloc_varsize_and_pin':LLOp(canmallocgc=True),
     'shrink_array':         LLOp(canrun=True),
     'zero_gc_pointers_inside': LLOp(),
     'free':                 LLOp(),
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -510,13 +510,7 @@
         ctype = inputconst(Void, self.object_type)
         cflags = inputconst(Void, flags)
         vlist = [ctype, cflags]
-        cnonmovable = self.classdef.classdesc.read_attribute(
-            '_alloc_nonmovable_', Constant(False))
-        if cnonmovable.value:
-            opname = 'malloc_nonmovable'
-        else:
-            opname = 'malloc'
-        vptr = llops.genop(opname, vlist,
+        vptr = llops.genop('malloc', vlist,
                            resulttype = Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -759,6 +759,9 @@
         safe high-level string. When the garbage collector cooperates, this
         allows for the process to be performed without an extra copy.
         Make sure to call keep_buffer_alive_until_here on the returned values.
+
+        Right now this is a version optimized for minimark which can pin values
+        in the nursery.
         """
         raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
         return raw_buf, lltype.nullptr(STRTYPE)
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -49,9 +49,6 @@
     def _teardown(self):
         pass
 
-    def can_malloc_nonmovable(self):
-        return not self.moving_gc
-
     def can_optimize_clean_setarrayitems(self):
         return True     # False in case of card marking
 
@@ -169,8 +166,12 @@
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
 
-    def malloc_nonmovable(self, typeid, length=0, zero=False):
-        return self.malloc(typeid, length, zero)
+    def malloc_fixedsize_and_pin(self, typeid, size):
+        return lltype.nullptr(llmemory.GCREF.TO)
+
+    def malloc_varsize_and_pin(self, typeid, length, size, itemsize,
+                               offset_to_length):
+        return lltype.nullptr(llmemory.GCREF.TO)
 
     def id(self, ptr):
         return lltype.cast_ptr_to_int(ptr)
diff --git a/pypy/rpython/memory/gc/hybrid.py b/pypy/rpython/memory/gc/hybrid.py
--- a/pypy/rpython/memory/gc/hybrid.py
+++ b/pypy/rpython/memory/gc/hybrid.py
@@ -197,17 +197,6 @@
 
     malloc_varsize_slowpath._dont_inline_ = True
 
-    def malloc_varsize_nonmovable(self, typeid, length):
-        return self.malloc_varsize_slowpath(typeid, length, True)
-
-    def malloc_nonmovable(self, typeid, length, zero):
-        # helper for testing, same as GCBase.malloc
-        if self.is_varsize(typeid):
-            gcref = self.malloc_varsize_slowpath(typeid, length, True)
-        else:
-            raise NotImplementedError("Not supported")
-        return llmemory.cast_ptr_to_adr(gcref)
-
     def can_move(self, addr):
         tid = self.header(addr).tid
         return not (tid & GCFLAG_EXTERNAL)
@@ -554,6 +543,3 @@
                   "gen3: unexpected GCFLAG_UNVISITED")
         ll_assert((tid & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX,
                   "gen3: wrong age field")
-
-    def can_malloc_nonmovable(self):
-        return True
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -574,6 +574,17 @@
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
+    def malloc_fixedsize_and_pin(self, typeid, size):
+        r = self.malloc_fixedsize_clear(typeid, size)
+        self.pin(llmemory.cast_ptr_to_adr(r))
+        return r
+        
+    def malloc_varsize_and_pin(self, typeid, length, size, itemsize,
+                               offset_to_length):
+        r = self.malloc_varsize_clear(typeid, length, size, itemsize,
+                                      offset_to_length)
+        self.pin(llmemory.cast_ptr_to_adr(r))
+        return r        
 
     def collect(self, gen=1):
         """Do a minor (gen=0) or major (gen>0) collection."""
@@ -775,9 +786,6 @@
             # that one will occur very soon
             self.nursery_free = self.nursery_top
 
-    def can_malloc_nonmovable(self):
-        return True
-
     def can_optimize_clean_setarrayitems(self):
         if self.card_page_indices > 0:
             return False
@@ -821,20 +829,6 @@
         (obj + offset_to_length).signed[0] = smallerlength
         return True
 
-
-    def malloc_fixedsize_nonmovable(self, typeid):
-        obj = self.external_malloc(typeid, 0)
-        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-
-    def malloc_varsize_nonmovable(self, typeid, length):
-        obj = self.external_malloc(typeid, length)
-        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-
-    def malloc_nonmovable(self, typeid, length, zero):
-        # helper for testing, same as GCBase.malloc
-        return self.external_malloc(typeid, length or 0)    # None -> 0
-
-
     # ----------
     # Simple helpers
 
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -1,15 +1,10 @@
 from pypy.rpython.memory.gctransform.transform import GCTransformer
-from pypy.rpython.memory.gctransform.support import find_gc_ptrs_in_type, \
-     get_rtti, ll_call_destructor, type_contains_pyobjs, var_ispyobj
+from pypy.rpython.memory.gctransform.support import get_rtti,\
+     ll_call_destructor, type_contains_pyobjs, var_ispyobj
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, llgroup
 from pypy.rpython import rmodel
 from pypy.rpython.memory import gctypelayout
-from pypy.rpython.memory.gc import marksweep
-from pypy.rpython.memory.gcheader import GCHeaderBuilder
-from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib import rgc
-from pypy.rlib.debug import ll_assert
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
 from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
@@ -17,11 +12,11 @@
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
 from pypy.rpython.memory.gctypelayout import ll_weakref_deref, WEAKREF
-from pypy.rpython.memory.gctypelayout import convert_weakref_to, WEAKREFPTR
+from pypy.rpython.memory.gctypelayout import WEAKREFPTR
 from pypy.rpython.memory.gctransform.log import log
 from pypy.tool.sourcetools import func_with_new_name
-from pypy.rpython.lltypesystem.lloperation import llop, LL_OPERATIONS
-import sys, types
+from pypy.rpython.lltypesystem.lloperation import LL_OPERATIONS
+import types
 
 
 TYPE_ID = llgroup.HALFWORD
@@ -288,6 +283,14 @@
         self.can_move_ptr = getfn(GCClass.can_move.im_func,
                                   [s_gc, annmodel.SomeAddress()],
                                   annmodel.SomeBool())
+        self.malloc_fixedsize_and_pin_ptr = getfn(
+            GCClass.malloc_fixedsize_and_pin,
+            [s_gc, s_typeid16, annmodel.SomeInteger(nonneg=True)],
+            s_gcref)
+        self.malloc_varsize_and_pin_ptr = getfn(
+            GCClass.malloc_varsize_and_pin,
+            [s_gc, s_typeid16] + [annmodel.SomeInteger(nonneg=True)] * 4,
+            s_gcref)
 
         if hasattr(GCClass, 'shrink_array'):
             self.shrink_array_ptr = getfn(
@@ -365,17 +368,6 @@
         else:
             self.malloc_varsize_clear_fast_ptr = None
 
-        if getattr(GCClass, 'malloc_varsize_nonmovable', False):
-            malloc_nonmovable = func_with_new_name(
-                GCClass.malloc_varsize_nonmovable.im_func,
-                "malloc_varsize_nonmovable")
-            self.malloc_varsize_nonmovable_ptr = getfn(
-                malloc_nonmovable,
-                [s_gc, s_typeid16,
-                 annmodel.SomeInteger(nonneg=True)], s_gcref)
-        else:
-            self.malloc_varsize_nonmovable_ptr = None
-
         if getattr(GCClass, 'raw_malloc_memory_pressure', False):
             def raw_malloc_memory_pressure_varsize(length, itemsize):
                 totalmem = length * itemsize
@@ -690,10 +682,12 @@
         c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
                                                   has_light_finalizer)
 
-        if not op.opname.endswith('_varsize') and not flags.get('varsize'):
+        if not 'varsize' in op.opname and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
             zero = flags.get('zero', False)
-            if (self.malloc_fast_ptr is not None and
+            if op.opname == 'malloc_and_pin':
+                malloc_ptr = self.malloc_fixedsize_and_pin_ptr
+            elif (self.malloc_fast_ptr is not None and
                 not c_has_finalizer.value and
                 (self.malloc_fast_is_clearing or not zero)):
                 malloc_ptr = self.malloc_fast_ptr
@@ -712,18 +706,14 @@
                                               info_varsize.ofstolength)
             c_varitemsize = rmodel.inputconst(lltype.Signed,
                                               info_varsize.varitemsize)
-            if flags.get('nonmovable') and self.malloc_varsize_nonmovable_ptr:
-                # we don't have tests for such cases, let's fail
-                # explicitely
-                malloc_ptr = self.malloc_varsize_nonmovable_ptr
-                args = [self.c_const_gc, c_type_id, v_length]
+            if op.opname == 'malloc_varsize_and_pin':
+                malloc_ptr = self.malloc_varsize_and_pin_ptr
+            elif self.malloc_varsize_clear_fast_ptr is not None:
+                malloc_ptr = self.malloc_varsize_clear_fast_ptr
             else:
-                if self.malloc_varsize_clear_fast_ptr is not None:
-                    malloc_ptr = self.malloc_varsize_clear_fast_ptr
-                else:
-                    malloc_ptr = self.malloc_varsize_clear_ptr
-                args = [self.c_const_gc, c_type_id, v_length, c_size,
-                        c_varitemsize, c_ofstolength]
+                malloc_ptr = self.malloc_varsize_clear_ptr
+            args = [self.c_const_gc, c_type_id, v_length, c_size,
+                    c_varitemsize, c_ofstolength]
         livevars = self.push_roots(hop)
         v_result = hop.genop("direct_call", [malloc_ptr] + args,
                              resulttype=llmemory.GCREF)
@@ -1109,19 +1099,15 @@
                   resultvar=hop.spaceop.result)
         self.pop_roots(hop, livevars)
 
-    def gct_malloc_nonmovable_varsize(self, hop):
+    def gct_malloc_and_pin(self, hop):
         TYPE = hop.spaceop.result.concretetype
-        if self.gcdata.gc.can_malloc_nonmovable():
-            return self.gct_malloc_varsize(hop, {'nonmovable':True})
-        c = rmodel.inputconst(TYPE, lltype.nullptr(TYPE.TO))
-        return hop.cast_result(c)
+        v_raw = self.gct_fv_gc_malloc(hop, {'flavor': 'gc'}, TYPE.TO)
+        hop.cast_result(v_raw)
 
-    def gct_malloc_nonmovable(self, hop):
+    def gct_malloc_varisze_and_pin(self, hop):
         TYPE = hop.spaceop.result.concretetype
-        if self.gcdata.gc.can_malloc_nonmovable():
-            return self.gct_malloc(hop, {'nonmovable':True})
-        c = rmodel.inputconst(TYPE, lltype.nullptr(TYPE.TO))
-        return hop.cast_result(c)
+        v_raw = self.gct_fv_gc_malloc(hop, {'flavor': 'gc'}, TYPE.TO)
+        hop.cast_result(v_raw)
 
     def _set_into_gc_array_part(self, op):
         if op.opname == 'setarrayitem':
diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -548,14 +548,14 @@
         flavor = flags['flavor']
         assert flavor != 'cpy', "cannot malloc CPython objects directly"
         meth = getattr(self, 'gct_fv_%s_malloc_varsize' % flavor, None)
-        assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor) 
+        assert meth, "%s has no support for malloc_varsize with flavor %r" % (self, flavor)
         return self.varsize_malloc_helper(hop, flags, meth, [])
 
-    def gct_malloc_nonmovable(self, *args, **kwds):
+    def gct_malloc_and_pin(self, *args, **kwds):
         return self.gct_malloc(*args, **kwds)
 
-    def gct_malloc_nonmovable_varsize(self, *args, **kwds):
-        return self.gct_malloc_varsize(*args, **kwds)
+    def gct_malloc_varsize_and_pin(self, hop, *args, **kwds):
+        return self.gct_malloc_varsize(hop, *args, **kwds)
 
     def gct_gc_add_memory_pressure(self, hop):
         if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -56,15 +56,24 @@
             return lltype.malloc(TYPE, n, flavor=flavor, zero=zero,
                                  track_allocation=track_allocation)
 
-    def malloc_nonmovable(self, TYPE, n=None, zero=False):
+    def malloc_and_pin(self, TYPE, n=None, zero=False):
         typeid = self.get_type_id(TYPE)
-        if not self.gc.can_malloc_nonmovable():
-            return lltype.nullptr(TYPE)
-        addr = self.gc.malloc_nonmovable(typeid, n, zero=zero)
-        result = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(TYPE))
-        if not self.gc.malloc_zero_filled:
-            gctypelayout.zero_gc_pointers(result)
-        return result
+        size = self.gc.fixed_size(typeid)
+        result = self.gc.malloc_fixedsize_and_pin(typeid, size)
+        if result:
+            assert self.gc.malloc_zero_filled
+        return lltype.cast_opaque_ptr(lltype.Ptr(TYPE), result)
+
+    def malloc_varsize_and_pin(self, TYPE, n=None, zero=False):
+        typeid = self.get_type_id(TYPE)
+        size = self.gc.fixed_size(typeid)
+        itemsize = self.gc.varsize_item_sizes(typeid)
+        offset_to_length = self.gc.varsize_offset_to_length(typeid)
+        result = self.gc.malloc_varsize_and_pin(typeid, n, size, itemsize,
+                                              offset_to_length)
+        if result:
+            assert self.gc.malloc_zero_filled
+        return lltype.cast_opaque_ptr(lltype.Ptr(TYPE), result)
 
     def add_memory_pressure(self, size):
         if hasattr(self.gc, 'raw_malloc_memory_pressure'):
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -24,7 +24,7 @@
 class GCTest(object):
     GC_PARAMS = {}
     GC_CAN_MOVE = False
-    GC_CAN_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_AND_PIN = False
     GC_CAN_SHRINK_ARRAY = False
     GC_CAN_SHRINK_BIG_ARRAY = False
     BUT_HOW_BIG_IS_A_BIG_STRING = 3*WORD
@@ -592,24 +592,23 @@
         assert self.interpret(func, []) == self.GC_CAN_MOVE
 
 
-    def test_malloc_nonmovable(self):
+    def test_malloc_and_pin(self):
         TP = lltype.GcArray(lltype.Char)
         def func():
-            a = rgc.malloc_nonmovable(TP, 3)
+            a = rgc.malloc_and_pin(TP, 3)
             if a:
                 assert not rgc.can_move(a)
                 return 1
             return 0
 
-        assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
+        assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_AND_PIN)
 
-    def test_malloc_nonmovable_fixsize(self):
+    def test_malloc_and_pin_fixsize(self):
         S = lltype.GcStruct('S', ('x', lltype.Float))
         TP = lltype.GcStruct('T', ('s', lltype.Ptr(S)))
         def func():
             try:
-                a = rgc.malloc_nonmovable(TP)
-                rgc.collect()
+                a = rgc.malloc_and_pin(TP)
                 if a:
                     assert not rgc.can_move(a)
                     return 1
@@ -617,7 +616,7 @@
             except Exception:
                 return 2
 
-        assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
+        assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_AND_PIN)
 
     def test_shrink_array(self):
         from pypy.rpython.lltypesystem.rstr import STR
@@ -793,7 +792,6 @@
 class TestSemiSpaceGC(GCTest, snippet.SemiSpaceGCTests):
     from pypy.rpython.memory.gc.semispace import SemiSpaceGC as GCClass
     GC_CAN_MOVE = True
-    GC_CAN_MALLOC_NONMOVABLE = False
     GC_CAN_SHRINK_ARRAY = True
     GC_CAN_SHRINK_BIG_ARRAY = True
 
@@ -814,7 +812,6 @@
 
 class TestHybridGC(TestGenerationalGC):
     from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass
-    GC_CAN_MALLOC_NONMOVABLE = True
     GC_CAN_SHRINK_BIG_ARRAY = False
 
     def test_ref_from_rawmalloced_to_regular(self):
@@ -878,14 +875,10 @@
         res = self.interpret(f, [15])
         assert res == 16
 
-    def test_malloc_nonmovable_fixsize(self):
-        py.test.skip("Not supported")
-
 class TestHybridGCSmallHeap(GCTest):
     from pypy.rpython.memory.gc.hybrid import HybridGC as GCClass
     GC_CAN_MOVE = False # with this size of heap, stuff gets allocated
                         # in 3rd gen.
-    GC_CAN_MALLOC_NONMOVABLE = True
     GC_PARAMS = {'space_size': 48*WORD,
                  'min_nursery_size': 12*WORD,
                  'nursery_size': 12*WORD,
@@ -927,14 +920,11 @@
         res = self.interpret(f, [200])
         assert res == 401
 
-    def test_malloc_nonmovable_fixsize(self):
-        py.test.skip("Not supported")
-
 
 class TestMiniMarkGC(TestSemiSpaceGC):
     from pypy.rpython.memory.gc.minimark import MiniMarkGC as GCClass
     GC_CAN_SHRINK_BIG_ARRAY = False
-    GC_CAN_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_AND_PIN = True
     BUT_HOW_BIG_IS_A_BIG_STRING = 11*WORD
 
     # those tests are here because they'll be messy and useless
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -42,7 +42,7 @@
 class GCTest(object):
     gcpolicy = None
     GC_CAN_MOVE = False
-    GC_CAN_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_AND_PIN = False
     GC_CAN_ALWAYS_PIN = False
     taggedpointers = False
 
@@ -643,44 +643,41 @@
         res = run([])
         assert res == self.GC_CAN_MOVE
 
-    def define_malloc_nonmovable(cls):
+    def define_malloc_and_pin(cls):
         TP = lltype.GcArray(lltype.Char)
         def func():
-            #try:
-            a = rgc.malloc_nonmovable(TP, 3, zero=True)
-            rgc.collect()
+            a = rgc.malloc_and_pin(TP, 3, zero=True)
             if a:
                 assert not rgc.can_move(a)
+                rgc.unpin(a)
                 return 1
             return 0
-            #except Exception, e:
-            #    return 2
 
         return func
 
-    def test_malloc_nonmovable(self):
-        run = self.runner("malloc_nonmovable")
-        assert int(self.GC_CAN_MALLOC_NONMOVABLE) == run([])
+    def test_malloc_and_pin(self):
+        run = self.runner("malloc_and_pin")
+        assert int(self.GC_CAN_MALLOC_AND_PIN) == run([])
 
-    def define_malloc_nonmovable_fixsize(cls):
+    def define_malloc_and_pin_fixsize(cls):
         S = lltype.GcStruct('S', ('x', lltype.Float))
         TP = lltype.GcStruct('T', ('s', lltype.Ptr(S)))
         def func():
             try:
-                a = rgc.malloc_nonmovable(TP)
-                rgc.collect()
+                a = rgc.malloc_and_pin(TP)
                 if a:
                     assert not rgc.can_move(a)
+                    rgc.unpin(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         return func
 
-    def test_malloc_nonmovable_fixsize(self):
-        run = self.runner("malloc_nonmovable_fixsize")
-        assert run([]) == int(self.GC_CAN_MALLOC_NONMOVABLE)
+    def test_malloc_and_pin_fixsize(self):
+        run = self.runner("malloc_and_pin_fixsize")
+        assert run([]) == int(self.GC_CAN_MALLOC_AND_PIN)
 
     def define_shrink_array(cls):
         from pypy.rpython.lltypesystem.rstr import STR
@@ -751,7 +748,6 @@
 
 class GenericMovingGCTests(GenericGCTests):
     GC_CAN_MOVE = True
-    GC_CAN_MALLOC_NONMOVABLE = False
     GC_CAN_TEST_ID = False
 
     def define_many_ids(cls):
@@ -1226,7 +1222,7 @@
 
 class TestHybridGC(TestGenerationGC):
     gcname = "hybrid"
-    GC_CAN_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_AND_PIN = True
 
     class gcpolicy(gc.FrameworkGcPolicy):
         class transformerclass(framework.FrameworkGCTransformer):
@@ -1289,14 +1285,12 @@
         res = run([])
         assert res == 42
 
-    def test_malloc_nonmovable_fixsize(self):
-        py.test.skip("not supported")
-
 
 class TestMiniMarkGC(TestHybridGC):
     gcname = "minimark"
     GC_CAN_TEST_ID = True
     GC_CAN_ALWAYS_PIN = True
+    GC_CAN_MALLOC_AND_PIN = True
 
     class gcpolicy(gc.FrameworkGcPolicy):
         class transformerclass(framework.FrameworkGCTransformer):
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -1131,18 +1131,6 @@
             assert sorted([u]) == [6]                    # 32-bit types
             assert sorted([i, r, d, l]) == [2, 3, 4, 5]  # 64-bit types
 
-    def test_nonmovable(self):
-        for (nonmovable, opname) in [(True, 'malloc_nonmovable'),
-                                     (False, 'malloc')]:
-            class A(object):
-                _alloc_nonmovable_ = nonmovable
-            def f():
-                return A()
-            t, typer, graph = self.gengraph(f, [])
-            assert summary(graph) == {opname: 1,
-                                      'cast_pointer': 1,
-                                      'setfield': 1}
-
 
 class TestOOtype(BaseTestRclass, OORtypeMixin):
 
diff --git a/pypy/translator/c/test/test_boehm.py b/pypy/translator/c/test/test_boehm.py
--- a/pypy/translator/c/test/test_boehm.py
+++ b/pypy/translator/c/test/test_boehm.py
@@ -391,23 +391,6 @@
         c_fn = self.getcompiled(fn, [])
         assert not c_fn()
 
-    def test_malloc_nonmovable(self):
-        TP = lltype.GcArray(lltype.Char)
-        def func():
-            try:
-                from pypy.rlib import rgc
-                a = rgc.malloc_nonmovable(TP, 3)
-                rgc.collect()
-                if a:
-                    assert not rgc.can_move(a)
-                    return 0
-                return 1
-            except Exception, e:
-                return 2
-
-        run = self.getcompiled(func)
-        assert run() == 0
-
     def test_shrink_array(self):
         from pypy.rpython.lltypesystem.rstr import STR
         from pypy.rlib import rgc
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -19,7 +19,7 @@
     removetypeptr = False
     taggedpointers = False
     GC_CAN_MOVE = False
-    GC_CAN_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_AND_PIN = False
     GC_CAN_SHRINK_ARRAY = False
 
     _isolated_func = None
@@ -723,24 +723,24 @@
     def test_can_move(self):
         assert self.run('can_move') == self.GC_CAN_MOVE
 
-    def define_malloc_nonmovable(cls):
+    def define_malloc_and_pin(cls):
         TP = lltype.GcArray(lltype.Char)
         def func():
             try:
-                a = rgc.malloc_nonmovable(TP, 3)
-                rgc.collect()
+                a = rgc.malloc_and_pin(TP, 3)
                 if a:
                     assert not rgc.can_move(a)
+                    rgc.unpin(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         return func
 
-    def test_malloc_nonmovable(self):
-        res = self.run('malloc_nonmovable')
-        assert res == self.GC_CAN_MALLOC_NONMOVABLE
+    def test_malloc_and_pin(self):
+        res = self.run('malloc_and_pin')
+        assert res == self.GC_CAN_MALLOC_AND_PIN
 
     def define_resizable_buffer(cls):
         from pypy.rpython.lltypesystem.rstr import STR
@@ -1189,7 +1189,6 @@
     gcpolicy = "semispace"
     should_be_moving = True
     GC_CAN_MOVE = True
-    GC_CAN_MALLOC_NONMOVABLE = False
     GC_CAN_SHRINK_ARRAY = True
 
     # for snippets
@@ -1366,7 +1365,6 @@
 class TestHybridGC(TestGenerationalGC):
     gcpolicy = "hybrid"
     should_be_moving = True
-    GC_CAN_MALLOC_NONMOVABLE = True
 
     def test_gc_set_max_heap_size(self):
         py.test.skip("not implemented")
@@ -1440,7 +1438,7 @@
 class TestMiniMarkGC(TestSemiSpaceGC):
     gcpolicy = "minimark"
     should_be_moving = True
-    GC_CAN_MALLOC_NONMOVABLE = True
+    GC_CAN_MALLOC_AND_PIN = True
     GC_CAN_SHRINK_ARRAY = True
 
     def test_gc_heap_stats(self):
diff --git a/pypy/translator/exceptiontransform.py b/pypy/translator/exceptiontransform.py
--- a/pypy/translator/exceptiontransform.py
+++ b/pypy/translator/exceptiontransform.py
@@ -256,7 +256,8 @@
               (block.exits[0].args[0].concretetype is lltype.Void or
                block.exits[0].args[0] is block.operations[-1].result) and
               block.operations[-1].opname not in ('malloc',     # special cases
-                                                  'malloc_nonmovable')):
+                                                  'malloc_and_pin',
+                                                  'malloc_varsize_and_pin')):
             last_operation -= 1
         lastblock = block
         for i in range(last_operation, -1, -1):
@@ -437,11 +438,11 @@
             flavor = spaceop.args[1].value['flavor']
             if flavor == 'gc':
                 insert_zeroing_op = True
-        elif spaceop.opname == 'malloc_nonmovable':
+        elif spaceop.opname in ['malloc_and_pin', 'malloc_varsize_and_pin']:
             # xxx we cannot insert zero_gc_pointers_inside after
-            # malloc_nonmovable, because it can return null.  For now
+            # malloc_and_pin, because it can return null.  For now
             # we simply always force the zero=True flag on
-            # malloc_nonmovable.
+            # malloc_and_pin.
             c_flags = spaceop.args[1]
             c_flags.value = c_flags.value.copy()
             spaceop.args[1].value['zero'] = True

From noreply at buildbot.pypy.org  Sun Apr 15 19:01:15 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 19:01:15 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: use some more pinning
Message-ID: <20120415170115.BA93382F4F@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54372:96e1dad774ae
Date: 2012-04-15 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/96e1dad774ae/

Log:	use some more pinning

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -75,8 +75,15 @@
     With other moving GCs like the MiniMark GC, it can be True for some
     time, then False for the same object, when we are sure that it won't
     move any more.
+
+    We keep it for False when uncompiled, because malloc_and_pin would actually
+    return an object. This executes more interesting paths, but also:
+
+    x = malloc_and_pin(T)
+    if x:
+       assert not can_move(x)
     """
-    return True
+    return False
 
 class CanMoveEntry(ExtRegistryEntry):
     _about_ = can_move
@@ -200,17 +207,20 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
+    TP = lltype.typeOf(p).TO
+    assert len(TP._names) == 2
+    field = getattr(p, TP._names[0])
+
+    if len(getattr(p, TP._arrayfld)) == smallerlength:
+        return p
     if llop.shrink_array(lltype.Bool, p, smallerlength):
         return p    # done by the GC
     # XXX we assume for now that the type of p is GcStruct containing a
     # variable array, with no further pointers anywhere, and exactly one
     # field in the fixed part -- like STR and UNICODE.
 
-    TP = lltype.typeOf(p).TO
     newp = lltype.malloc(TP, smallerlength)
 
-    assert len(TP._names) == 2
-    field = getattr(p, TP._names[0])
     setattr(newp, TP._names[0], field)
 
     ARRAY = getattr(TP, TP._arrayfld)
diff --git a/pypy/rpython/lltypesystem/llheap.py b/pypy/rpython/lltypesystem/llheap.py
--- a/pypy/rpython/lltypesystem/llheap.py
+++ b/pypy/rpython/lltypesystem/llheap.py
@@ -19,6 +19,7 @@
     return weakref_create(objgetter())
 
 malloc_and_pin = malloc
+malloc_varsize_and_pin = malloc
 
 def shrink_array(p, smallersize):
     return False
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -739,6 +739,8 @@
         # if 'buf' points inside 'data'.  This is only possible if we
         # followed the 2nd case in get_nonmovingbuffer(); in the first case,
         # 'buf' points to its own raw-malloced memory.
+        import pdb
+        pdb.set_trace()
         data = llstrtype(data)
         data_start = cast_ptr_to_adr(data) + \
             offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
@@ -763,8 +765,15 @@
         Right now this is a version optimized for minimark which can pin values
         in the nursery.
         """
-        raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
-        return raw_buf, lltype.nullptr(STRTYPE)
+        ll_s = rgc.malloc_and_pin(STRTYPE, count)
+        if not ll_s:
+            raw_buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
+            return raw_buf, lltype.nullptr(STRTYPE)
+        else:
+            data_start = cast_ptr_to_adr(ll_s) + \
+                offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
+            raw_buf = cast(TYPEP, data_start)
+            return raw_buf, ll_s
     alloc_buffer._always_inline_ = True # to get rid of the returned tuple
     alloc_buffer._annenforceargs_ = [int]
 
@@ -777,7 +786,9 @@
         """
         assert allocated_size >= needed_size
 
-        if gc_buf and (allocated_size == needed_size):
+        if gc_buf:
+            if allocated_size != needed_size:
+                gc_buf = rgc.ll_shrink_array(gc_buf, needed_size)
             return hlstrtype(gc_buf)
 
         new_buf = lltype.malloc(STRTYPE, needed_size)
diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -377,6 +377,12 @@
     def gct_zero_gc_pointers_inside(self, hop):
         pass
 
+    def gct_gc_pin(self, hop):
+        pass
+
+    def gct_gc_unpin(self, hop):
+        pass
+
     def gct_gc_writebarrier_before_copy(self, hop):
         # We take the conservative default and return False here, meaning
         # that rgc.ll_arraycopy() will do the copy by hand (i.e. with a
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -7,11 +7,11 @@
 
 import os, sys, errno
 import py
-from pypy.rpython.module.support import ll_strcpy, OOSupport
-from pypy.tool.sourcetools import func_with_new_name, func_renamer
+from pypy.rpython.module.support import OOSupport
+from pypy.tool.sourcetools import func_renamer
 from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.extfunc import (
-    BaseLazyRegistering, lazy_register, register_external)
+    BaseLazyRegistering, register_external)
 from pypy.rpython.extfunc import registering, registering_if, extdef
 from pypy.annotation.model import (
     SomeInteger, SomeString, SomeTuple, SomeFloat, SomeUnicodeString)
@@ -20,15 +20,9 @@
 from pypy.rpython.lltypesystem import lltype
 from pypy.rpython.tool import rffi_platform as platform
 from pypy.rlib import rposix
-from pypy.tool.udir import udir
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.rpython.lltypesystem.rstr import mallocstr
-from pypy.rpython.annlowlevel import llstr
-from pypy.rpython.lltypesystem.llmemory import sizeof,\
-     itemoffsetof, cast_ptr_to_adr, cast_adr_to_ptr, offsetof
+from pypy.rpython.lltypesystem.llmemory import itemoffsetof, offsetof
 from pypy.rpython.lltypesystem.rstr import STR
-from pypy.rpython.annlowlevel import llstr
-from pypy.rlib import rgc
 from pypy.rlib.objectmodel import specialize
 
 str0 = SomeString(no_nul=True)

From noreply at buildbot.pypy.org  Sun Apr 15 19:04:08 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 19:04:08 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: oops
Message-ID: <20120415170408.30B69820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54373:1b620d7189ac
Date: 2012-04-15 19:03 +0200
http://bitbucket.org/pypy/pypy/changeset/1b620d7189ac/

Log:	oops

diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -739,8 +739,6 @@
         # if 'buf' points inside 'data'.  This is only possible if we
         # followed the 2nd case in get_nonmovingbuffer(); in the first case,
         # 'buf' points to its own raw-malloced memory.
-        import pdb
-        pdb.set_trace()
         data = llstrtype(data)
         data_start = cast_ptr_to_adr(data) + \
             offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)

From noreply at buildbot.pypy.org  Sun Apr 15 20:29:44 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 20:29:44 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: dont care about flags on
	old objects
Message-ID: <20120415182944.30BFB82F4F@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54375:b5d9d9c4cec1
Date: 2012-04-15 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/b5d9d9c4cec1/

Log:	dont care about flags on old objects

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -797,12 +797,10 @@
                 not self.header(obj).tid & GCFLAG_PINNED)
 
     def pin(self, obj):
-        if self.is_in_nursery(obj):
-            self.header(obj).tid |= GCFLAG_PINNED
+        self.header(obj).tid |= GCFLAG_PINNED
 
     def unpin(self, obj):
-        if self.is_in_nursery(obj):
-            self.header(obj).tid &= ~GCFLAG_PINNED
+        self.header(obj).tid &= ~GCFLAG_PINNED
 
     def shrink_array(self, obj, smallerlength):
         #

From noreply at buildbot.pypy.org  Sun Apr 15 20:29:42 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 20:29:42 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: write a test and kill assert
Message-ID: <20120415182942.D6328820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54374:7c1a5cf25df8
Date: 2012-04-15 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/7c1a5cf25df8/

Log:	write a test and kill assert

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1502,7 +1502,6 @@
             if hdr.tid & GCFLAG_VISITED:
                 return
             hdr.tid |= GCFLAG_VISITED
-            ll_assert(not self.header(obj).tid & GCFLAG_HAS_SHADOW, "support shadow with pinning")
             ll_assert(not self.header(obj).tid & GCFLAG_HAS_CARDS, "support cards with pinning")
             self.surviving_pinned_objects.insert(
                 llarena.getfakearenaaddress(obj - size_gc_header))
diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -535,6 +535,15 @@
         two = self.gc.nursery_free
         assert one != two
 
+    def test_pin_shadow(self):
+        s = self.malloc(S)
+        self.stackroots.append(s)
+        self.gc.id(s) # allocate shadow
+        self.gc.pin(llmemory.cast_ptr_to_adr(s))
+        self.gc.minor_collection(1)
+        assert self.gc.nursery_free != self.gc.nursery
+        # we still have a pinned object
+
     def test_writebarrier_before_copy(self):
         from pypy.rpython.memory.gc import minimark
         largeobj_size =  self.gc.nonlarge_max + 1

From noreply at buildbot.pypy.org  Sun Apr 15 20:29:45 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 20:29:45 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: a sorting attempt
Message-ID: <20120415182945.912B3820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54376:3ada686cfd21
Date: 2012-04-15 20:29 +0200
http://bitbucket.org/pypy/pypy/changeset/3ada686cfd21/

Log:	a sorting attempt

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,8 +1,4 @@
 
 * implement limit on no of pinned objects + replace insert with qsort
 
-* use pinning
-
-* make sure objects with shadows work
-
 * implement tracing for pinned objects (check that it works)
\ No newline at end of file
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -2,11 +2,11 @@
 from pypy.annotation import model as annmodel
 from pypy.rpython.lltypesystem import lltype
 from pypy.rpython.lltypesystem import ll2ctypes
-from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr
+from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr
 from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy
 from pypy.annotation.model import lltype_to_annotation
 from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic
+from pypy.rlib.objectmodel import Symbolic, we_are_translated
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import rarithmetic, rgc
 from pypy.rpython.extregistry import ExtRegistryEntry
@@ -14,7 +14,6 @@
 from pypy.rpython.tool.rfficache import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rpython.annlowlevel import llhelper
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import llmemory
@@ -715,14 +714,15 @@
         free_nonmovingbuffer call.
         """
         rgc.pin(data)
-        if rgc.can_move(data):
+        if rgc.can_move(data) or not we_are_translated():
             count = len(data)
             buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
             for i in range(count):
                 buf[i] = data[i]
             return buf
         else:
-            data_start = cast_ptr_to_adr(llstrtype(data)) + \
+            ll_s = llstrtype(data)
+            data_start = cast_ptr_to_adr(ll_s) + \
                 offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
             return cast(TYPEP, data_start)
     get_nonmovingbuffer._annenforceargs_ = [strtype]
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -205,6 +205,10 @@
         # minimal allocated size of the nursery is 2x the following
         # number (by default, at least 132KB on 32-bit and 264KB on 64-bit).
         "large_object": (16384+512)*WORD,
+
+        # number of allowed pinned objects, must be smaller than the size
+        # of one chunk for address stack
+        "max_number_of_pinned_objects": 100,
         }
 
     def __init__(self, config,
@@ -216,6 +220,7 @@
                  major_collection_threshold=2.5,
                  growth_rate_max=2.5,   # for tests
                  card_page_indices=0,
+                 max_number_of_pinned_objects=100,
                  large_object=8*WORD,
                  ArenaCollectionClass=None,
                  **kwds):
@@ -226,6 +231,7 @@
         self.small_request_threshold = small_request_threshold
         self.major_collection_threshold = major_collection_threshold
         self.growth_rate_max = growth_rate_max
+        self.max_number_of_pinned_objects = max_number_of_pinned_objects
         self.num_major_collects = 0
         self.min_heap_size = 0.0
         self.max_heap_size = 0.0
@@ -309,6 +315,7 @@
         # minor collect. This is a sorted deque that should be consulted when
         # considering next nursery ceiling
         self.nursery_barriers = self.AddressDeque()
+        self.pinned_objects_in_nursery = 0
     
         #
         # Allocate a nursery.  In case of auto_nursery_size, start by
@@ -797,9 +804,13 @@
                 not self.header(obj).tid & GCFLAG_PINNED)
 
     def pin(self, obj):
+        if self.pinned_objects_in_nursery >= self.max_number_of_pinned_objects:
+            return
+        self.pinned_objects_in_nursery += 1
         self.header(obj).tid |= GCFLAG_PINNED
 
     def unpin(self, obj):
+        self.pinned_objects_in_nursery -= 1
         self.header(obj).tid &= ~GCFLAG_PINNED
 
     def shrink_array(self, obj, smallerlength):
@@ -1264,6 +1275,8 @@
         # the young arrays.
         self.nursery_barriers.delete()
         self.surviving_pinned_objects = self.AddressStack()
+        self.pinned_objects_in_nursery = 0
+        # we track only surviving objects, don't care about the rest
         if self.young_rawmalloced_objects:
             self.remove_young_arrays_from_old_objects_pointing_to_young()
         #
@@ -1318,6 +1331,7 @@
         nursery_barriers = self.AddressDeque()
         prev = self.nursery
         size_gc_header = self.gcheaderbuilder.size_gc_header
+        self.surviving_pinned_objects.sort()
         while self.surviving_pinned_objects.non_empty():
             next = self.surviving_pinned_objects.pop()
             assert next >= prev
@@ -1501,7 +1515,8 @@
                 return
             hdr.tid |= GCFLAG_VISITED
             ll_assert(not self.header(obj).tid & GCFLAG_HAS_CARDS, "support cards with pinning")
-            self.surviving_pinned_objects.insert(
+            self.pinned_objects_in_nursery += 1
+            self.surviving_pinned_objects.append(
                 llarena.getfakearenaaddress(obj - size_gc_header))
             return
             #
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -1,6 +1,5 @@
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rlib.objectmodel import free_non_gc_object, we_are_translated
-from pypy.rlib.rarithmetic import r_uint, LONG_BIT
 from pypy.rlib.debug import ll_assert
 from pypy.tool.identity_dict import identity_dict
 
@@ -59,7 +58,30 @@
 
     unused_chunks = FreeList()
     cache[chunk_size] = unused_chunks, null_chunk
-    return unused_chunks, null_chunk
+
+    def partition(array, left, right):
+        last_item = array[right]
+        pivot = last_item
+        storeindex = left
+        for i in range(left, right):
+            if array[i] >= pivot:
+                array[i], array[storeindex] = array[storeindex], array[i]
+                storeindex += 1
+        # Move pivot to its final place
+        array[storeindex], array[right] = last_item, array[storeindex]
+        return storeindex
+
+    def quicksort(array, left, right):
+        # sort array[left:right+1] (i.e. bounds included)
+        if right > left:
+            pivotnewindex = partition(array, left, right)
+            quicksort(array, left, pivotnewindex - 1)
+            quicksort(array, pivotnewindex + 1, right)
+
+    def sort_chunk(chunk, size):
+        quicksort(chunk.items, 0, size - 1)
+        
+    return unused_chunks, null_chunk, sort_chunk
 
 
 def get_address_stack(chunk_size=DEFAULT_CHUNK_SIZE, cache={}):
@@ -68,7 +90,7 @@
     except KeyError:
         pass
 
-    unused_chunks, null_chunk = get_chunk_manager(chunk_size)
+    unused_chunks, null_chunk, sort_chunk = get_chunk_manager(chunk_size)
 
     class AddressStack(object):
         _alloc_flavor_ = "raw"
@@ -173,46 +195,9 @@
                 chunk.items[count] = got
                 got = next
 
-        def insert(self, addr):
-            """ Insert addr in the already sorted stack to make sure
-            the smallest one is on top
-            """
-            if self.used_in_last_chunk == 0:
-                self.append(addr)
-                return
-            got = self.pop()
-            read = self.used_in_last_chunk - 1
-            if read == -1 and got <= addr:
-                self.append(addr)
-                self.append(got)
-                return
-            read_chunk = self.chunk
-            self.append(got)
-            if got > addr:
-                self.append(addr)
-                return
-            write = self.used_in_last_chunk
-            if self.used_in_last_chunk == chunk_size:
-                self.enlarge()
-                write = 0
-            self.used_in_last_chunk += 1
-            write_chunk = self.chunk
-            while got < addr and not read_chunk is null_chunk:
-                write_chunk.items[write] = got
-                write -= 1
-                if write < 0:
-                    write_chunk = write_chunk.next
-                    write = chunk_size - 1
-                got = read_chunk.items[read]
-                read -= 1
-                if read < 0:
-                    read_chunk = read_chunk.next
-                    read = chunk_size - 1
-            if got < addr:
-                write_chunk.items[write] = got
-                write_chunk.items[0] = addr
-            else:
-                write_chunk.items[write] = addr
+        def sort(self):
+            ll_assert(self.chunk.next == null_chunk, "too big for sorting")
+            sort_chunk(self.chunk, self.used_in_last_chunk - 1)
 
     cache[chunk_size] = AddressStack
     return AddressStack
diff --git a/pypy/rpython/memory/test/test_support.py b/pypy/rpython/memory/test/test_support.py
--- a/pypy/rpython/memory/test/test_support.py
+++ b/pypy/rpython/memory/test/test_support.py
@@ -1,3 +1,5 @@
+
+import random
 from pypy.rlib.objectmodel import free_non_gc_object
 from pypy.rpython.memory.support import get_address_stack
 from pypy.rpython.memory.support import get_address_deque
@@ -94,25 +96,20 @@
             assert a == addrs[i]
         assert not ll.non_empty()
 
-    def test_insert(self):
-        AddressStack = get_address_stack(chunk_size=5)
-        ll = AddressStack()
-        lla = llarena.arena_malloc(10, 2)
-        addrs = [lla + i for i in range(10)]
-        ll.insert(addrs[2])
-        ll.insert(addrs[1])
-        ll.insert(addrs[5])
-        ll.insert(addrs[4])
-        ll.insert(addrs[6])
-        ll.insert(addrs[9])
-        ll.insert(addrs[0])
-        ll.insert(addrs[8])
-        ll.insert(addrs[7])
-        ll.insert(addrs[3])
-        expected = range(10)
-        for i in expected:
-            a = ll.pop()
-            assert a == addrs[i]
+    def test_sort(self):
+        AddressStack = get_address_stack(chunk_size=15)
+        for _ in range(13):
+            ll = AddressStack()
+            lla = llarena.arena_malloc(10, 2)
+            addrs = [lla + i for i in range(10)]
+            random.shuffle(addrs)
+            for i in addrs:
+                ll.append(i)
+            ll.sort()
+            expected = range(10)
+            for i in expected:
+                a = ll.pop()
+                assert a == addrs[i]
 
 
 class TestAddressDeque:

From noreply at buildbot.pypy.org  Sun Apr 15 20:38:09 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 20:38:09 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: fix the test
Message-ID: <20120415183809.36C29820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54377:d4a2fdd9a4ac
Date: 2012-04-15 20:37 +0200
http://bitbucket.org/pypy/pypy/changeset/d4a2fdd9a4ac/

Log:	fix the test

diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -986,5 +986,38 @@
         res = self.interpret(f, [10])
         assert res == 2
 
+    def test_pin_collect_tracking(self):
+        from pypy.rpython.lltypesystem import llmemory
+
+        S = lltype.GcStruct('S')
+        TP = lltype.GcStruct('x', ('x', lltype.Signed),
+                             ('y', lltype.Ptr(S)))
+        
+        def f(i):
+            e = lltype.malloc(TP)
+            e.x = 3
+            lltype.malloc(TP)
+            e2 = lltype.malloc(TP)
+            e2.x = 5
+            rgc.pin(e2)
+            rgc.pin(e)
+            prev = llmemory.cast_ptr_to_adr(e)
+            prev2 = llmemory.cast_ptr_to_adr(e2)
+            for k in range(i):
+                lltype.malloc(TP)
+            res = int(llmemory.cast_ptr_to_adr(e) == prev)
+            res += int(llmemory.cast_ptr_to_adr(e2) == prev2)
+            rgc.unpin(e)
+            for k in range(i):
+                lltype.malloc(TP)
+            rgc.unpin(e2)
+            assert e.x == 3 # noone overwrote it
+            assert e2.x == 5 # noone overwrote it
+            return res
+
+        res = self.interpret(f, [10])
+        assert res == 2
+        
+
 class TestMiniMarkGCCardMarking(TestMiniMarkGC):
     GC_PARAMS = {'card_page_indices': 4}
diff --git a/pypy/rpython/memory/test/test_support.py b/pypy/rpython/memory/test/test_support.py
--- a/pypy/rpython/memory/test/test_support.py
+++ b/pypy/rpython/memory/test/test_support.py
@@ -98,12 +98,13 @@
 
     def test_sort(self):
         AddressStack = get_address_stack(chunk_size=15)
+        lla = llarena.arena_malloc(10, 2)
+        addrs = [lla + i for i in range(10)]
         for _ in range(13):
             ll = AddressStack()
-            lla = llarena.arena_malloc(10, 2)
-            addrs = [lla + i for i in range(10)]
-            random.shuffle(addrs)
-            for i in addrs:
+            addr_copy = addrs[:]
+            random.shuffle(addr_copy)
+            for i in addr_copy:
                 ll.append(i)
             ll.sort()
             expected = range(10)

From noreply at buildbot.pypy.org  Sun Apr 15 20:46:19 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 20:46:19 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: grumble
Message-ID: <20120415184619.6E7FF820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54378:3b2c45587c05
Date: 2012-04-15 20:46 +0200
http://bitbucket.org/pypy/pypy/changeset/3b2c45587c05/

Log:	grumble

diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -197,7 +197,7 @@
 
         def sort(self):
             ll_assert(self.chunk.next == null_chunk, "too big for sorting")
-            sort_chunk(self.chunk, self.used_in_last_chunk - 1)
+            sort_chunk(self.chunk, self.used_in_last_chunk)
 
     cache[chunk_size] = AddressStack
     return AddressStack

From noreply at buildbot.pypy.org  Sun Apr 15 20:57:33 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 20:57:33 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: a failing test
Message-ID: <20120415185733.97461820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54379:98cefb78a238
Date: 2012-04-15 20:56 +0200
http://bitbucket.org/pypy/pypy/changeset/98cefb78a238/

Log:	a failing test

diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -989,34 +989,23 @@
     def test_pin_collect_tracking(self):
         from pypy.rpython.lltypesystem import llmemory
 
-        S = lltype.GcStruct('S')
+        S = lltype.GcStruct('S', ('x', lltype.Signed))
         TP = lltype.GcStruct('x', ('x', lltype.Signed),
                              ('y', lltype.Ptr(S)))
         
         def f(i):
-            e = lltype.malloc(TP)
-            e.x = 3
-            lltype.malloc(TP)
-            e2 = lltype.malloc(TP)
-            e2.x = 5
-            rgc.pin(e2)
-            rgc.pin(e)
-            prev = llmemory.cast_ptr_to_adr(e)
-            prev2 = llmemory.cast_ptr_to_adr(e2)
+            s = lltype.malloc(S)
+            c = lltype.malloc(TP)
+            c.y = s
+            c.x = 3
+            rgc.pin(c)
             for k in range(i):
                 lltype.malloc(TP)
-            res = int(llmemory.cast_ptr_to_adr(e) == prev)
-            res += int(llmemory.cast_ptr_to_adr(e2) == prev2)
-            rgc.unpin(e)
-            for k in range(i):
-                lltype.malloc(TP)
-            rgc.unpin(e2)
-            assert e.x == 3 # noone overwrote it
-            assert e2.x == 5 # noone overwrote it
-            return res
+            rgc.unpin(c)
+            return c.y == s
 
         res = self.interpret(f, [10])
-        assert res == 2
+        assert res == 1
         
 
 class TestMiniMarkGCCardMarking(TestMiniMarkGC):

From noreply at buildbot.pypy.org  Sun Apr 15 20:57:34 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 15 Apr 2012 20:57:34 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: another failing test
Message-ID: <20120415185734.DDA47820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54380:f1d2d360a809
Date: 2012-04-15 20:57 +0200
http://bitbucket.org/pypy/pypy/changeset/f1d2d360a809/

Log:	another failing test

diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -541,6 +541,8 @@
         self.gc.id(s) # allocate shadow
         self.gc.pin(llmemory.cast_ptr_to_adr(s))
         self.gc.minor_collection(1)
+        self.gc.unpin(llmemory.cast_ptr_to_adr(s))
+        self.gc.minor_collection(1)
         assert self.gc.nursery_free != self.gc.nursery
         # we still have a pinned object
 

From noreply at buildbot.pypy.org  Mon Apr 16 05:59:07 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 16 Apr 2012 05:59:07 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: mmap(-1,...) test and fix
Message-ID: <20120416035907.CBFFE820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54381:0691f4fdc66f
Date: 2012-04-16 06:58 +0300
http://bitbucket.org/pypy/pypy/changeset/0691f4fdc66f/

Log:	mmap(-1,...) test and fix

diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -739,6 +739,8 @@
         # assume -1 and 0 both mean invalid file descriptor
         # to 'anonymously' map memory.
         if fileno != -1 and fileno != 0:
+            if not rposix.validate_fd(fileno):
+                raise OSError(rposix.get_errno(), 'Bad file descriptor')
             fh = rwin32._get_osfhandle(fileno)
             if fh == INVALID_HANDLE:
                 errno = rposix.get_errno()
@@ -747,24 +749,24 @@
             # SEEK_SET = 0
             # libc._lseek(fileno, 0, SEEK_SET)
 
-        # check file size
-        try:
-            st = os.fstat(fileno)
-        except OSError:
-            pass     # ignore errors and trust map_size
-        else:
-            mode = st[stat.ST_MODE]
-            size = st[stat.ST_SIZE]
-            if stat.S_ISREG(mode):
-                if map_size == 0:
-                    if offset > size:
-                        raise RValueError(
-                            "mmap offset is greater than file size")
-                    map_size = int(size - offset)
-                    if map_size != size - offset:
-                        raise RValueError("mmap length is too large")
-                elif offset + map_size > size:
-                    raise RValueError("mmap length is greater than file size")
+            # check file size
+            try:
+                st = os.fstat(fileno)
+            except OSError:
+                pass     # ignore errors and trust map_size
+            else:
+                mode = st[stat.ST_MODE]
+                size = st[stat.ST_SIZE]
+                if stat.S_ISREG(mode):
+                    if map_size == 0:
+                        if offset > size:
+                            raise RValueError(
+                                "mmap offset is greater than file size")
+                        map_size = int(size - offset)
+                        if map_size != size - offset:
+                            raise RValueError("mmap length is too large")
+                    elif offset + map_size > size:
+                        raise RValueError("mmap length is greater than file size")
 
         m = MMap(access, offset)
         m.file_handle = INVALID_HANDLE
diff --git a/pypy/rlib/test/test_rmmap.py b/pypy/rlib/test/test_rmmap.py
--- a/pypy/rlib/test/test_rmmap.py
+++ b/pypy/rlib/test/test_rmmap.py
@@ -33,8 +33,6 @@
         interpret(f, [])
 
     def test_file_size(self):
-        if os.name == "nt":
-            skip("Only Unix checks file size")
         def func(no):
 
             try:
@@ -433,15 +431,16 @@
     def test_windows_crasher_1(self):
         if sys.platform != "win32":
             skip("Windows-only test")
-
-        m = mmap.mmap(-1, 1000, tagname="foo")
-        # same tagname, but larger size
-        try:
-            m2 = mmap.mmap(-1, 5000, tagname="foo")
-            m2.getitem(4500)
-        except WindowsError:
-            pass
-        m.close()
+        def func():
+            m = mmap.mmap(-1, 1000, tagname="foo")
+            # same tagname, but larger size
+            try:
+                m2 = mmap.mmap(-1, 5000, tagname="foo")
+                m2.getitem(4500)
+            except WindowsError:
+                pass
+            m.close()
+        interpret(func, [])
 
     def test_windows_crasher_2(self):
         if sys.platform != "win32":

From noreply at buildbot.pypy.org  Mon Apr 16 09:51:30 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon, 16 Apr 2012 09:51:30 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification/py3k: close broken merge
	branch
Message-ID: <20120416075130.140F4820B3@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification/py3k
Changeset: r54382:21a2a3102d8c
Date: 2012-04-16 09:50 +0200
http://bitbucket.org/pypy/pypy/changeset/21a2a3102d8c/

Log:	close broken merge branch

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -9,24 +9,35 @@
 
 class Signature(object):
     _immutable_ = True
-    _immutable_fields_ = ["argnames[*]"]
-    __slots__ = ("argnames", "varargname", "kwargname")
+    _immutable_fields_ = ["argnames[*]", "kwonlyargnames[*]"]
+    __slots__ = ("argnames", "kwonlyargnames", "varargname", "kwargname")
 
-    def __init__(self, argnames, varargname=None, kwargname=None):
+    def __init__(self, argnames, varargname=None, kwargname=None, kwonlyargnames=None):
         self.argnames = argnames
         self.varargname = varargname
         self.kwargname = kwargname
+        if kwonlyargnames is None:
+            kwonlyargnames = []
+        self.kwonlyargnames = kwonlyargnames
 
     @jit.elidable
     def find_argname(self, name):
         try:
             return self.argnames.index(name)
         except ValueError:
-            return -1
+            pass
+        try:
+            return len(self.argnames) + self.kwonlyargnames.index(name)
+        except ValueError:
+            pass
+        return -1
 
     def num_argnames(self):
         return len(self.argnames)
 
+    def num_kwonlyargnames(self):
+        return len(self.kwonlyargnames)
+
     def has_vararg(self):
         return self.varargname is not None
 
@@ -43,20 +54,22 @@
         argnames = self.argnames
         if self.varargname is not None:
             argnames = argnames + [self.varargname]
+        argnames = argnames + self.kwonlyargnames
         if self.kwargname is not None:
             argnames = argnames + [self.kwargname]
         return argnames
 
     def __repr__(self):
-        return "Signature(%r, %r, %r)" % (
-                self.argnames, self.varargname, self.kwargname)
+        return "Signature(%r, %r, %r, %r)" % (
+                self.argnames, self.varargname, self.kwargname, self.kwonlyargnames)
 
     def __eq__(self, other):
         if not isinstance(other, Signature):
             return NotImplemented
         return (self.argnames == other.argnames and
                 self.varargname == other.varargname and
-                self.kwargname == other.kwargname)
+                self.kwargname == other.kwargname and
+                self.kwonlyargnames == other.kwonlyargnames)
 
     def __ne__(self, other):
         if not isinstance(other, Signature):
@@ -169,11 +182,9 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
-        keywords, values_w = space.view_as_kwargs(w_starstararg)
-        if keywords is not None: # this path also taken for empty dicts
-            self._add_keywordargs_no_unwrapping(keywords, values_w)
-            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
+            if not space.is_true(w_starstararg):
+                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -188,8 +199,11 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-        return True
+        if keys_w:
+            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+            return True
+        else:
+            return False    # empty dict; don't disable the JIT
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -226,26 +240,6 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
-    @jit.look_inside_iff(lambda self, keywords, keywords_w:
-            jit.isconstant(len(keywords) and
-            jit.isconstant(self.keywords)))
-    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
-        if self.keywords is None:
-            self.keywords = keywords[:] # copy to make non-resizable
-            self.keywords_w = keywords_w[:]
-        else:
-            # looks quadratic, but the JIT should remove all of it nicely.
-            # Also, all the lists should be small
-            for key in keywords:
-                for otherkey in self.keywords:
-                    if otherkey == key:
-                        raise operationerrfmt(self.space.w_TypeError,
-                                              "got multiple values "
-                                              "for keyword argument "
-                                              "'%s'", key)
-            self.keywords = self.keywords + keywords
-            self.keywords_w = self.keywords_w + keywords_w
-
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -268,7 +262,7 @@
     # XXX: this should be @jit.look_inside_iff, but we need key word arguments,
     # and it doesn't support them for now.
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
-                         blindargs=0):
+                         w_kw_defs=None, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         Return the number of arguments filled in.
@@ -276,19 +270,19 @@
         if jit.we_are_jitted() and self._dont_jit:
             return self._match_signature_jit_opaque(w_firstarg, scope_w,
                                                     signature, defaults_w,
-                                                    blindargs)
+                                                    w_kw_defs, blindargs)
         return self._really_match_signature(w_firstarg, scope_w, signature,
-                                            defaults_w, blindargs)
+                                            defaults_w, w_kw_defs, blindargs)
 
     @jit.dont_look_inside
     def _match_signature_jit_opaque(self, w_firstarg, scope_w, signature,
-                                    defaults_w, blindargs):
+                                    defaults_w, w_kw_defs, blindargs):
         return self._really_match_signature(w_firstarg, scope_w, signature,
-                                            defaults_w, blindargs)
+                                            defaults_w, w_kw_defs, blindargs)
 
     @jit.unroll_safe
     def _really_match_signature(self, w_firstarg, scope_w, signature,
-                                defaults_w=None, blindargs=0):
+                                defaults_w=None, w_kw_defs=None, blindargs=0):
         #
         #   args_w = list of the normal actual parameters, wrapped
         #   kwds_w = real dictionary {'keyword': wrapped parameter}
@@ -300,6 +294,7 @@
         # so all values coming from there can be assumed constant. It assumes
         # that the length of the defaults_w does not vary too much.
         co_argcount = signature.num_argnames() # expected formal arguments, without */**
+        co_kwonlyargcount = signature.num_kwonlyargnames()
         has_vararg = signature.has_vararg()
         has_kwarg = signature.has_kwarg()
         extravarargs = None
@@ -351,11 +346,12 @@
                 starargs_w = args_w[args_left:]
             else:
                 starargs_w = []
-            scope_w[co_argcount] = self.space.newtuple(starargs_w)
+            loc = co_argcount + co_kwonlyargcount
+            scope_w[loc] = self.space.newtuple(starargs_w)
         elif avail > co_argcount:
             raise ArgErrCount(avail, num_kwds,
                               co_argcount, has_vararg, has_kwarg,
-                              defaults_w, 0)
+                              defaults_w, w_kw_defs, 0)
 
         # the code assumes that keywords can potentially be large, but that
         # argnames is typically not too large
@@ -388,7 +384,7 @@
                     used_keywords[i] = True # mark as used
                     num_remainingkwds -= 1
         missing = 0
-        if input_argcount < co_argcount:
+        if input_argcount < co_argcount + co_kwonlyargcount:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
             for i in range(input_argcount, co_argcount):
                 if scope_w[i] is not None:
@@ -401,10 +397,27 @@
                     # because it might be related to a problem with */** or
                     # keyword arguments, which will be checked for below.
                     missing += 1
+            for i in range(co_argcount, co_argcount + co_kwonlyargcount):
+                if scope_w[i] is not None:
+                    continue
+                elif w_kw_defs is None:
+                    missing += 1
+                    continue
+                name = signature.kwonlyargnames[i - co_argcount]
+                w_name = self.space.wrap(name)
+                w_def = self.space.finditem(w_kw_defs, w_name)
+                if w_def is not None:
+                    scope_w[i] = w_def
+                else:
+                    missing += 1
+
+        # TODO: Put a nice error message
+        #if co_kwonlyargcount:
+        #    assert co_kwonlyargcount == len(signature.kwonlyargnames)
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict(kwargs=True)
+            w_kwds = self.space.newdict()
             if num_remainingkwds:
                 #
                 limit = len(keywords)
@@ -418,26 +431,27 @@
                             w_key = self.keyword_names_w[i - limit]
                         self.space.setitem(w_kwds, w_key, keywords_w[i])
                 #
-            scope_w[co_argcount + has_vararg] = w_kwds
+            scope_w[co_argcount + co_kwonlyargcount + has_vararg] = w_kwds
         elif num_remainingkwds:
             if co_argcount == 0:
                 raise ArgErrCount(avail, num_kwds,
                               co_argcount, has_vararg, has_kwarg,
-                              defaults_w, missing)
+                              defaults_w, w_kw_defs, missing)
             raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
                                     used_keywords, self.keyword_names_w)
 
         if missing:
             raise ArgErrCount(avail, num_kwds,
                               co_argcount, has_vararg, has_kwarg,
-                              defaults_w, missing)
+                              defaults_w, w_kw_defs, missing)
 
-        return co_argcount + has_vararg + has_kwarg
+        return co_argcount + has_vararg + has_kwarg + co_kwonlyargcount
 
 
 
     def parse_into_scope(self, w_firstarg,
-                         scope_w, fnname, signature, defaults_w=None):
+                         scope_w, fnname, signature, defaults_w=None,
+                         w_kw_defs=None):
         """Parse args and kwargs to initialize a frame
         according to the signature of code object.
         Store the argumentvalues into scope_w.
@@ -445,29 +459,32 @@
         """
         try:
             return self._match_signature(w_firstarg,
-                                         scope_w, signature, defaults_w, 0)
+                                         scope_w, signature, defaults_w,
+                                         w_kw_defs, 0)
         except ArgErr, e:
             raise operationerrfmt(self.space.w_TypeError,
                                   "%s() %s", fnname, e.getmsg())
 
-    def _parse(self, w_firstarg, signature, defaults_w, blindargs=0):
+    def _parse(self, w_firstarg, signature, defaults_w, w_kw_defs, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         """
         scopelen = signature.scope_length()
         scope_w = [None] * scopelen
         self._match_signature(w_firstarg, scope_w, signature, defaults_w,
-                              blindargs)
+                              w_kw_defs, blindargs)
         return scope_w
 
 
     def parse_obj(self, w_firstarg,
-                  fnname, signature, defaults_w=None, blindargs=0):
+                  fnname, signature, defaults_w=None, w_kw_defs=None,
+                  blindargs=0):
         """Parse args and kwargs to initialize a frame
         according to the signature of code object.
         """
         try:
-            return self._parse(w_firstarg, signature, defaults_w, blindargs)
+            return self._parse(w_firstarg, signature, defaults_w, w_kw_defs,
+                               blindargs)
         except ArgErr, e:
             raise operationerrfmt(self.space.w_TypeError,
                                   "%s() %s", fnname, e.getmsg())
@@ -523,22 +540,22 @@
 
 
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
-                         blindargs=0):
+                         w_kw_defs=None, blindargs=0):
         self.combine_if_necessary()
         # _match_signature is destructive
         return Arguments._match_signature(
                self, w_firstarg, scope_w, signature,
-               defaults_w, blindargs)
+               defaults_w, w_kw_defs, blindargs)
 
     def unpack(self):
         self.combine_if_necessary()
         return Arguments.unpack(self)
 
-    def match_signature(self, signature, defaults_w):
+    def match_signature(self, signature, defaults_w, w_kw_defs=None):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         """
-        return self._parse(None, signature, defaults_w)
+        return self._parse(None, signature, defaults_w, w_kw_defs)
 
     def unmatch_signature(self, signature, data_w):
         """kind of inverse of match_signature"""
@@ -651,7 +668,7 @@
 class ArgErrCount(ArgErr):
 
     def __init__(self, got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
-                 defaults_w, missing_args):
+                 defaults_w, w_kw_defs, missing_args):
         self.expected_nargs = expected_nargs
         self.has_vararg = has_vararg
         self.has_kwarg = has_kwarg
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -10,6 +10,16 @@
 from pypy.interpreter.astcompiler import ast, consts
 
 
+try:
+    all
+except NameError:
+    def all(iterable):
+        for x in iterable:
+            if not x:
+                return False
+        return True
+
+
 class TestAstBuilder:
 
     def setup_class(cls):
@@ -60,33 +70,6 @@
         for stmt in mod.body:
             assert isinstance(stmt, ast.Assign)
 
-    def test_print(self):
-        pri = self.get_first_stmt("print x")
-        assert isinstance(pri, ast.Print)
-        assert pri.dest is None
-        assert pri.nl
-        assert len(pri.values) == 1
-        assert isinstance(pri.values[0], ast.Name)
-        pri = self.get_first_stmt("print x, 34")
-        assert len(pri.values) == 2
-        assert isinstance(pri.values[0], ast.Name)
-        assert isinstance(pri.values[1], ast.Num)
-        pri = self.get_first_stmt("print")
-        assert pri.nl
-        assert pri.values is None
-        pri = self.get_first_stmt("print x,")
-        assert len(pri.values) == 1
-        assert not pri.nl
-        pri = self.get_first_stmt("print >> y, 4")
-        assert isinstance(pri.dest, ast.Name)
-        assert len(pri.values) == 1
-        assert isinstance(pri.values[0], ast.Num)
-        assert pri.nl
-        pri = self.get_first_stmt("print >> y")
-        assert isinstance(pri.dest, ast.Name)
-        assert pri.values is None
-        assert pri.nl
-
     def test_del(self):
         d = self.get_first_stmt("del x")
         assert isinstance(d, ast.Delete)
@@ -125,21 +108,14 @@
 
     def test_raise(self):
         ra = self.get_first_stmt("raise")
-        assert ra.type is None
-        assert ra.inst is None
-        assert ra.tback is None
+        assert ra.exc is None
+        assert ra.cause is None
         ra = self.get_first_stmt("raise x")
-        assert isinstance(ra.type, ast.Name)
-        assert ra.inst is None
-        assert ra.tback is None
-        ra = self.get_first_stmt("raise x, 3")
-        assert isinstance(ra.type, ast.Name)
-        assert isinstance(ra.inst, ast.Num)
-        assert ra.tback is None
-        ra = self.get_first_stmt("raise x, 4, 'hi'")
-        assert isinstance(ra.type, ast.Name)
-        assert isinstance(ra.inst, ast.Num)
-        assert isinstance(ra.tback, ast.Str)
+        assert isinstance(ra.exc, ast.Name)
+        assert ra.cause is None
+        ra = self.get_first_stmt("raise x from 3")
+        assert isinstance(ra.exc, ast.Name)
+        assert isinstance(ra.cause, ast.Num)
 
     def test_import(self):
         im = self.get_first_stmt("import x")
@@ -225,20 +201,12 @@
         glob = self.get_first_stmt("global x, y")
         assert glob.names == ["x", "y"]
 
-    def test_exec(self):
-        exc = self.get_first_stmt("exec x")
-        assert isinstance(exc, ast.Exec)
-        assert isinstance(exc.body, ast.Name)
-        assert exc.globals is None
-        assert exc.locals is None
-        exc = self.get_first_stmt("exec 'hi' in x")
-        assert isinstance(exc.body, ast.Str)
-        assert isinstance(exc.globals, ast.Name)
-        assert exc.locals is None
-        exc = self.get_first_stmt("exec 'hi' in x, 2")
-        assert isinstance(exc.body, ast.Str)
-        assert isinstance(exc.globals, ast.Name)
-        assert isinstance(exc.locals, ast.Num)
+    def test_nonlocal(self):
+        nonloc = self.get_first_stmt("nonlocal x")
+        assert isinstance(nonloc, ast.Nonlocal)
+        assert nonloc.names == ["x"]
+        nonloc = self.get_first_stmt("nonlocal x, y")
+        assert nonloc.names == ["x", "y"]
 
     def test_assert(self):
         asrt = self.get_first_stmt("assert x")
@@ -343,13 +311,15 @@
         assert isinstance(fr.orelse[0].value, ast.Num)
 
     def test_try(self):
-        tr = self.get_first_stmt("try: x\nfinally: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "finally: pass")
         assert isinstance(tr, ast.TryFinally)
         assert len(tr.body) == 1
         assert isinstance(tr.body[0].value, ast.Name)
         assert len(tr.finalbody) == 1
         assert isinstance(tr.finalbody[0], ast.Pass)
-        tr = self.get_first_stmt("try: x\nexcept: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except: pass")
         assert isinstance(tr, ast.TryExcept)
         assert len(tr.body) == 1
         assert isinstance(tr.body[0].value, ast.Name)
@@ -361,7 +331,8 @@
         assert len(handler.body) == 1
         assert isinstance(handler.body[0], ast.Pass)
         assert tr.orelse is None
-        tr = self.get_first_stmt("try: x\nexcept Exception: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except Exception: pass")
         assert len(tr.handlers) == 1
         handler = tr.handlers[0]
         assert isinstance(handler.type, ast.Name)
@@ -369,40 +340,48 @@
         assert handler.name is None
         assert len(handler.body) == 1
         assert tr.orelse is None
-        tr = self.get_first_stmt("try: x\nexcept Exception, e: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except Exception as e: pass")
         assert len(tr.handlers) == 1
         handler = tr.handlers[0]
         assert isinstance(handler.type, ast.Name)
-        assert isinstance(handler.name, ast.Name)
-        assert handler.name.ctx == ast.Store
-        assert handler.name.id == "e"
+        assert handler.type.id == "Exception"
+        assert handler.name == "e"
         assert len(handler.body) == 1
-        tr = self.get_first_stmt("try: x\nexcept: pass\nelse: 4")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except: pass" + "\n" +
+                                 "else: 4")
         assert len(tr.body) == 1
         assert isinstance(tr.body[0].value, ast.Name)
         assert len(tr.handlers) == 1
         assert isinstance(tr.handlers[0].body[0], ast.Pass)
         assert len(tr.orelse) == 1
         assert isinstance(tr.orelse[0].value, ast.Num)
-        tr = self.get_first_stmt("try: x\nexcept Exc, a: 5\nexcept F: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except Exc as a: 5" + "\n" +
+                                 "except F: pass")
         assert len(tr.handlers) == 2
         h1, h2 = tr.handlers
         assert isinstance(h1.type, ast.Name)
-        assert isinstance(h1.name, ast.Name)
+        assert h1.name == "a"
         assert isinstance(h1.body[0].value, ast.Num)
         assert isinstance(h2.type, ast.Name)
         assert h2.name is None
         assert isinstance(h2.body[0], ast.Pass)
-        tr = self.get_first_stmt("try: x\nexcept Exc as a: 5\nexcept F: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except Exc as a: 5" + "\n" +
+                                 "except F: pass")
         assert len(tr.handlers) == 2
         h1, h2 = tr.handlers
         assert isinstance(h1.type, ast.Name)
-        assert isinstance(h1.name, ast.Name)
+        assert h1.name == "a"
         assert isinstance(h1.body[0].value, ast.Num)
         assert isinstance(h2.type, ast.Name)
         assert h2.name is None
         assert isinstance(h2.body[0], ast.Pass)
-        tr = self.get_first_stmt("try: x\nexcept: 4\nfinally: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except: 4" + "\n" +
+                                 "finally: pass")
         assert isinstance(tr, ast.TryFinally)
         assert len(tr.finalbody) == 1
         assert isinstance(tr.finalbody[0], ast.Pass)
@@ -414,7 +393,10 @@
         assert isinstance(exc.handlers[0].body[0].value, ast.Num)
         assert len(exc.body) == 1
         assert isinstance(exc.body[0].value, ast.Name)
-        tr = self.get_first_stmt("try: x\nexcept: 4\nelse: 'hi'\nfinally: pass")
+        tr = self.get_first_stmt("try: x" + "\n" +
+                                 "except: 4" + "\n" +
+                                 "else: 'hi'" + "\n" +
+                                 "finally: pass")
         assert isinstance(tr, ast.TryFinally)
         assert len(tr.finalbody) == 1
         assert isinstance(tr.finalbody[0], ast.Pass)
@@ -496,72 +478,40 @@
         assert args.defaults is None
         assert args.kwarg is None
         assert args.vararg is None
+        assert func.returns is None
         args = self.get_first_stmt("def f(a, b): pass").args
         assert len(args.args) == 2
         a1, a2 = args.args
-        assert isinstance(a1, ast.Name)
-        assert a1.id == "a"
-        assert a1.ctx == ast.Param
-        assert isinstance(a2, ast.Name)
-        assert a2.id == "b"
-        assert a2.ctx == ast.Param
+        assert isinstance(a1, ast.arg)
+        assert a1.arg == "a"
+        assert isinstance(a2, ast.arg)
+        assert a2.arg == "b"
         assert args.vararg is None
         assert args.kwarg is None
         args = self.get_first_stmt("def f(a=b): pass").args
         assert len(args.args) == 1
         arg = args.args[0]
-        assert isinstance(arg, ast.Name)
-        assert arg.id == "a"
-        assert arg.ctx == ast.Param
+        assert isinstance(arg, ast.arg)
+        assert arg.arg == "a"
         assert len(args.defaults) == 1
         default = args.defaults[0]
         assert isinstance(default, ast.Name)
         assert default.id == "b"
         assert default.ctx == ast.Load
         args = self.get_first_stmt("def f(*a): pass").args
-        assert args.args is None
-        assert args.defaults is None
+        assert not args.args
+        assert not args.defaults
         assert args.kwarg is None
         assert args.vararg == "a"
         args = self.get_first_stmt("def f(**a): pass").args
-        assert args.args is None
-        assert args.defaults is None
+        assert not args.args
+        assert not args.defaults
         assert args.vararg is None
         assert args.kwarg == "a"
-        args = self.get_first_stmt("def f((a, b)): pass").args
-        assert args.defaults is None
-        assert args.kwarg is None
-        assert args.vararg is None
-        assert len(args.args) == 1
-        tup = args.args[0]
-        assert isinstance(tup, ast.Tuple)
-        assert tup.ctx == ast.Store
-        assert len(tup.elts) == 2
-        e1, e2 = tup.elts
-        assert isinstance(e1, ast.Name)
-        assert e1.ctx == ast.Store
-        assert e1.id == "a"
-        assert isinstance(e2, ast.Name)
-        assert e2.ctx == ast.Store
-        assert e2.id == "b"
-        args = self.get_first_stmt("def f((a, (b, c))): pass").args
-        assert len(args.args) == 1
-        tup = args.args[0]
-        assert isinstance(tup, ast.Tuple)
-        assert len(tup.elts) == 2
-        tup2 = tup.elts[1]
-        assert isinstance(tup2, ast.Tuple)
-        assert tup2.ctx == ast.Store
-        for elt in tup2.elts:
-            assert isinstance(elt, ast.Name)
-            assert elt.ctx == ast.Store
-        assert tup2.elts[0].id == "b"
-        assert tup2.elts[1].id == "c"
         args = self.get_first_stmt("def f(a, b, c=d, *e, **f): pass").args
         assert len(args.args) == 3
         for arg in args.args:
-            assert isinstance(arg, ast.Name)
-            assert arg.ctx == ast.Param
+            assert isinstance(arg, ast.arg)
         assert len(args.defaults) == 1
         assert isinstance(args.defaults[0], ast.Name)
         assert args.defaults[0].ctx == ast.Load
@@ -570,9 +520,65 @@
         input = "def f(a=b, c): pass"
         exc = py.test.raises(SyntaxError, self.get_ast, input).value
         assert exc.msg == "non-default argument follows default argument"
-        input = "def f((x)=23): pass"
+
+    def test_kwonly_arguments(self):
+        fn = self.get_first_stmt("def f(a, b, c, *, kwarg): pass")
+        assert isinstance(fn, ast.FunctionDef)
+        assert len(fn.args.kwonlyargs) == 1
+        assert isinstance(fn.args.kwonlyargs[0], ast.arg)
+        assert fn.args.kwonlyargs[0].arg == "kwarg"
+        assert fn.args.kw_defaults == [None]
+        fn = self.get_first_stmt("def f(a, b, c, *args, kwarg): pass")
+        assert isinstance(fn, ast.FunctionDef)
+        assert len(fn.args.kwonlyargs) == 1
+        assert isinstance(fn.args.kwonlyargs[0], ast.arg)
+        assert fn.args.kwonlyargs[0].arg == "kwarg"
+        assert fn.args.kw_defaults == [None]
+        fn = self.get_first_stmt("def f(a, b, c, *, kwarg=2): pass")
+        assert isinstance(fn, ast.FunctionDef)
+        assert len(fn.args.kwonlyargs) == 1
+        assert isinstance(fn.args.kwonlyargs[0], ast.arg)
+        assert fn.args.kwonlyargs[0].arg == "kwarg"
+        assert len(fn.args.kw_defaults) == 1
+        assert isinstance(fn.args.kw_defaults[0], ast.Num)
+        input = "def f(p1, *, **k1):  pass"
         exc = py.test.raises(SyntaxError, self.get_ast, input).value
-        assert exc.msg == "parenthesized arg with default"
+        assert exc.msg == "named arguments must follows bare *"
+
+    def test_function_annotation(self):
+        func = self.get_first_stmt("def f() -> X: pass")
+        assert isinstance(func.returns, ast.Name)
+        assert func.returns.id == "X"
+        assert func.returns.ctx == ast.Load
+        for stmt in "def f(x : 42): pass", "def f(x : 42=a): pass":
+            func = self.get_first_stmt(stmt)
+            assert isinstance(func.args.args[0].annotation, ast.Num)
+        assert isinstance(func.args.defaults[0], ast.Name)
+        func = self.get_first_stmt("def f(*x : 42): pass")
+        assert isinstance(func.args.varargannotation, ast.Num)
+        func = self.get_first_stmt("def f(**kw : 42): pass")
+        assert isinstance(func.args.kwargannotation, ast.Num)
+        func = self.get_first_stmt("def f(*, kw : 42=a): pass")
+        assert isinstance(func.args.kwonlyargs[0].annotation, ast.Num)
+
+    def test_lots_of_kwonly_arguments(self):
+        fundef = "def f("
+        for i in range(255):
+            fundef += "i%d, "%i
+        fundef += "*, key=100):\n pass\n"
+        py.test.raises(SyntaxError, self.get_first_stmt, fundef)
+
+        fundef2 = "def foo(i,*,"
+        for i in range(255):
+            fundef2 += "i%d, "%i
+        fundef2 += "lastarg):\n  pass\n"
+        py.test.raises(SyntaxError, self.get_first_stmt, fundef)
+
+        fundef3 = "def f(i,*,"
+        for i in range(253):
+            fundef3 += "i%d, "%i
+        fundef3 += "lastarg):\n  pass\n"
+        self.get_first_stmt(fundef3)
 
     def test_decorators(self):
         to_examine = (("def f(): pass", ast.FunctionDef),
@@ -664,6 +670,17 @@
         assert isinstance(tup.elts[0], ast.Name)
         assert tup.elts[0].ctx == ast.Store
 
+    def test_assign_starred(self):
+        assign = self.get_first_stmt("*a, b = x")
+        assert isinstance(assign, ast.Assign)
+        assert len(assign.targets) == 1
+        names = assign.targets[0]
+        assert len(names.elts) == 2
+        assert isinstance(names.elts[0], ast.Starred)
+        assert isinstance(names.elts[1], ast.Name)
+        assert isinstance(names.elts[0].value, ast.Name)
+        assert names.elts[0].value.id == "a"
+
     def test_name(self):
         name = self.get_first_expr("hi")
         assert isinstance(name, ast.Name)
@@ -744,7 +761,6 @@
             ("{x for x in z}", "set comprehension"),
             ("{x : x for x in z}", "dict comprehension"),
             ("'str'", "literal"),
-            ("u'str'", "literal"),
             ("b'bytes'", "literal"),
             ("()", "()"),
             ("23", "literal"),
@@ -752,7 +768,7 @@
             ("{1, 2, 3}", "literal"),
             ("(x > 4)", "comparison"),
             ("(x if y else a)", "conditional expression"),
-            ("`x`", "repr")
+            ("...", "Ellipsis"),
         )
         test_contexts = (
             ("assign to", "%s = 23"),
@@ -782,15 +798,11 @@
             "from x import y as %s",
             "for %s in x: pass",
         )
-        for name in ("None", "__debug__"):
+        for name in "__debug__",:
             for template in invalid:
                 input = template % (name,)
                 exc = py.test.raises(SyntaxError, self.get_ast, input).value
                 assert exc.msg == "cannot assign to %s" % (name,)
-        # This is ok.
-        self.get_ast("from None import x")
-        self.get_ast("from x import None as y")
-        self.get_ast("import None as x")
 
     def test_lambda(self):
         lam = self.get_first_expr("lambda x: expr")
@@ -799,13 +811,13 @@
         assert isinstance(args, ast.arguments)
         assert args.vararg is None
         assert args.kwarg is None
-        assert args.defaults is None
+        assert not args.defaults
         assert len(args.args) == 1
-        assert isinstance(args.args[0], ast.Name)
+        assert isinstance(args.args[0], ast.arg)
         assert isinstance(lam.body, ast.Name)
         lam = self.get_first_expr("lambda: True")
         args = lam.args
-        assert args.args is None
+        assert not args.args
         lam = self.get_first_expr("lambda x=x: y")
         assert len(lam.args.args) == 1
         assert len(lam.args.defaults) == 1
@@ -1024,9 +1036,7 @@
         slc = self.get_first_expr("x[::]").slice
         assert slc.upper is None
         assert slc.lower is None
-        assert isinstance(slc.step, ast.Name)
-        assert slc.step.id == "None"
-        assert slc.step.ctx == ast.Load
+        assert slc.step is None
         slc = self.get_first_expr("x[1:]").slice
         assert isinstance(slc.lower, ast.Num)
         assert slc.upper is None
@@ -1034,7 +1044,7 @@
         slc = self.get_first_expr("x[1::]").slice
         assert isinstance(slc.lower, ast.Num)
         assert slc.upper is None
-        assert isinstance(slc.step, ast.Name)
+        assert slc.step is None
         slc = self.get_first_expr("x[:2]").slice
         assert slc.lower is None
         assert isinstance(slc.upper, ast.Num)
@@ -1042,7 +1052,7 @@
         slc = self.get_first_expr("x[:2:]").slice
         assert slc.lower is None
         assert isinstance(slc.upper, ast.Num)
-        assert isinstance(slc.step, ast.Name)
+        assert slc.step is None
         slc = self.get_first_expr("x[2:2]").slice
         assert isinstance(slc.lower, ast.Num)
         assert isinstance(slc.upper, ast.Num)
@@ -1050,7 +1060,7 @@
         slc = self.get_first_expr("x[2:2:]").slice
         assert isinstance(slc.lower, ast.Num)
         assert isinstance(slc.upper, ast.Num)
-        assert isinstance(slc.step, ast.Name)
+        assert slc.step is None
         slc = self.get_first_expr("x[::2]").slice
         assert slc.lower is None
         assert slc.upper is None
@@ -1066,8 +1076,6 @@
         slc = self.get_first_expr("x[1:2:3]").slice
         for field in (slc.lower, slc.upper, slc.step):
             assert isinstance(field, ast.Num)
-        sub = self.get_first_expr("x[...]")
-        assert isinstance(sub.slice, ast.Ellipsis)
         sub = self.get_first_expr("x[1,2,3]")
         slc = sub.slice
         assert isinstance(slc, ast.Index)
@@ -1083,10 +1091,11 @@
         assert isinstance(complex_slc.upper, ast.Num)
         assert complex_slc.step is None
 
-    def test_repr(self):
-        rep = self.get_first_expr("`x`")
-        assert isinstance(rep, ast.Repr)
-        assert isinstance(rep.value, ast.Name)
+    def test_ellipsis(self):
+        e = self.get_first_expr("...")
+        assert isinstance(e, ast.Ellipsis)
+        sub = self.get_first_expr("x[...]")
+        assert isinstance(sub.slice.value, ast.Ellipsis)
 
     def test_string(self):
         space = self.space
@@ -1096,8 +1105,12 @@
         s = self.get_first_expr("'hi' ' implicitly' ' extra'")
         assert isinstance(s, ast.Str)
         assert space.eq_w(s.s, space.wrap("hi implicitly extra"))
+        s = self.get_first_expr("b'hi' b' implicitly' b' extra'")
+        assert isinstance(s, ast.Bytes)
+        assert space.eq_w(s.s, space.wrapbytes("hi implicitly extra"))
+        raises(SyntaxError, self.get_first_expr, "b'hello' 'world'")
         sentence = u"Die M&#228;nner &#228;rgen sich!"
-        source = u"# coding: utf-7\nstuff = u'%s'" % (sentence,)
+        source = u"# coding: utf-7\nstuff = '%s'" % (sentence,)
         info = pyparse.CompileInfo("<test>", "exec")
         tree = self.parser.parse_source(source.encode("utf-7"), info)
         assert info.encoding == "utf-7"
@@ -1105,6 +1118,17 @@
         assert isinstance(s, ast.Str)
         assert space.eq_w(s.s, space.wrap(sentence))
 
+    def test_string_pep3120(self):
+        space = self.space
+        japan = u'&#26085;&#26412;'
+        source = u"foo = '%s'" % japan
+        info = pyparse.CompileInfo("<test>", "exec")
+        tree = self.parser.parse_source(source.encode("utf-8"), info)
+        assert info.encoding == "utf-8"
+        s = ast_from_node(space, tree, info).body[0].value
+        assert isinstance(s, ast.Str)
+        assert space.eq_w(s.s, space.wrap(japan))
+
     def test_number(self):
         def get_num(s):
             node = self.get_first_expr(s)
@@ -1115,14 +1139,11 @@
         space = self.space
         assert space.eq_w(get_num("32"), space.wrap(32))
         assert space.eq_w(get_num("32.5"), space.wrap(32.5))
-        assert space.eq_w(get_num("32L"), space.newlong(32))
-        assert space.eq_w(get_num("32l"), space.newlong(32))
-        assert space.eq_w(get_num("0L"), space.newlong(0))
         assert space.eq_w(get_num("2"), space.wrap(2))
         assert space.eq_w(get_num("13j"), space.wrap(13j))
         assert space.eq_w(get_num("13J"), space.wrap(13J))
-        assert space.eq_w(get_num("053"), space.wrap(053))
-        assert space.eq_w(get_num("00053"), space.wrap(053))
+        assert space.eq_w(get_num("0o53"), space.wrap(053))
+        assert space.eq_w(get_num("0o0053"), space.wrap(053))
         for num in ("0x53", "0X53", "0x0000053", "0X00053"):
             assert space.eq_w(get_num(num), space.wrap(0x53))
         assert space.eq_w(get_num("0Xb0d2"), space.wrap(0xb0d2))
@@ -1131,7 +1152,7 @@
         assert space.eq_w(get_num("00000"), space.wrap(0))
         assert space.eq_w(get_num("-3"), space.wrap(-3))
         assert space.eq_w(get_num("-0"), space.wrap(0))
-        assert space.eq_w(get_num("-0xAAAAAAL"), space.wrap(-0xAAAAAAL))
+        assert space.eq_w(get_num("-0xAAAAAA"), space.wrap(-0xAAAAAAL))
         n = get_num(str(-sys.maxint - 1))
         assert space.is_true(space.isinstance(n, space.w_int))
         for num in ("0o53", "0O53", "0o0000053", "0O00053"):
@@ -1142,6 +1163,12 @@
         py.test.raises(SyntaxError, self.get_ast, "0x")
         py.test.raises(SyntaxError, self.get_ast, "0b")
         py.test.raises(SyntaxError, self.get_ast, "0o")
+        py.test.raises(SyntaxError, self.get_ast, "32L")
+        py.test.raises(SyntaxError, self.get_ast, "32l")
+        py.test.raises(SyntaxError, self.get_ast, "0L")
+        py.test.raises(SyntaxError, self.get_ast, "-0xAAAAAAL")
+        py.test.raises(SyntaxError, self.get_ast, "053")
+        py.test.raises(SyntaxError, self.get_ast, "00053")
 
     def check_comprehension(self, brackets, ast_type):
         def brack(s):
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1,9 +1,11 @@
+from __future__ import division
 import py
 from pypy.interpreter.astcompiler import codegen, astbuilder, symtable, optimize
 from pypy.interpreter.pyparser import pyparse
 from pypy.interpreter.pyparser.test import expressions
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter.pyparser.error import SyntaxError, IndentationError
+from pypy.interpreter.error import OperationError
 from pypy.tool import stdlib_opcode as ops
 
 def compile_with_astcompiler(expr, mode, space):
@@ -40,15 +42,31 @@
         source = str(py.code.Source(source))
         space = self.space
         code = compile_with_astcompiler(source, 'exec', space)
-        # 2.7 bytecode is too different, the standard `dis` module crashes
+        # 3.2 bytecode is too different, the standard `dis` module crashes
         # on older cpython versions
-        if sys.version_info >= (2, 7):
+        if sys.version_info >= (3, 2):
+            # this will only (maybe) work in the far future, when we run pypy
+            # on top of Python 3. For now, it's just disabled
             print
             code.dump()
         w_dict = space.newdict()
         code.exec_code(space, w_dict, w_dict)
         return w_dict
 
+    # on Python3 some reprs are different than Python2. Here is a collection
+    # of how the repr should be on on Python3 for some objects
+    PY3_REPR = {
+        int: "<class 'int'>",
+        float: "<class 'float'>",
+        }
+
+    def get_py3_repr(self, val):
+        try:
+            return self.PY3_REPR.get(val, repr(val))
+        except TypeError:
+            # e.g., for unhashable types
+            return repr(val)
+
     def check(self, w_dict, evalexpr, expected):
         # for now, we compile evalexpr with CPython's compiler but run
         # it with our own interpreter to extract the data from w_dict
@@ -57,13 +75,16 @@
         pyco_expr = PyCode._from_code(space, co_expr)
         w_res = pyco_expr.exec_host_bytecode(w_dict, w_dict)
         res = space.str_w(space.repr(w_res))
-        if not isinstance(expected, float):
-            noL = lambda expr: expr.replace('L', '')
-            assert noL(res) == noL(repr(expected))
-        else:
+        expected_repr = self.get_py3_repr(expected)
+        if isinstance(expected, float):
             # Float representation can vary a bit between interpreter
             # versions, compare the numbers instead.
             assert eval(res) == expected
+        elif isinstance(expected, long):
+            assert expected_repr.endswith('L')
+            assert res == expected_repr[:-1] # in py3 we don't have the L suffix
+        else:
+            assert res == expected_repr
 
     def simple_test(self, source, evalexpr, expected):
         w_g = self.run(source)
@@ -89,12 +110,12 @@
         yield self.st, func, "f(0)", 0
 
     def test_argtuple(self):
-        yield (self.simple_test, "def f( x, (y,z) ): return x,y,z",
-               "f((1,2),(3,4))", ((1,2),3,4))
-        yield (self.simple_test, "def f( x, (y,(z,t)) ): return x,y,z,t",
-               "f(1,(2,(3,4)))", (1,2,3,4))
-        yield (self.simple_test, "def f(((((x,),y),z),t),u): return x,y,z,t,u",
-               "f(((((1,),2),3),4),5)", (1,2,3,4,5))
+        yield (self.error_test, "def f( x, (y,z) ): return x,y,z",
+               SyntaxError)
+        yield (self.error_test, "def f( x, (y,(z,t)) ): return x,y,z,t",
+               SyntaxError)
+        yield (self.error_test, "def f(((((x,),y),z),t),u): return x,y,z,t,u",
+               SyntaxError)
 
     def test_constants(self):
         for c in expressions.constants:
@@ -223,8 +244,7 @@
     def test_funccalls(self):
         decl = py.code.Source("""
             def f(*args, **kwds):
-                kwds = kwds.items()
-                kwds.sort()
+                kwds = sorted(kwds.items())
                 return list(args) + kwds
         """)
         decl = str(decl) + '\n'
@@ -236,6 +256,16 @@
         yield self.st, decl + "x=f(5, b=2, **{'a': 8})", "x", [5, ('a', 8),
                                                                   ('b', 2)]
 
+    def test_kwonly(self):
+        decl = py.code.Source("""
+            def f(a, *, b):
+                return a, b
+        """)
+        decl = str(decl) + '\n'
+        self.st(decl + "x=f(1, b=2)", "x", (1, 2))
+        operr = py.test.raises(OperationError, 'self.st(decl + "x=f(1, 2)", "x", (1, 2))')
+        assert operr.value.w_type is self.space.w_TypeError
+
     def test_listmakers(self):
         yield (self.st,
                "l = [(j, i) for j in range(10) for i in range(j)"
@@ -315,7 +345,7 @@
                     from __foo__.bar import x
             try:
                 A().m()
-            except ImportError, e:
+            except ImportError as e:
                 msg = str(e)
             ''', "msg", "No module named __foo__")
 
@@ -377,13 +407,13 @@
                     return a
              ''',                            "foo docstring"),
             ('''def foo():
-                    """doc"""; print 1
+                    """doc"""; assert 1
                     a=1
              ''',                            "doc"),
             ('''
                 class Foo(object): pass
                 foo = Foo()
-                exec "'moduledoc'" in foo.__dict__
+                exec("'moduledoc'", foo.__dict__)
              ''',                            "moduledoc"),
             ]:
             yield self.simple_test, source, "foo.__doc__", expected
@@ -448,16 +478,6 @@
         yield self.st, decl, 'A,A1,A2,B2,C,C1,C2,D1,E,G,G1,G2,N1', \
                              (6,6 ,4 ,1 ,5,5 ,5 ,3 ,8,2,2 ,2 ,7 )
 
-        decl = py.code.Source("""
-            def f((a, b)):
-                def g((c, d)):
-                    return (a, b, c, d)
-                return g
-            x = f((1, 2))((3, 4))
-        """)
-        decl = str(decl) + "\n"
-        yield self.st, decl, 'x', (1, 2, 3, 4)
-
         source = """if 1:
         def f(a):
             del a
@@ -508,7 +528,7 @@
                 else:                 # line 5
                     if 1: pass        # line 6
             import dis
-            co = ireturn_example.func_code
+            co = ireturn_example.__code__
             linestarts = list(dis.findlinestarts(co))
             addrreturn = linestarts[-1][0]
             x = [addrreturn == (len(co.co_code) - 4)]
@@ -516,8 +536,8 @@
         """, 'x', [True, 3, 4, 6]
 
     def test_type_of_constants(self):
-        yield self.simple_test, "x=[0, 0L]", 'type(x[1])', long
-        yield self.simple_test, "x=[(1,0), (1,0L)]", 'type(x[1][1])', long
+        yield self.simple_test, "x=[0, 0.]", 'type(x[1])', float
+        yield self.simple_test, "x=[(1,0), (1,0.)]", 'type(x[1][1])', float
         yield self.simple_test, "x=['2?-', '2?-']", 'id(x[0])==id(x[1])', True
 
     def test_pprint(self):
@@ -643,29 +663,27 @@
                 #Indexing
                 for key, value in self.reference.items():
                     self.assertEqual(d[key], value)
-                knownkey = self.other.keys()[0]
+                knownkey = next(iter(self.other))
                 self.failUnlessRaises(KeyError, lambda:d[knownkey])
                 #len
                 self.assertEqual(len(p), 0)
                 self.assertEqual(len(d), len(self.reference))
                 #has_key
                 for k in self.reference:
-                    self.assert_(d.has_key(k))
                     self.assert_(k in d)
                 for k in self.other:
-                    self.failIf(d.has_key(k))
                     self.failIf(k in d)
                 #cmp
-                self.assertEqual(cmp(p,p), 0)
-                self.assertEqual(cmp(d,d), 0)
-                self.assertEqual(cmp(p,d), -1)
-                self.assertEqual(cmp(d,p), 1)
+                self.assert_(p == p)
+                self.assert_(d == d)
+                self.assert_(p < d)
+                self.assert_(d > p)
                 #__non__zero__
                 if p: self.fail("Empty mapping must compare to False")
                 if not d: self.fail("Full mapping must compare to True")
                 # keys(), items(), iterkeys() ...
                 def check_iterandlist(iter, lst, ref):
-                    self.assert_(hasattr(iter, 'next'))
+                    self.assert_(hasattr(iter, '__next__'))
                     self.assert_(hasattr(iter, '__iter__'))
                     x = list(iter)
                     self.assert_(set(x)==set(lst)==set(ref))
@@ -674,8 +692,8 @@
                 check_iterandlist(d.itervalues(), d.values(), self.reference.values())
                 check_iterandlist(d.iteritems(), d.items(), self.reference.items())
                 #get
-                key, value = d.iteritems().next()
-                knownkey, knownvalue = self.other.iteritems().next()
+                key, value = next(d.iteritems())
+                knownkey, knownvalue = next(self.other.iteritems())
                 self.assertEqual(d.get(key, knownvalue), value)
                 self.assertEqual(d.get(knownkey, knownvalue), knownvalue)
                 self.failIf(knownkey in d)
@@ -698,6 +716,15 @@
         decl = str(decl) + '\n'
         yield self.simple_test, decl, 'r', None
 
+    def test_assert(self):
+        decl = py.code.Source("""
+        try:
+            assert 0, 'hi'
+        except AssertionError as e:
+            msg = str(e)
+        """)
+        yield self.simple_test, decl, 'msg', 'hi'
+
     def test_indentation_error(self):
         source = py.code.Source("""
         x
@@ -763,7 +790,7 @@
         yield self.st, "y = lambda x: x", "y(4)", 4
 
     def test_backquote_repr(self):
-        yield self.st, "x = None; y = `x`", "y", "None"
+        py.test.raises(SyntaxError, self.simple_test, "y = `0`", None, None)
 
     def test_deleting_attributes(self):
         test = """if 1:
@@ -778,33 +805,72 @@
             raise AssertionError("attribute not removed")"""
         yield self.st, test, "X.__name__", "X"
 
+    def test_nonlocal(self):
+        test = """if 1:
+        def f():
+            y = 0
+            def g(x):
+                nonlocal y
+                y = x + 1
+            g(3)
+            return y"""
+        yield self.st, test, "f()", 4
+
+    def test_raise_from(self):
+        test = """if 1:
+        def f():
+            try:
+                raise TypeError() from ValueError()
+            except TypeError:
+                return 42
+        """
+        yield self.st, test, "f()", 42
+    # This line is needed for py.code to find the source.
+
+    def test_extended_unpacking(self):
+        func = """def f():
+            (a, *b, c) = 1, 2, 3, 4, 5
+            return a, b, c
+        """
+        yield self.st, func, "f()", (1, [2, 3, 4], 5)
+        func = """def f():
+            [a, *b, c] = 1, 2, 3, 4, 5
+            return a, b, c
+        """
+        yield self.st, func, "f()", (1, [2, 3, 4], 5)
+        func = """def f():
+            *a, = [1, 2, 3]
+            return a
+        """
+        yield self.st, func, "f()", [1, 2, 3]
+        func = """def f():
+            for a, *b, c in [(1, 2, 3, 4)]:
+                return a, b, c
+        """
+        yield self.st, func, "f()", (1, [2, 3], 4)
+        py.test.raises(SyntaxError, self.simple_test, "*a, *b = [1, 2]",
+                       None, None)
+        py.test.raises(SyntaxError, self.simple_test, "a = [*b, c]",
+                       None, None)
+        py.test.raises(SyntaxError, self.simple_test, "for *a in x: pass",
+                       None, None)
+
 
 class AppTestCompiler:
 
     def test_docstring_not_loaded(self):
-        import StringIO, dis, sys
+        import io, dis, sys
         ns = {}
-        exec "def f():\n    'hi'" in ns
+        exec("def f():\n    'hi'", ns)
         f = ns["f"]
         save = sys.stdout
-        sys.stdout = output = StringIO.StringIO()
+        sys.stdout = output = io.StringIO()
         try:
             dis.dis(f)
         finally:
             sys.stdout = save
         assert "0 ('hi')" not in output.getvalue()
 
-    def test_print_to(self):
-         exec """if 1:
-         from StringIO import StringIO
-         s = StringIO()
-         print >> s, "hi", "lovely!"
-         assert s.getvalue() == "hi lovely!\\n"
-         s = StringIO()
-         print >> s, "hi", "lovely!",
-         assert s.getvalue() == "hi lovely!"
-         """ in {}
-
 class TestOptimizations:
     def count_instructions(self, source):
         code, blocks = generate_function_code(source, self.space)
@@ -841,14 +907,14 @@
 
     def test_const_fold_unicode_subscr(self, monkeypatch):
         source = """def f():
-        return u"abc"[0]
+        return "abc"[0]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
 
         # getitem outside of the BMP should not be optimized
         source = """def f():
-        return u"\U00012345"[0]
+        return "\U00012345"[0]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
@@ -856,7 +922,7 @@
 
         monkeypatch.setattr(optimize, "MAXUNICODE", 0xFFFF)
         source = """def f():
-        return u"\uE01F"[0]
+        return "\uE01F"[0]
         """
         counts = self.count_instructions(source)
         assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
@@ -865,11 +931,11 @@
         # getslice is not yet optimized.
         # Still, check a case which yields the empty string.
         source = """def f():
-        return u"abc"[:0]
+        return "abc"[:0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 2, ops.SLICE+2: 1,
-                          ops.RETURN_VALUE: 1}
+        assert counts == {ops.LOAD_CONST: 3, ops.BUILD_SLICE: 1,
+                          ops.BINARY_SUBSCR: 1, ops.RETURN_VALUE: 1}
 
     def test_remove_dead_code(self):
         source = """def f(x):
@@ -902,10 +968,10 @@
         space = self.space
         w_generator = space.appexec([], """():
             d = {}
-            exec '''def f(x):
+            exec('''def f(x):
                 return
                 yield 6
-            ''' in d
+            ''', d)
             return d['f'](5)
         """)
         assert 'generator' in space.str_w(space.repr(w_generator))
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -194,22 +194,22 @@
     def immutable_unique_id(self, space):
         return None
 
-    def str_w(self, space):
-        w_msg = typed_unwrap_error_msg(space, "string", self)
+    def bytes_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "bytes", self)
         raise OperationError(space.w_TypeError, w_msg)
 
     def unicode_w(self, space):
         raise OperationError(space.w_TypeError,
-                             typed_unwrap_error_msg(space, "unicode", self))
+                             typed_unwrap_error_msg(space, "string", self))
 
     def int_w(self, space):
         raise OperationError(space.w_TypeError,
                              typed_unwrap_error_msg(space, "integer", self))
-    
+
     def uint_w(self, space):
         raise OperationError(space.w_TypeError,
                              typed_unwrap_error_msg(space, "integer", self))
-    
+
     def bigint_w(self, space):
         raise OperationError(space.w_TypeError,
                              typed_unwrap_error_msg(space, "integer", self))
@@ -273,6 +273,7 @@
     http://pypy.readthedocs.org/en/latest/objspace.html"""
 
     full_exceptions = True  # full support for exceptions (normalization & more)
+    py3k = True             # are we interpreting py3k bytecode?
 
     def __init__(self, config=None):
         "NOT_RPYTHON: Basic initialization of objects."
@@ -296,7 +297,6 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self.frame_trace_action = FrameTraceAction(self)
-        self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
         self.our_magic = default_magic
@@ -337,9 +337,8 @@
 
     def finish(self):
         self.wait_for_thread_shutdown()
-        w_exitfunc = self.sys.getdictvalue(self, 'exitfunc')
-        if w_exitfunc is not None:
-            self.call_function(w_exitfunc)
+        w_atexit = self.getbuiltinmodule('atexit')
+        self.call_method(w_atexit, '_run_exitfuncs')
         from pypy.interpreter.module import Module
         for w_mod in self.builtin_modules.values():
             mod = self.interpclass_w(w_mod)
@@ -468,9 +467,9 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: rctime take precedence over time
+        # a bit of custom logic: time2 or rctime take precedence over time
         # XXX this could probably be done as a "requires" in the config
-        if 'rctime' in modules and 'time' in modules:
+        if ('time2' in modules or 'rctime' in modules) and 'time' in modules:
             modules.remove('time')
 
         if not self.config.objspace.nofaking:
@@ -507,24 +506,24 @@
         self.exceptions_module = Module(self, w_name)
         self.exceptions_module.install()
 
+        from pypy.module.imp import Module
+        w_name = self.wrap('imp')
+        mod = Module(self, w_name)
+        mod.install()
+
         from pypy.module.sys import Module
         w_name = self.wrap('sys')
         self.sys = Module(self, w_name)
         self.sys.install()
 
-        from pypy.module.imp import Module
-        w_name = self.wrap('imp')
-        mod = Module(self, w_name)
-        mod.install()
-
         from pypy.module.__builtin__ import Module
-        w_name = self.wrap('__builtin__')
+        w_name = self.wrap('builtins')
         self.builtin = Module(self, w_name)
         w_builtin = self.wrap(self.builtin)
         w_builtin.install()
         self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin)
 
-        bootstrap_modules = set(('sys', 'imp', '__builtin__', 'exceptions'))
+        bootstrap_modules = set(('sys', 'imp', 'builtins', 'exceptions'))
         installed_builtin_modules = list(bootstrap_modules)
 
         exception_types_w = self.export_builtin_exceptions()
@@ -561,9 +560,15 @@
     def export_builtin_exceptions(self):
         """NOT_RPYTHON"""
         w_dic = self.exceptions_module.getdict(self)
-        w_keys = self.call_method(w_dic, "keys")
         exc_types_w = {}
-        for w_name in self.unpackiterable(w_keys):
+        w_iter = self.iter(w_dic)
+        while True:
+            try:
+                w_name = self.next(w_iter)
+            except OperationError, e:
+                if not e.match(self, self.w_StopIteration):
+                    raise
+                break
             name = self.str_w(w_name)
             if not name.startswith('__'):
                 excname = name
@@ -610,7 +615,7 @@
             self.fromcache(State).build_api(self)
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('imp')
-        self.getbuiltinmodule('__builtin__')
+        self.getbuiltinmodule('builtins')
         for mod in self.builtin_modules.values():
             mod.setup_after_space_initialization()
 
@@ -914,12 +919,6 @@
         """
         return None
 
-    def view_as_kwargs(self, w_dict):
-        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
-        strings and one of wrapped values. otherwise return (None, None)
-        """
-        return (None, None)
-
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
@@ -927,19 +926,14 @@
     def exception_match(self, w_exc_type, w_check_class):
         """Checks if the given exception type matches 'w_check_class'."""
         if self.is_w(w_exc_type, w_check_class):
-            return True   # fast path (also here to handle string exceptions)
-        try:
-            if self.is_true(self.isinstance(w_check_class, self.w_tuple)):
-                for w_t in self.fixedview(w_check_class):
-                    if self.exception_match(w_exc_type, w_t):
-                        return True
-                else:
-                    return False
-            return self.exception_issubclass_w(w_exc_type, w_check_class)
-        except OperationError, e:
-            if e.match(self, self.w_TypeError):   # string exceptions maybe
+            return True   # fast path
+        if self.is_true(self.isinstance(w_check_class, self.w_tuple)):
+            for w_t in self.fixedview(w_check_class):
+                if self.exception_match(w_exc_type, w_t):
+                    return True
+            else:
                 return False
-            raise
+        return self.exception_issubclass_w(w_exc_type, w_check_class)
 
     def call_obj_args(self, w_callable, w_obj, args):
         if not self.config.objspace.disable_call_speedhacks:
@@ -960,15 +954,10 @@
             # start of hack for performance
             from pypy.interpreter.function import Function, Method
             if isinstance(w_func, Method):
-                w_inst = w_func.w_instance
-                if w_inst is not None:
-                    if nargs < 4:
-                        func = w_func.w_function
-                        if isinstance(func, Function):
-                            return func.funccall(w_inst, *args_w)
-                elif args_w and (
-                        self.abstract_isinstance_w(args_w[0], w_func.w_class)):
-                    w_func = w_func.w_function
+                if nargs < 4:
+                    func = w_func.w_function
+                    if isinstance(func, Function):
+                        return func.funccall(w_func.w_instance, *args_w)
 
             if isinstance(w_func, Function):
                 return w_func.funccall(*args_w)
@@ -988,16 +977,10 @@
         if not self.config.objspace.disable_call_speedhacks:
             # start of hack for performance
             if isinstance(w_func, Method):
-                w_inst = w_func.w_instance
-                if w_inst is not None:
-                    w_func = w_func.w_function
-                    # reuse callable stack place for w_inst
-                    frame.settopvalue(w_inst, nargs)
-                    nargs += 1
-                elif nargs > 0 and (
-                    self.abstract_isinstance_w(frame.peekvalue(nargs-1),   #    :-(
-                                               w_func.w_class)):
-                    w_func = w_func.w_function
+                # reuse callable stack place for w_inst
+                frame.settopvalue(w_func.w_instance, nargs)
+                nargs += 1
+                w_func = w_func.w_function
 
             if isinstance(w_func, Function):
                 return w_func.funccall_valuestack(nargs, frame)
@@ -1030,26 +1013,8 @@
                 return w_value
         return None
 
-    def is_oldstyle_instance(self, w_obj):
-        # xxx hack hack hack
-        from pypy.module.__builtin__.interp_classobj import W_InstanceObject
-        obj = self.interpclass_w(w_obj)
-        return obj is not None and isinstance(obj, W_InstanceObject)
-
     def callable(self, w_obj):
-        if self.lookup(w_obj, "__call__") is not None:
-            if self.is_oldstyle_instance(w_obj):
-                # ugly old style class special treatment, but well ...
-                try:
-                    self.getattr(w_obj, self.wrap("__call__"))
-                    return self.w_True
-                except OperationError, e:
-                    if not e.match(self, self.w_AttributeError):
-                        raise
-                    return self.w_False
-            else:
-                return self.w_True
-        return self.w_False
+        return self.wrap(self.lookup(w_obj, "__call__") is not None)
 
     def issequence_w(self, w_obj):
         return (self.findattr(w_obj, self.wrap("__getitem__")) is not None)
@@ -1059,7 +1024,7 @@
 
     # The code below only works
     # for the simple case (new-style instance).
-    # These methods are patched with the full logic by the __builtin__
+    # These methods are patched with the full logic by the builtins
     # module when it is loaded
 
     def abstract_issubclass_w(self, w_cls1, w_cls2):
@@ -1078,8 +1043,7 @@
         # Equivalent to 'obj.__class__'.
         return self.type(w_obj)
 
-    # CPython rules allows old style classes or subclasses
-    # of BaseExceptions to be exceptions.
+    # CPython rules allows subclasses of BaseExceptions to be exceptions.
     # This is slightly less general than the case above, so we prefix
     # it with exception_
 
@@ -1305,7 +1269,7 @@
         # unclear if there is any use at all for getting the bytes in
         # the unicode buffer.)
         try:
-            return self.str_w(w_obj)
+            return self.bytes_w(w_obj)
         except OperationError, e:
             if not e.match(self, self.w_TypeError):
                 raise
@@ -1318,12 +1282,31 @@
         return self.str_w(w_obj)
 
     def str_w(self, w_obj):
-        return w_obj.str_w(self)
+        if self.isinstance_w(w_obj, self.w_unicode):
+            try:
+                return self.unicode_w(w_obj).encode('ascii')
+            except UnicodeEncodeError:
+                w_bytes = self.call_method(w_obj, 'encode', self.wrap('utf-8'))
+                return self.bytes_w(w_bytes)
+        else:
+            return w_obj.bytes_w(self)
+
+    def bytes_w(self, w_obj):
+        return w_obj.bytes_w(self)
 
     def str0_w(self, w_obj):
         "Like str_w, but rejects strings with NUL bytes."
         from pypy.rlib import rstring
-        result = w_obj.str_w(self)
+        result = self.str_w(w_obj)
+        if '\x00' in result:
+            raise OperationError(self.w_TypeError, self.wrap(
+                    'argument must be a string without NUL characters'))
+        return rstring.assert_str0(result)
+
+    def bytes0_w(self, w_obj):
+        "Like bytes_w, but rejects strings with NUL bytes."
+        from pypy.rlib import rstring
+        result = self.bytes_w(w_obj)
         if '\x00' in result:
             raise OperationError(self.w_TypeError, self.wrap(
                     'argument must be a string without NUL characters'))
@@ -1454,8 +1437,7 @@
         # not os.close().  It's likely designed for 'select'.  It's irregular
         # in the sense that it expects either a real int/long or an object
         # with a fileno(), but not an object with an __int__().
-        if (not self.isinstance_w(w_fd, self.w_int) and
-            not self.isinstance_w(w_fd, self.w_long)):
+        if not self.isinstance_w(w_fd, self.w_int):
             try:
                 w_fileno = self.getattr(w_fd, self.wrap("fileno"))
             except OperationError, e:
@@ -1538,16 +1520,11 @@
     ('getitem',         'getitem',   2, ['__getitem__']),
     ('setitem',         'setitem',   3, ['__setitem__']),
     ('delitem',         'delitem',   2, ['__delitem__']),
-    ('getslice',        'getslice',  3, ['__getslice__']),
-    ('setslice',        'setslice',  4, ['__setslice__']),
-    ('delslice',        'delslice',  3, ['__delslice__']),
     ('trunc',           'trunc',     1, ['__trunc__']),
     ('pos',             'pos',       1, ['__pos__']),
     ('neg',             'neg',       1, ['__neg__']),
-    ('nonzero',         'truth',     1, ['__nonzero__']),
+    ('nonzero',         'truth',     1, ['__bool__']),
     ('abs' ,            'abs',       1, ['__abs__']),
-    ('hex',             'hex',       1, ['__hex__']),
-    ('oct',             'oct',       1, ['__oct__']),
     ('ord',             'ord',       1, []),
     ('invert',          '~',         1, ['__invert__']),
     ('add',             '+',         2, ['__add__', '__radd__']),
@@ -1567,7 +1544,6 @@
     ('int',             'int',       1, ['__int__']),
     ('index',           'index',     1, ['__index__']),
     ('float',           'float',     1, ['__float__']),
-    ('long',            'long',      1, ['__long__']),
     ('inplace_add',     '+=',        2, ['__iadd__']),
     ('inplace_sub',     '-=',        2, ['__isub__']),
     ('inplace_mul',     '*=',        2, ['__imul__']),
@@ -1587,11 +1563,9 @@
     ('ne',              '!=',        2, ['__ne__', '__ne__']),
     ('gt',              '>',         2, ['__gt__', '__lt__']),
     ('ge',              '>=',        2, ['__ge__', '__le__']),
-    ('cmp',             'cmp',       2, ['__cmp__']),   # rich cmps preferred
-    ('coerce',          'coerce',    2, ['__coerce__', '__coerce__']),
     ('contains',        'contains',  2, ['__contains__']),
     ('iter',            'iter',      1, ['__iter__']),
-    ('next',            'next',      1, ['next']),
+    ('next',            'next',      1, ['__next__']),
 #    ('call',            'call',      3, ['__call__']),
     ('get',             'get',       3, ['__get__']),
     ('set',             'set',       3, ['__set__']),
@@ -1601,7 +1575,7 @@
     ]
 
 ObjSpace.BuiltinModuleTable = [
-    '__builtin__',
+    'builtins',
     'sys',
     ]
 
@@ -1678,7 +1652,7 @@
 
 ObjSpace.IrregularOpTable = [
     'wrap',
-    'str_w',
+    'bytes_w',
     'int_w',
     'float_w',
     'uint_w',

From noreply at buildbot.pypy.org  Mon Apr 16 10:03:18 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 16 Apr 2012 10:03:18 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: fix translation
Message-ID: <20120416080318.340D4820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54383:107ee52d2e02
Date: 2012-04-16 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/107ee52d2e02/

Log:	fix translation

diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -8,6 +8,7 @@
 from pypy.rpython.memory.gctransform import \
      refcounting, boehm, framework, asmgcroot
 from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.memory import gctypelayout
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
 class BasicGcPolicy(object):
@@ -340,7 +341,7 @@
         return framework.WEAKREF
 
     def convert_weakref_to(self, ptarget):
-        return framework.convert_weakref_to(ptarget)
+        return gctypelayout.convert_weakref_to(ptarget)
 
     def OP_GC_RELOAD_POSSIBLY_MOVED(self, funcgen, op):
         if isinstance(op.args[1], Constant):

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:20 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix comment
Message-ID: <20120416094620.2273E820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54384:bf7d80dbb9be
Date: 2012-04-14 08:26 +0200
http://bitbucket.org/pypy/pypy/changeset/bf7d80dbb9be/

Log:	fix comment

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:21 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: make the path that the JIT cannot see
	smaller
Message-ID: <20120416094621.60BF6820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54385:f48f3db6048d
Date: 2012-04-14 09:49 +0200
http://bitbucket.org/pypy/pypy/changeset/f48f3db6048d/

Log:	make the path that the JIT cannot see smaller

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -171,7 +171,13 @@
         space = self.space
         keywords, values_w = space.view_as_kwargs(w_starstararg)
         if keywords is not None: # this path also taken for empty dicts
-            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
             return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
             keys_w = space.unpackiterable(w_starstararg)
@@ -229,22 +235,16 @@
     @jit.look_inside_iff(lambda self, keywords, keywords_w:
             jit.isconstant(len(keywords) and
             jit.isconstant(self.keywords)))
-    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
-        if self.keywords is None:
-            self.keywords = keywords[:] # copy to make non-resizable
-            self.keywords_w = keywords_w[:]
-        else:
-            # looks quadratic, but the JIT should remove all of it nicely.
-            # Also, all the lists should be small
-            for key in keywords:
-                for otherkey in self.keywords:
-                    if otherkey == key:
-                        raise operationerrfmt(self.space.w_TypeError,
-                                              "got multiple values "
-                                              "for keyword argument "
-                                              "'%s'", key)
-            self.keywords = self.keywords + keywords
-            self.keywords_w = self.keywords_w + keywords_w
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
 
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:22 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: a warning
Message-ID: <20120416094622.9B8AC820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54386:fc0c57a5f6bf
Date: 2012-04-14 10:23 +0200
http://bitbucket.org/pypy/pypy/changeset/fc0c57a5f6bf/

Log:	a warning

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:23 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:23 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: always look into all of
 _match_signature. This is unsafe, the next commits will
Message-ID: <20120416094623.D67BC820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54387:64a143f417f3
Date: 2012-04-14 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/64a143f417f3/

Log:	always look into all of _match_signature. This is unsafe, the next
	commits will pull out the parts that need to be hidden from the JIT

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -269,30 +269,13 @@
 
     ###  Parsing for function calls  ###
 
-    # XXX: this should be @jit.look_inside_iff, but we need key word arguments,
-    # and it doesn't support them for now.
+    @jit.unroll_safe
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
                          blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         Return the number of arguments filled in.
         """
-        if jit.we_are_jitted() and self._dont_jit:
-            return self._match_signature_jit_opaque(w_firstarg, scope_w,
-                                                    signature, defaults_w,
-                                                    blindargs)
-        return self._really_match_signature(w_firstarg, scope_w, signature,
-                                            defaults_w, blindargs)
-
-    @jit.dont_look_inside
-    def _match_signature_jit_opaque(self, w_firstarg, scope_w, signature,
-                                    defaults_w, blindargs):
-        return self._really_match_signature(w_firstarg, scope_w, signature,
-                                            defaults_w, blindargs)
-
-    @jit.unroll_safe
-    def _really_match_signature(self, w_firstarg, scope_w, signature,
-                                defaults_w=None, blindargs=0):
         #
         #   args_w = list of the normal actual parameters, wrapped
         #   kwds_w = real dictionary {'keyword': wrapped parameter}

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:25 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:25 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: move
 _check_not_duplicate_keywords to the global level
Message-ID: <20120416094625.449BB820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54388:fd475d40e0cc
Date: 2012-04-14 10:32 +0200
http://bitbucket.org/pypy/pypy/changeset/fd475d40e0cc/

Log:	move _check_not_duplicate_keywords to the global level

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -179,7 +179,8 @@
                 self.keywords = keywords[:] # copy to make non-resizable
                 self.keywords_w = values_w[:]
             else:
-                self._check_not_duplicate_kwargs(keywords, values_w)
+                _check_not_duplicate_kwargs(
+                    self.space, self.keywords, keywords, values_w)
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
             return not jit.isconstant(len(self.keywords))
@@ -236,19 +237,6 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
-    @jit.look_inside_iff(lambda self, keywords, keywords_w:
-            jit.isconstant(len(keywords) and
-            jit.isconstant(self.keywords)))
-    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
-        # looks quadratic, but the JIT should remove all of it nicely.
-        # Also, all the lists should be small
-        for key in keywords:
-            for otherkey in self.keywords:
-                if otherkey == key:
-                    raise operationerrfmt(self.space.w_TypeError,
-                                          "got multiple values "
-                                          "for keyword argument "
-                                          "'%s'", key)
 
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
@@ -482,6 +470,26 @@
                 space.setitem(w_kwds, w_key, self.keywords_w[i])
         return w_args, w_kwds
 
+# JIT helper functions
+# these functions contain functionality that the JIT is not always supposed to
+# look at. They should not get a self arguments, which makes the amount of
+# arguments annoying :-(
+
+ at jit.look_inside_iff(lambda space, existingkeywords, keywords, keywords_w:
+        jit.isconstant(len(keywords) and
+        jit.isconstant(existingkeywords)))
+def _check_not_duplicate_kwargs(space, existingkeywords, keywords_w):
+    # looks quadratic, but the JIT should remove all of it nicely.
+    # Also, all the lists should be small
+    for key in keywords:
+        for otherkey in existingkeywords:
+            if otherkey == key:
+                raise operationerrfmt(space.w_TypeError,
+                                      "got multiple values "
+                                      "for keyword argument "
+                                      "'%s'", key)
+
+
 class ArgumentsForTranslation(Arguments):
     def __init__(self, space, args_w, keywords=None, keywords_w=None,
                  w_stararg=None, w_starstararg=None):

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:26 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:26 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: oops
Message-ID: <20120416094626.82120820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54389:a37af8c02561
Date: 2012-04-14 10:34 +0200
http://bitbucket.org/pypy/pypy/changeset/a37af8c02561/

Log:	oops

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -478,7 +478,7 @@
 @jit.look_inside_iff(lambda space, existingkeywords, keywords, keywords_w:
         jit.isconstant(len(keywords) and
         jit.isconstant(existingkeywords)))
-def _check_not_duplicate_kwargs(space, existingkeywords, keywords_w):
+def _check_not_duplicate_kwargs(space, existingkeywords, keywords, keywords_w):
     # looks quadratic, but the JIT should remove all of it nicely.
     # Also, all the lists should be small
     for key in keywords:

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:27 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:27 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: move dangerous loops from
 _match_signature to the global level
Message-ID: <20120416094627.BDF08820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54390:73f4eae55465
Date: 2012-04-14 10:34 +0200
http://bitbucket.org/pypy/pypy/changeset/73f4eae55465/

Log:	move dangerous loops from _match_signature to the global level

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -336,32 +336,12 @@
         # argnames is typically not too large
         num_remainingkwds = num_kwds
         used_keywords = None
-        if keywords:
-            # letting JIT unroll the loop is *only* safe if the callsite didn't
-            # use **args because num_kwds can be arbitrarily large otherwise.
+        if num_kwds:
             used_keywords = [False] * num_kwds
-            for i in range(num_kwds):
-                name = keywords[i]
-                # If name was not encoded as a string, it could be None. In that
-                # case, it's definitely not going to be in the signature.
-                if name is None:
-                    continue
-                j = signature.find_argname(name)
-                if j < 0:
-                    continue
-                elif j < input_argcount:
-                    # check that no keyword argument conflicts with these. note
-                    # that for this purpose we ignore the first blindargs,
-                    # which were put into place by prepend().  This way,
-                    # keywords do not conflict with the hidden extra argument
-                    # bound by methods.
-                    if blindargs <= j:
-                        raise ArgErrMultipleValues(name)
-                else:
-                    assert scope_w[j] is None
-                    scope_w[j] = keywords_w[i]
-                    used_keywords[i] = True # mark as used
-                    num_remainingkwds -= 1
+            num_remainingkwds = _match_keywords(
+                    signature, blindargs, input_argcount, keywords,
+                    keywords_w, scope_w, used_keywords,
+                    self._dont_jit)
         missing = 0
         if input_argcount < co_argcount:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
@@ -377,21 +357,13 @@
                     # keyword arguments, which will be checked for below.
                     missing += 1
 
-        # collect extra keyword arguments into the **kwarg
         if has_kwarg:
             w_kwds = self.space.newdict(kwargs=True)
+            # collect extra keyword arguments into the **kwarg
             if num_remainingkwds:
-                #
-                limit = len(keywords)
-                if self.keyword_names_w is not None:
-                    limit -= len(self.keyword_names_w)
-                for i in range(len(keywords)):
-                    if not used_keywords[i]:
-                        if i < limit:
-                            w_key = self.space.wrap(keywords[i])
-                        else:
-                            w_key = self.keyword_names_w[i - limit]
-                        self.space.setitem(w_kwds, w_key, keywords_w[i])
+                _collect_keyword_args(
+                        self.space, keywords, keywords_w, w_kwds,
+                        used_keywords, self.keyword_names_w, self._dont_jit)
                 #
             scope_w[co_argcount + has_vararg] = w_kwds
         elif num_remainingkwds:
@@ -489,6 +461,53 @@
                                       "for keyword argument "
                                       "'%s'", key)
 
+ at jit.look_inside_iff(
+    lambda signature, blindargs, input_argcount, keywords, keywords_w,
+           scope_w, used_keywords, jitoff: not jitoff)
+def _match_keywords(signature, blindargs, input_argcount,
+                    keywords, keywords_w, scope_w, used_keywords, _):
+    # letting JIT unroll the loop is *only* safe if the callsite didn't
+    # use **args because num_kwds can be arbitrarily large otherwise.
+    num_kwds = num_remainingkwds = len(keywords)
+    for i in range(num_kwds):
+        name = keywords[i]
+        # If name was not encoded as a string, it could be None. In that
+        # case, it's definitely not going to be in the signature.
+        if name is None:
+            continue
+        j = signature.find_argname(name)
+        if j < 0:
+            continue
+        elif j < input_argcount:
+            # check that no keyword argument conflicts with these. note
+            # that for this purpose we ignore the first blindargs,
+            # which were put into place by prepend().  This way,
+            # keywords do not conflict with the hidden extra argument
+            # bound by methods.
+            if blindargs <= j:
+                raise ArgErrMultipleValues(name)
+        else:
+            assert scope_w[j] is None
+            scope_w[j] = keywords_w[i]
+            used_keywords[i] = True # mark as used
+            num_remainingkwds -= 1
+    return num_remainingkwds
+
+ at jit.look_inside_iff(
+    lambda space, keywords, keywords_w, w_kwds, used_keywords,
+           keyword_names_w, jitoff: not jitoff)
+def _collect_keyword_args(space, keywords, keywords_w, w_kwds, used_keywords,
+                          keyword_names_w, _):
+    limit = len(keywords)
+    if keyword_names_w is not None:
+        limit -= len(keyword_names_w)
+    for i in range(len(keywords)):
+        if not used_keywords[i]:
+            if i < limit:
+                w_key = space.wrap(keywords[i])
+            else:
+                w_key = keyword_names_w[i - limit]
+            space.setitem(w_kwds, w_key, keywords_w[i])
 
 class ArgumentsForTranslation(Arguments):
     def __init__(self, space, args_w, keywords=None, keywords_w=None,

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:29 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:29 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: a test that checks that the
 **args dict that is created is giving space.newdict
Message-ID: <20120416094629.3FE63820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54391:5041230ccf58
Date: 2012-04-14 10:44 +0200
http://bitbucket.org/pypy/pypy/changeset/5041230ccf58/

Log:	a test that checks that the **args dict that is created is giving
	space.newdict the flag that turns it into a kwargsdict

diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -57,6 +57,9 @@
     def __nonzero__(self):
         raise NotImplementedError
 
+class kwargsdict(dict):
+    pass
+
 class DummySpace(object):
     def newtuple(self, items):
         return tuple(items)
@@ -79,6 +82,8 @@
         return None, None
 
     def newdict(self, kwargs=False):
+        if kwargs:
+            return kwargsdict()
         return {}
 
     def newlist(self, l=[]):
@@ -299,6 +304,22 @@
             args._match_signature(None, l, Signature(["a", "b", "c"], None, "**"))
             assert l == [1, 2, 3, {'d': 4}]
 
+    def test_match_kwds_creates_kwdict(self):
+        space = DummySpace()
+        kwds = [("c", 3), ('d', 4)]
+        for i in range(4):
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            w_kwds = dummy_wrapped_dict(kwds[i:])
+            if i == 3:
+                w_kwds = None
+            args = Arguments(space, [1, 2], keywords, keywords_w, w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"], None, "**"))
+            assert l == [1, 2, 3, {'d': 4}]
+            assert isinstance(l[-1], kwargsdict)
+
     def test_duplicate_kwds(self):
         space = DummySpace()
         excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:30 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:30 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: a test for _dont_jit
Message-ID: <20120416094630.7E438820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54392:311b1249ebd5
Date: 2012-04-14 10:49 +0200
http://bitbucket.org/pypy/pypy/changeset/311b1249ebd5/

Log:	a test for _dont_jit

diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -79,6 +79,8 @@
         return list(it)
 
     def view_as_kwargs(self, x):
+        if len(x) == 0:
+            return [], []
         return None, None
 
     def newdict(self, kwargs=False):
@@ -320,6 +322,19 @@
             assert l == [1, 2, 3, {'d': 4}]
             assert isinstance(l[-1], kwargsdict)
 
+    def test_match_kwds_creates_kwdict(self):
+        space = DummySpace()
+        kwds = [("c", 3), ('d', 4)]
+        for i in range(4):
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            w_kwds = dummy_wrapped_dict(kwds[i:])
+            if i == 3:
+                w_kwds = None
+            args = Arguments(space, [1, 2], keywords, keywords_w, w_starstararg=w_kwds)
+            assert args._dont_jit == (i != 3)
+
     def test_duplicate_kwds(self):
         space = DummySpace()
         excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:31 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:31 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: move another function that
 could force the Arguments instance to the global
Message-ID: <20120416094631.B3932820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54393:a3a1be50ae8e
Date: 2012-04-14 10:50 +0200
http://bitbucket.org/pypy/pypy/changeset/a3a1be50ae8e/

Log:	move another function that could force the Arguments instance to the
	global level

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -199,43 +199,17 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-        return True
-
-    def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
-        space = self.space
         keywords_w = [None] * len(keys_w)
         keywords = [None] * len(keys_w)
-        i = 0
-        for w_key in keys_w:
-            try:
-                key = space.str_w(w_key)
-            except OperationError, e:
-                if e.match(space, space.w_TypeError):
-                    raise OperationError(
-                        space.w_TypeError,
-                        space.wrap("keywords must be strings"))
-                if e.match(space, space.w_UnicodeEncodeError):
-                    # Allow this to pass through
-                    key = None
-                else:
-                    raise
-            else:
-                if self.keywords and key in self.keywords:
-                    raise operationerrfmt(self.space.w_TypeError,
-                                          "got multiple values "
-                                          "for keyword argument "
-                                          "'%s'", key)
-            keywords[i] = key
-            keywords_w[i] = space.getitem(w_starstararg, w_key)
-            i += 1
+        _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords, keywords_w, self.keywords)
+        self.keyword_names_w = keys_w
         if self.keywords is None:
             self.keywords = keywords
             self.keywords_w = keywords_w
         else:
             self.keywords = self.keywords + keywords
             self.keywords_w = self.keywords_w + keywords_w
-        self.keyword_names_w = keys_w
+        return True
 
 
     def fixedunpack(self, argcount):
@@ -461,6 +435,32 @@
                                       "for keyword argument "
                                       "'%s'", key)
 
+def _do_combine_starstarargs_wrapped(space, keys_w, w_starstararg, keywords,
+        keywords_w, existingkeywords):
+    i = 0
+    for w_key in keys_w:
+        try:
+            key = space.str_w(w_key)
+        except OperationError, e:
+            if e.match(space, space.w_TypeError):
+                raise OperationError(
+                    space.w_TypeError,
+                    space.wrap("keywords must be strings"))
+            if e.match(space, space.w_UnicodeEncodeError):
+                # Allow this to pass through
+                key = None
+            else:
+                raise
+        else:
+            if existingkeywords and key in existingkeywords:
+                raise operationerrfmt(space.w_TypeError,
+                                      "got multiple values "
+                                      "for keyword argument "
+                                      "'%s'", key)
+        keywords[i] = key
+        keywords_w[i] = space.getitem(w_starstararg, w_key)
+        i += 1
+
 @jit.look_inside_iff(
     lambda signature, blindargs, input_argcount, keywords, keywords_w,
            scope_w, used_keywords, jitoff: not jitoff)

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:32 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:32 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: reshuffle logic to make it
	clearer
Message-ID: <20120416094632.E7D50820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54394:b86443b52757
Date: 2012-04-14 11:05 +0200
http://bitbucket.org/pypy/pypy/changeset/b86443b52757/

Log:	reshuffle logic to make it clearer

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -316,6 +316,23 @@
                     signature, blindargs, input_argcount, keywords,
                     keywords_w, scope_w, used_keywords,
                     self._dont_jit)
+        if has_kwarg:
+            w_kwds = self.space.newdict(kwargs=True)
+            # collect extra keyword arguments into the **kwarg
+            if num_remainingkwds:
+                _collect_keyword_args(
+                        self.space, keywords, keywords_w, w_kwds,
+                        used_keywords, self.keyword_names_w, self._dont_jit)
+                #
+            scope_w[co_argcount + has_vararg] = w_kwds
+        elif num_remainingkwds:
+            if co_argcount == 0:
+                raise ArgErrCount(avail, num_kwds,
+                              co_argcount, has_vararg, has_kwarg,
+                              defaults_w, 0)
+            raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
+                                    used_keywords, self.keyword_names_w)
+
         missing = 0
         if input_argcount < co_argcount:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
@@ -330,28 +347,10 @@
                     # because it might be related to a problem with */** or
                     # keyword arguments, which will be checked for below.
                     missing += 1
-
-        if has_kwarg:
-            w_kwds = self.space.newdict(kwargs=True)
-            # collect extra keyword arguments into the **kwarg
-            if num_remainingkwds:
-                _collect_keyword_args(
-                        self.space, keywords, keywords_w, w_kwds,
-                        used_keywords, self.keyword_names_w, self._dont_jit)
-                #
-            scope_w[co_argcount + has_vararg] = w_kwds
-        elif num_remainingkwds:
-            if co_argcount == 0:
+            if missing:
                 raise ArgErrCount(avail, num_kwds,
-                              co_argcount, has_vararg, has_kwarg,
-                              defaults_w, missing)
-            raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
-                                    used_keywords, self.keyword_names_w)
-
-        if missing:
-            raise ArgErrCount(avail, num_kwds,
-                              co_argcount, has_vararg, has_kwarg,
-                              defaults_w, missing)
+                                  co_argcount, has_vararg, has_kwarg,
+                                  defaults_w, missing)
 
         return co_argcount + has_vararg + has_kwarg
 

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:34 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:34 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: the test is actually
 worthless,
 because it cannot test the actually interesting cases about constant
 length-kwargs
Message-ID: <20120416094634.2D592820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54395:47edb5fb8a13
Date: 2012-04-14 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/47edb5fb8a13/

Log:	the test is actually worthless, because it cannot test the actually
	interesting cases about constant length-kwargs

diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -322,19 +322,6 @@
             assert l == [1, 2, 3, {'d': 4}]
             assert isinstance(l[-1], kwargsdict)
 
-    def test_match_kwds_creates_kwdict(self):
-        space = DummySpace()
-        kwds = [("c", 3), ('d', 4)]
-        for i in range(4):
-            kwds_w = dict(kwds[:i])
-            keywords = kwds_w.keys()
-            keywords_w = kwds_w.values()
-            w_kwds = dummy_wrapped_dict(kwds[i:])
-            if i == 3:
-                w_kwds = None
-            args = Arguments(space, [1, 2], keywords, keywords_w, w_starstararg=w_kwds)
-            assert args._dont_jit == (i != 3)
-
     def test_duplicate_kwds(self):
         space = DummySpace()
         excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:35 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:35 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: refactor how the _dont_jit
 flag is computed, and rename it
Message-ID: <20120416094635.755F8820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54396:f7c9cd45c880
Date: 2012-04-14 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/f7c9cd45c880/

Log:	refactor how the _dont_jit flag is computed, and rename it

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -110,12 +110,10 @@
             make_sure_not_resized(self.keywords_w)
 
         make_sure_not_resized(self.arguments_w)
-        if w_stararg is not None:
-            self._combine_starargs_wrapped(w_stararg)
-        # if we have a call where **args are used at the callsite
-        # we shouldn't let the JIT see the argument matching
-        self._dont_jit = (w_starstararg is not None and
-                          self._combine_starstarargs_wrapped(w_starstararg))
+        self._combine_wrapped(w_stararg, w_starstararg)
+        # a flag that specifies whether the JIT can unroll loops that operate
+        # on the keywords
+        self._jit_few_keywords = self.keywords is None or jit.isconstant(len(self.keywords))
 
     def __repr__(self):
         """ NOT_RPYTHON """
@@ -129,7 +127,7 @@
 
     ###  Manipulation  ###
 
-    @jit.look_inside_iff(lambda self: not self._dont_jit)
+    @jit.look_inside_iff(lambda self: self._jit_few_keywords)
     def unpack(self): # slowish
         "Return a ([w1,w2...], {'kw':w3...}) pair."
         kwds_w = {}
@@ -183,7 +181,7 @@
                     self.space, self.keywords, keywords, values_w)
                 self.keywords = self.keywords + keywords
                 self.keywords_w = self.keywords_w + values_w
-            return not jit.isconstant(len(self.keywords))
+            return
         if space.isinstance_w(w_starstararg, space.w_dict):
             keys_w = space.unpackiterable(w_starstararg)
         else:
@@ -209,7 +207,6 @@
         else:
             self.keywords = self.keywords + keywords
             self.keywords_w = self.keywords_w + keywords_w
-        return True
 
 
     def fixedunpack(self, argcount):
@@ -315,14 +312,14 @@
             num_remainingkwds = _match_keywords(
                     signature, blindargs, input_argcount, keywords,
                     keywords_w, scope_w, used_keywords,
-                    self._dont_jit)
+                    self._jit_few_keywords)
         if has_kwarg:
             w_kwds = self.space.newdict(kwargs=True)
             # collect extra keyword arguments into the **kwarg
             if num_remainingkwds:
                 _collect_keyword_args(
                         self.space, keywords, keywords_w, w_kwds,
-                        used_keywords, self.keyword_names_w, self._dont_jit)
+                        used_keywords, self.keyword_names_w, self._jit_few_keywords)
                 #
             scope_w[co_argcount + has_vararg] = w_kwds
         elif num_remainingkwds:
@@ -462,7 +459,7 @@
 
 @jit.look_inside_iff(
     lambda signature, blindargs, input_argcount, keywords, keywords_w,
-           scope_w, used_keywords, jitoff: not jitoff)
+           scope_w, used_keywords, jiton: jiton)
 def _match_keywords(signature, blindargs, input_argcount,
                     keywords, keywords_w, scope_w, used_keywords, _):
     # letting JIT unroll the loop is *only* safe if the callsite didn't
@@ -494,7 +491,7 @@
 
 @jit.look_inside_iff(
     lambda space, keywords, keywords_w, w_kwds, used_keywords,
-           keyword_names_w, jitoff: not jitoff)
+        keyword_names_w, jifon: jiton)
 def _collect_keyword_args(space, keywords, keywords_w, w_kwds, used_keywords,
                           keyword_names_w, _):
     limit = len(keywords)

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:36 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:36 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: make ArgErrCount take a
	signature
Message-ID: <20120416094636.BBCA9820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54397:9539374db6d9
Date: 2012-04-14 11:30 +0200
http://bitbucket.org/pypy/pypy/changeset/9539374db6d9/

Log:	make ArgErrCount take a signature

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -299,9 +299,7 @@
                 starargs_w = []
             scope_w[co_argcount] = self.space.newtuple(starargs_w)
         elif avail > co_argcount:
-            raise ArgErrCount(avail, num_kwds,
-                              co_argcount, has_vararg, has_kwarg,
-                              defaults_w, 0)
+            raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
 
         # the code assumes that keywords can potentially be large, but that
         # argnames is typically not too large
@@ -324,9 +322,7 @@
             scope_w[co_argcount + has_vararg] = w_kwds
         elif num_remainingkwds:
             if co_argcount == 0:
-                raise ArgErrCount(avail, num_kwds,
-                              co_argcount, has_vararg, has_kwarg,
-                              defaults_w, 0)
+                raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
             raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
                                     used_keywords, self.keyword_names_w)
 
@@ -345,9 +341,7 @@
                     # keyword arguments, which will be checked for below.
                     missing += 1
             if missing:
-                raise ArgErrCount(avail, num_kwds,
-                                  co_argcount, has_vararg, has_kwarg,
-                                  defaults_w, missing)
+                raise ArgErrCount(avail, num_kwds, signature, defaults_w, missing)
 
         return co_argcount + has_vararg + has_kwarg
 
@@ -660,11 +654,9 @@
 
 class ArgErrCount(ArgErr):
 
-    def __init__(self, got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
+    def __init__(self, got_nargs, nkwds, signature,
                  defaults_w, missing_args):
-        self.expected_nargs = expected_nargs
-        self.has_vararg = has_vararg
-        self.has_kwarg = has_kwarg
+        self.signature = signature
 
         self.num_defaults = 0 if defaults_w is None else len(defaults_w)
         self.missing_args = missing_args
@@ -672,16 +664,16 @@
         self.num_kwds = nkwds
 
     def getmsg(self):
-        n = self.expected_nargs
+        n = self.signature.num_argnames()
         if n == 0:
             msg = "takes no arguments (%d given)" % (
                 self.num_args + self.num_kwds)
         else:
             defcount = self.num_defaults
-            has_kwarg = self.has_kwarg
+            has_kwarg = self.signature.has_kwarg()
             num_args = self.num_args
             num_kwds = self.num_kwds
-            if defcount == 0 and not self.has_vararg:
+            if defcount == 0 and not self.signature.has_vararg():
                 msg1 = "exactly"
                 if not has_kwarg:
                     num_args += num_kwds
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -569,34 +569,47 @@
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
         # defaults_w, missing_args
-        err = ArgErrCount(1, 0, 0, False, False, None, 0)
+        sig = Signature([], None, None)
+        err = ArgErrCount(1, 0, sig, None, 0)
         s = err.getmsg()
         assert s == "takes no arguments (1 given)"
-        err = ArgErrCount(0, 0, 1, False, False, [], 1)
+
+        sig = Signature(['a'], None, None)
+        err = ArgErrCount(0, 0, sig, [], 1)
         s = err.getmsg()
         assert s == "takes exactly 1 argument (0 given)"
-        err = ArgErrCount(3, 0, 2, False, False, [], 0)
+
+        sig = Signature(['a', 'b'], None, None)
+        err = ArgErrCount(3, 0, sig, [], 0)
         s = err.getmsg()
         assert s == "takes exactly 2 arguments (3 given)"
-        err = ArgErrCount(3, 0, 2, False, False, ['a'], 0)
+        err = ArgErrCount(3, 0, sig, ['a'], 0)
         s = err.getmsg()
         assert s == "takes at most 2 arguments (3 given)"
-        err = ArgErrCount(1, 0, 2, True, False, [], 1)
+
+        sig = Signature(['a', 'b'], '*', None)
+        err = ArgErrCount(1, 0, sig, [], 1)
         s = err.getmsg()
         assert s == "takes at least 2 arguments (1 given)"
-        err = ArgErrCount(0, 1, 2, True, False, ['a'], 1)
+        err = ArgErrCount(0, 1, sig, ['a'], 1)
         s = err.getmsg()
         assert s == "takes at least 1 non-keyword argument (0 given)"
-        err = ArgErrCount(2, 1, 1, False, True, [], 0)
+
+        sig = Signature(['a'], None, '**')
+        err = ArgErrCount(2, 1, sig, [], 0)
         s = err.getmsg()
         assert s == "takes exactly 1 non-keyword argument (2 given)"
-        err = ArgErrCount(0, 1, 1, False, True, [], 1)
+        err = ArgErrCount(0, 1, sig, [], 1)
         s = err.getmsg()
         assert s == "takes exactly 1 non-keyword argument (0 given)"
-        err = ArgErrCount(0, 1, 1, True, True, [], 1)
+
+        sig = Signature(['a'], '*', '**')
+        err = ArgErrCount(0, 1, sig, [], 1)
         s = err.getmsg()
         assert s == "takes at least 1 non-keyword argument (0 given)"
-        err = ArgErrCount(2, 1, 1, False, True, ['a'], 0)
+
+        sig = Signature(['a'], None, '**')
+        err = ArgErrCount(2, 1, sig, ['a'], 0)
         s = err.getmsg()
         assert s == "takes at most 1 non-keyword argument (2 given)"
 

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:37 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:37 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: clean up local variables a bit
Message-ID: <20120416094637.F1401820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54398:59aaff22bf66
Date: 2012-04-14 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/59aaff22bf66/

Log:	clean up local variables a bit

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -233,7 +233,6 @@
                          blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
-        Return the number of arguments filled in.
         """
         #
         #   args_w = list of the normal actual parameters, wrapped
@@ -246,9 +245,6 @@
         # so all values coming from there can be assumed constant. It assumes
         # that the length of the defaults_w does not vary too much.
         co_argcount = signature.num_argnames() # expected formal arguments, without */**
-        has_vararg = signature.has_vararg()
-        has_kwarg = signature.has_kwarg()
-        extravarargs = None
         input_argcount =  0
 
         if w_firstarg is not None:
@@ -256,28 +252,22 @@
             if co_argcount > 0:
                 scope_w[0] = w_firstarg
                 input_argcount = 1
-            else:
-                extravarargs = [w_firstarg]
         else:
             upfront = 0
 
         args_w = self.arguments_w
         num_args = len(args_w)
+        avail = num_args + upfront
 
         keywords = self.keywords
-        keywords_w = self.keywords_w
         num_kwds = 0
         if keywords is not None:
             num_kwds = len(keywords)
 
-        avail = num_args + upfront
 
         if input_argcount < co_argcount:
             # put as many positional input arguments into place as available
-            if avail > co_argcount:
-                take = co_argcount - input_argcount
-            else:
-                take = num_args
+            take = min(num_args, co_argcount - upfront)
 
             # letting the JIT unroll this loop is safe, because take is always
             # smaller than co_argcount
@@ -286,11 +276,10 @@
             input_argcount += take
 
         # collect extra positional arguments into the *vararg
-        if has_vararg:
+        if signature.has_vararg():
             args_left = co_argcount - upfront
             if args_left < 0:  # check required by rpython
-                assert extravarargs is not None
-                starargs_w = extravarargs
+                starargs_w = [w_firstarg]
                 if num_args:
                     starargs_w = starargs_w + args_w
             elif num_args > args_left:
@@ -303,15 +292,16 @@
 
         # the code assumes that keywords can potentially be large, but that
         # argnames is typically not too large
-        num_remainingkwds = num_kwds
+        num_remainingkwds = 0
         used_keywords = None
+        keywords_w = self.keywords_w
         if num_kwds:
             used_keywords = [False] * num_kwds
             num_remainingkwds = _match_keywords(
                     signature, blindargs, input_argcount, keywords,
                     keywords_w, scope_w, used_keywords,
                     self._jit_few_keywords)
-        if has_kwarg:
+        if signature.has_kwarg():
             w_kwds = self.space.newdict(kwargs=True)
             # collect extra keyword arguments into the **kwarg
             if num_remainingkwds:
@@ -319,7 +309,7 @@
                         self.space, keywords, keywords_w, w_kwds,
                         used_keywords, self.keyword_names_w, self._jit_few_keywords)
                 #
-            scope_w[co_argcount + has_vararg] = w_kwds
+            scope_w[co_argcount + signature.has_vararg()] = w_kwds
         elif num_remainingkwds:
             if co_argcount == 0:
                 raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
@@ -343,8 +333,6 @@
             if missing:
                 raise ArgErrCount(avail, num_kwds, signature, defaults_w, missing)
 
-        return co_argcount + has_vararg + has_kwarg
-
 
 
     def parse_into_scope(self, w_firstarg,
@@ -355,11 +343,12 @@
         scope_w must be big enough for signature.
         """
         try:
-            return self._match_signature(w_firstarg,
-                                         scope_w, signature, defaults_w, 0)
+            self._match_signature(w_firstarg,
+                                  scope_w, signature, defaults_w, 0)
         except ArgErr, e:
             raise operationerrfmt(self.space.w_TypeError,
                                   "%s() %s", fnname, e.getmsg())
+        return signature.scope_length()
 
     def _parse(self, w_firstarg, signature, defaults_w, blindargs=0):
         """Parse args and kwargs according to the signature of a code object,

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:39 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:39 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: improve comment (which was
	rather outdated
Message-ID: <20120416094639.35BB5820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54399:ae0810d34f7d
Date: 2012-04-14 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/ae0810d34f7d/

Log:	improve comment (which was rather outdated

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -234,10 +234,8 @@
         """Parse args and kwargs according to the signature of a code object,
         or raise an ArgErr in case of failure.
         """
-        #
+        #   w_firstarg = a first argument to be inserted (e.g. self) or None
         #   args_w = list of the normal actual parameters, wrapped
-        #   kwds_w = real dictionary {'keyword': wrapped parameter}
-        #   argnames = list of formal parameter names
         #   scope_w = resulting list of wrapped values
         #
 

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:40 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:40 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: another small simplification
Message-ID: <20120416094640.6A1E2820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54400:c48491599f51
Date: 2012-04-14 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/c48491599f51/

Log:	another small simplification

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -243,13 +243,11 @@
         # so all values coming from there can be assumed constant. It assumes
         # that the length of the defaults_w does not vary too much.
         co_argcount = signature.num_argnames() # expected formal arguments, without */**
-        input_argcount =  0
 
         if w_firstarg is not None:
             upfront = 1
             if co_argcount > 0:
                 scope_w[0] = w_firstarg
-                input_argcount = 1
         else:
             upfront = 0
 
@@ -263,6 +261,7 @@
             num_kwds = len(keywords)
 
 
+        input_argcount = upfront
         if input_argcount < co_argcount:
             # put as many positional input arguments into place as available
             take = min(num_args, co_argcount - upfront)

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:41 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:41 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: restructure a bit more,
 and add comments what the various parts are doing
Message-ID: <20120416094641.A2D88820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54401:a9c25d15ff4e
Date: 2012-04-14 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/a9c25d15ff4e/

Log:	restructure a bit more, and add comments what the various parts are
	doing

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -244,6 +244,7 @@
         # that the length of the defaults_w does not vary too much.
         co_argcount = signature.num_argnames() # expected formal arguments, without */**
 
+        # put the special w_firstarg into the scope, if it exists
         if w_firstarg is not None:
             upfront = 1
             if co_argcount > 0:
@@ -261,9 +262,9 @@
             num_kwds = len(keywords)
 
 
+        # put as many positional input arguments into place as available
         input_argcount = upfront
         if input_argcount < co_argcount:
-            # put as many positional input arguments into place as available
             take = min(num_args, co_argcount - upfront)
 
             # letting the JIT unroll this loop is safe, because take is always
@@ -287,32 +288,37 @@
         elif avail > co_argcount:
             raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
 
-        # the code assumes that keywords can potentially be large, but that
-        # argnames is typically not too large
+        # if a **kwargs argument is needed, create the dict
+        w_kwds = None
+        if signature.has_kwarg():
+            w_kwds = self.space.newdict(kwargs=True)
+            scope_w[co_argcount + signature.has_vararg()] = w_kwds
+
+        # handle keyword arguments
         num_remainingkwds = 0
         used_keywords = None
         keywords_w = self.keywords_w
         if num_kwds:
             used_keywords = [False] * num_kwds
+            # match the keywords given at the call site to the argument names
+            # the called function takes
             num_remainingkwds = _match_keywords(
                     signature, blindargs, input_argcount, keywords,
                     keywords_w, scope_w, used_keywords,
                     self._jit_few_keywords)
-        if signature.has_kwarg():
-            w_kwds = self.space.newdict(kwargs=True)
-            # collect extra keyword arguments into the **kwarg
             if num_remainingkwds:
-                _collect_keyword_args(
-                        self.space, keywords, keywords_w, w_kwds,
-                        used_keywords, self.keyword_names_w, self._jit_few_keywords)
-                #
-            scope_w[co_argcount + signature.has_vararg()] = w_kwds
-        elif num_remainingkwds:
-            if co_argcount == 0:
-                raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
-            raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
-                                    used_keywords, self.keyword_names_w)
+                if w_kwds is not None:
+                    # collect extra keyword arguments into the **kwarg
+                    _collect_keyword_args(
+                            self.space, keywords, keywords_w, w_kwds,
+                            used_keywords, self.keyword_names_w, self._jit_few_keywords)
+                else:
+                    if co_argcount == 0:
+                        raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
+                    raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
+                                            used_keywords, self.keyword_names_w)
 
+        # check for missing arguments and fill them with defaults, if available
         missing = 0
         if input_argcount < co_argcount:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
@@ -323,9 +329,6 @@
                 if defnum >= 0:
                     scope_w[i] = defaults_w[defnum]
                 else:
-                    # error: not enough arguments.  Don't signal it immediately
-                    # because it might be related to a problem with */** or
-                    # keyword arguments, which will be checked for below.
                     missing += 1
             if missing:
                 raise ArgErrCount(avail, num_kwds, signature, defaults_w, missing)

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:42 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:42 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: a **kwargs pypyjit_demo
Message-ID: <20120416094642.EF8CD820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54402:6a77563ee515
Date: 2012-04-14 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/6a77563ee515/

Log:	a **kwargs pypyjit_demo

diff --git a/pypy/jit/tl/pypyjit_demo.py b/pypy/jit/tl/pypyjit_demo.py
--- a/pypy/jit/tl/pypyjit_demo.py
+++ b/pypy/jit/tl/pypyjit_demo.py
@@ -1,19 +1,27 @@
 import pypyjit
 pypyjit.set_param(threshold=200)
 
+kwargs = {"z": 1}
 
-def g(*args):
-    return len(args)
+def f(*args, **kwargs):
+    result = g(1, *args, **kwargs)
+    return result + 2
 
-def f(n):
-    s = 0
-    for i in range(n):
-        l = [i, n, 2]
-        s += g(*l)
-    return s
+def g(x, y, z=2):
+    return x - y + z
+
+def main():
+    res = 0
+    i = 0
+    while i < 10000:
+        res = f(res, z=i)
+        g(1, res, **kwargs)
+        i += 1
+    return res
+
 
 try:
-    print f(301)
+    print main()
 
 except Exception, e:
     print "Exception: ", type(e)

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:44 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:44 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: typo
Message-ID: <20120416094644.42EA7820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54403:2f6f4c9e9a4c
Date: 2012-04-14 12:25 +0200
http://bitbucket.org/pypy/pypy/changeset/2f6f4c9e9a4c/

Log:	typo

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -474,7 +474,7 @@
 
 @jit.look_inside_iff(
     lambda space, keywords, keywords_w, w_kwds, used_keywords,
-        keyword_names_w, jifon: jiton)
+        keyword_names_w, jiton: jiton)
 def _collect_keyword_args(space, keywords, keywords_w, w_kwds, used_keywords,
                           keyword_names_w, _):
     limit = len(keywords)

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:45 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:45 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: a test for view_as_kwargs
Message-ID: <20120416094645.79CC1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54404:a9db929b3390
Date: 2012-04-14 13:12 +0200
http://bitbucket.org/pypy/pypy/changeset/a9db929b3390/

Log:	a test for view_as_kwargs

diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -889,6 +889,9 @@
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance)
 
+    def view_as_kwargs(self, w_d):
+        return w_d.view_as_kwargs() # assume it's a multidict
+
     def finditem_str(self, w_dict, s):
         return w_dict.getitem_str(s) # assume it's a multidict
 
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -86,6 +86,19 @@
     d = W_DictMultiObject(space, strategy, storage)
     w_l = d.w_keys() # does not crash
 
+def test_view_as_kwargs():
+    from pypy.objspace.std.dictmultiobject import EmptyDictStrategy
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert (space.view_as_kwargs(d) == keys, values)
+
+    strategy = EmptyDictStrategy(space)
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    assert (space.view_as_kwargs(d) == [], [])
 
 from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
 def get_impl(self):

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:46 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:46 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: it still makes sense to make
 it possible to view a stringdict as a kwargs dict, 
Message-ID: <20120416094646.B56D4820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54405:f996e07586bc
Date: 2012-04-14 13:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f996e07586bc/

Log:	it still makes sense to make it possible to view a stringdict as a
	kwargs dict, because the unrolling is much more efficient than going
	through the generic space.unpackiterable.

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -508,6 +508,13 @@
     def w_keys(self, w_dict):
         return self.space.newlist_str(self.listview_str(w_dict))
 
+    def view_as_kwargs(self, w_dict):
+        d = self.unerase(w_dict.dstorage)
+        keys, values = [], []
+        for key, val in d.iteritems():
+            keys.append(key)
+            values.append(val)
+        return keys, values
 
 class _WrappedIteratorMixin(object):
     _mixin_ = True
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -1108,6 +1108,10 @@
         assert self.impl.getitem(s) == 1000
         assert s.unwrapped
 
+    def test_view_as_kwargs(self):
+        self.fill_impl()
+        assert self.fakespace.view_as_kwargs(self.impl) == (["fish", "fish2"], [1000, 2000])
+
 ## class TestMeasuringDictImplementation(BaseTestRDictImplementation):
 ##     ImplementionClass = MeasuringDictImplementation
 ##     DevolvedClass = MeasuringDictImplementation

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:48 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:48 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: small simplification
Message-ID: <20120416094648.12B47820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54406:6c92c1b370c0
Date: 2012-04-14 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/6c92c1b370c0/

Log:	small simplification

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -455,9 +455,9 @@
         if name is None:
             continue
         j = signature.find_argname(name)
-        if j < 0:
-            continue
-        elif j < input_argcount:
+        # if j == -1 nothing happens, because j < input_argcount and
+        # blindargs > j
+        if j < input_argcount:
             # check that no keyword argument conflicts with these. note
             # that for this purpose we ignore the first blindargs,
             # which were put into place by prepend().  This way,

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:49 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:49 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: make the result of
 view_as_kwargs not resizable
Message-ID: <20120416094649.4B899820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54407:1df3aae4ac9f
Date: 2012-04-14 14:55 +0200
http://bitbucket.org/pypy/pypy/changeset/1df3aae4ac9f/

Log:	make the result of view_as_kwargs not resizable

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -174,8 +174,8 @@
         keywords, values_w = space.view_as_kwargs(w_starstararg)
         if keywords is not None: # this path also taken for empty dicts
             if self.keywords is None:
-                self.keywords = keywords[:] # copy to make non-resizable
-                self.keywords_w = values_w[:]
+                self.keywords = keywords
+                self.keywords_w = values_w
             else:
                 _check_not_duplicate_kwargs(
                     self.space, self.keywords, keywords, values_w)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -510,10 +510,13 @@
 
     def view_as_kwargs(self, w_dict):
         d = self.unerase(w_dict.dstorage)
-        keys, values = [], []
+        l = len(d)
+        keys, values = [None] * l, [None] * l
+        i = 0
         for key, val in d.iteritems():
-            keys.append(key)
-            values.append(val)
+            keys[i] = key
+            values[i] = val
+            i += 1
         return keys, values
 
 class _WrappedIteratorMixin(object):
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -145,7 +145,8 @@
         w_dict.dstorage = storage
 
     def view_as_kwargs(self, w_dict):
-        return self.unerase(w_dict.dstorage)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        return keys[:], values_w[:] # copy to make non-resizable
 
 
 class KwargsDictIterator(IteratorImplementation):

From noreply at buildbot.pypy.org  Mon Apr 16 11:46:51 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 16 Apr 2012 11:46:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120416094651.EFBED820B3@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54408:dc0b12d91fe2
Date: 2012-04-16 11:45 +0200
http://bitbucket.org/pypy/pypy/changeset/dc0b12d91fe2/

Log:	merge

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -513,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1435,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1410,6 +1410,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -204,6 +204,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.call_location()
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:

From noreply at buildbot.pypy.org  Mon Apr 16 14:01:22 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 14:01:22 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20120416120122.58825820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54409:f28e32b3135a
Date: 2012-04-16 13:50 +0200
http://bitbucket.org/pypy/pypy/changeset/f28e32b3135a/

Log:	hg merge default

diff too long, truncating to 10000 out of 14415 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return pyc
+def pytest_unconfigure(config):
+    hook = config._assertstate.hook
+    if hook is not None:
+        sys.meta_path.remove(hook)
 
-def before_module_import(mod):
-    if mod.config._assertstate.mode != "on":
-        return
-    # Some deep magic: load the source, rewrite the asserts, and write a
-    # fake pyc, so that it'll be loaded when the module is imported.
-    source = mod.fspath.read()
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
-        return
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, str(mod.fspath), "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the assertion
-        # rewriting, but I don't know of a fast way to tell.
-        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
-        return
-    mod._pyc = _write_pyc(co, mod.fspath)
-    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+def pytest_collection(session):
+    # this hook is only called when test modules are collected
+    # so for example not in the master process of pytest-xdist
+    # (which does not collect test modules)
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.set_session(session)
 
-def after_module_import(mod):
-    if not hasattr(mod, "_pyc"):
-        return
-    state = mod.config._assertstate
-    try:
-        mod._pyc.remove()
-    except py.error.ENOENT:
-        state.trace("couldn't find pyc: %r" % (mod._pyc,))
-    else:
-        state.trace("removed pyc: %r" % (mod._pyc,))
+def pytest_runtest_setup(item):
+    def callbinrepr(op, left, right):
+        hook_result = item.ihook.pytest_assertrepr_compare(
+            config=item.config, op=op, left=left, right=right)
+        for new_expl in hook_result:
+            if new_expl:
+                res = '\n~'.join(new_expl)
+                if item.config.getvalue("assertmode") == "rewrite":
+                    # The result will be fed back a python % formatting
+                    # operation, which will fail if there are extraneous
+                    # '%'s in the string. Escape them here.
+                    res = res.replace("%", "%%")
+                return res
+    util._reprcompare = callbinrepr
 
-def warn_about_missing_assertion():
+def pytest_runtest_teardown(item):
+    util._reprcompare = None
+
+def pytest_sessionfinish(session):
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.session = None
+
+def _load_modules(mode):
+    """Lazily import assertion related code."""
+    global rewrite, reinterpret
+    from _pytest.assertion import reinterpret
+    if mode == "rewrite":
+        from _pytest.assertion import rewrite
+
+def warn_about_missing_assertion(mode):
     try:
         assert False
     except AssertionError:
         pass
     else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
+        if mode == "rewrite":
+            specifically = ("assertions which are not in test modules "
+                            "will be ignored")
+        else:
+            specifically = "failing tests may report as passing"
+
+        sys.stderr.write("WARNING: " + specifically +
+                        " because assert statements are not executed "
+                        "by the underlying Python interpreter "
+                        "(are you using python -O?)\n")
 
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -53,7 +53,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
+                "compute assert expression before the assert or use --assert=plain)")
 
 def run(offending_line, frame=None):
     if frame is None:
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
@@ -482,7 +481,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --nomagic)")
+                "compute assert expression before the assert or use --assert=plain)")
     else:
         return None
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,14 +1,258 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
+import errno
 import itertools
+import imp
+import marshal
+import os
+import struct
 import sys
+import types
 
 import py
 from _pytest.assertion import util
 
 
+# Windows gives ENOENT in places *nix gives ENOTDIR.
+if sys.platform.startswith("win"):
+    PATH_COMPONENT_NOT_DIR = errno.ENOENT
+else:
+    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
+
+# py.test caches rewritten pycs in __pycache__.
+if hasattr(imp, "get_tag"):
+    PYTEST_TAG = imp.get_tag() + "-PYTEST"
+else:
+    if hasattr(sys, "pypy_version_info"):
+        impl = "pypy"
+    elif sys.platform == "java":
+        impl = "jython"
+    else:
+        impl = "cpython"
+    ver = sys.version_info
+    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
+    del ver, impl
+
+PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
+
+REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+
+class AssertionRewritingHook(object):
+    """Import hook which rewrites asserts."""
+
+    def __init__(self):
+        self.session = None
+        self.modules = {}
+
+    def set_session(self, session):
+        self.fnpats = session.config.getini("python_files")
+        self.session = session
+
+    def find_module(self, name, path=None):
+        if self.session is None:
+            return None
+        sess = self.session
+        state = sess.config._assertstate
+        state.trace("find_module called for: %s" % name)
+        names = name.rsplit(".", 1)
+        lastname = names[-1]
+        pth = None
+        if path is not None and len(path) == 1:
+            pth = path[0]
+        if pth is None:
+            try:
+                fd, fn, desc = imp.find_module(lastname, path)
+            except ImportError:
+                return None
+            if fd is not None:
+                fd.close()
+            tp = desc[2]
+            if tp == imp.PY_COMPILED:
+                if hasattr(imp, "source_from_cache"):
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
+            elif tp != imp.PY_SOURCE:
+                # Don't know what this is.
+                return None
+        else:
+            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
+        fn_pypath = py.path.local(fn)
+        # Is this a test file?
+        if not sess.isinitpath(fn):
+            # We have to be very careful here because imports in this code can
+            # trigger a cycle.
+            self.session = None
+            try:
+                for pat in self.fnpats:
+                    if fn_pypath.fnmatch(pat):
+                        state.trace("matched test file %r" % (fn,))
+                        break
+                else:
+                    return None
+            finally:
+                self.session = sess
+        else:
+            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+        # The requested module looks like a test file, so rewrite it. This is
+        # the most magical part of the process: load the source, rewrite the
+        # asserts, and load the rewritten source. We also cache the rewritten
+        # module code in a special pyc. We must be aware of the possibility of
+        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # tricky race conditions, we maintain the following invariant: The
+        # cached pyc is always a complete, valid pyc. Operations on it must be
+        # atomic. POSIX's atomic rename comes in handy.
+        write = not sys.dont_write_bytecode
+        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
+        if write:
+            try:
+                os.mkdir(cache_dir)
+            except OSError:
+                e = sys.exc_info()[1].errno
+                if e == errno.EEXIST:
+                    # Either the __pycache__ directory already exists (the
+                    # common case) or it's blocked by a non-dir node. In the
+                    # latter case, we'll ignore it in _write_pyc.
+                    pass
+                elif e == PATH_COMPONENT_NOT_DIR:
+                    # One of the path components was not a directory, likely
+                    # because we're in a zip file.
+                    write = False
+                elif e == errno.EACCES:
+                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    write = False
+                else:
+                    raise
+        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
+        pyc = os.path.join(cache_dir, cache_name)
+        # Notice that even if we're in a read-only directory, I'm going to check
+        # for a cached pyc. This may not be optimal...
+        co = _read_pyc(fn_pypath, pyc)
+        if co is None:
+            state.trace("rewriting %r" % (fn,))
+            co = _rewrite_test(state, fn_pypath)
+            if co is None:
+                # Probably a SyntaxError in the test.
+                return None
+            if write:
+                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+        else:
+            state.trace("found cached rewritten pyc for %r" % (fn,))
+        self.modules[name] = co, pyc
+        return self
+
+    def load_module(self, name):
+        co, pyc = self.modules.pop(name)
+        # I wish I could just call imp.load_compiled here, but __file__ has to
+        # be set properly. In Python 3.2+, this all would be handled correctly
+        # by load_compiled.
+        mod = sys.modules[name] = imp.new_module(name)
+        try:
+            mod.__file__ = co.co_filename
+            # Normally, this attribute is 3.2+.
+            mod.__cached__ = pyc
+            py.builtin.exec_(co, mod.__dict__)
+        except:
+            del sys.modules[name]
+            raise
+        return sys.modules[name]
+
+def _write_pyc(co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as (C)Python, since
+    # these "pycs" should never be seen by builtin import. However, there's
+    # little reason deviate, and I hope sometime to be able to use
+    # imp.load_compiled to load them. (See the comment in load_module above.)
+    mtime = int(source_path.mtime())
+    try:
+        fp = open(pyc, "wb")
+    except IOError:
+        err = sys.exc_info()[1].errno
+        if err == PATH_COMPONENT_NOT_DIR:
+            # This happens when we get a EEXIST in find_module creating the
+            # __pycache__ directory and __pycache__ is by some non-dir node.
+            return False
+        raise
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return True
+
+RN = "\r\n".encode("utf-8")
+N = "\n".encode("utf-8")
+
+def _rewrite_test(state, fn):
+    """Try to read and rewrite *fn* and return the code object."""
+    try:
+        source = fn.read("rb")
+    except EnvironmentError:
+        return None
+    # On Python versions which are not 2.7 and less than or equal to 3.1, the
+    # parser expects *nix newlines.
+    if REWRITE_NEWLINES:
+        source = source.replace(RN, N) + N
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        state.trace("failed to parse: %r" % (fn,))
+        return None
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, fn.strpath, "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the
+        # assertion rewriting, but I don't know of a fast way to tell.
+        state.trace("failed to compile: %r" % (fn,))
+        return None
+    return co
+
+def _make_rewritten_pyc(state, fn, pyc, co):
+    """Try to dump rewritten code to *pyc*."""
+    if sys.platform.startswith("win"):
+        # Windows grants exclusive access to open files and doesn't have atomic
+        # rename, so just write into the final file.
+        _write_pyc(co, fn, pyc)
+    else:
+        # When not on windows, assume rename is atomic. Dump the code object
+        # into a file specific to this process and atomically replace it.
+        proc_pyc = pyc + "." + str(os.getpid())
+        if _write_pyc(co, fn, proc_pyc):
+            os.rename(proc_pyc, pyc)
+
+def _read_pyc(source, pyc):
+    """Possibly read a py.test pyc containing rewritten code.
+
+    Return rewritten code if successful or None if not.
+    """
+    try:
+        fp = open(pyc, "rb")
+    except IOError:
+        return None
+    try:
+        try:
+            mtime = int(source.mtime())
+            data = fp.read(8)
+        except EnvironmentError:
+            return None
+        # Check for invalid or out of date pyc file.
+        if (len(data) != 8 or
+            data[:4] != imp.get_magic() or
+            struct.unpack("<l", data[4:])[0] != mtime):
+            return None
+        co = marshal.load(fp)
+        if not isinstance(co, types.CodeType):
+            # That's interesting....
+            return None
+        return co
+    finally:
+        fp.close()
+
+
 def rewrite_asserts(mod):
     """Rewrite the assert statements in mod."""
     AssertionRewriter().run(mod)
@@ -17,13 +261,8 @@
 _saferepr = py.io.saferepr
 from _pytest.assertion.util import format_explanation as _format_explanation
 
-def _format_boolop(operands, explanations, is_or):
-    show_explanations = []
-    for operand, expl in zip(operands, explanations):
-        show_explanations.append(expl)
-        if operand == is_or:
-            break
-    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+def _format_boolop(explanations, is_or):
+    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -109,8 +348,8 @@
                     return
                 lineno += len(doc) - 1
                 expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +357,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +382,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +437,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +460,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +480,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +545,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,21 +25,39 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +86,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("</testcase>")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="xfail-marked test passes unexpectedly"/>')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('<failure message="test failure">%s</failure>',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<failure message="collection failure">%s</failure>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<skipped message="collection skipped">%s</skipped>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('<error message="test setup failure">%s</error>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="expected test failure">%s</skipped>',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('<skipped type="pytest.skip" '
-                           'message="%s">%s</skipped>',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +185,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n<testcase classname="pytest" name="internal">'
-            '    <error message="internal error">'
-            '%s</error></testcase>' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +203,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write('<testsuite ')
-        logfile.write('name="" ')
-        logfile.write('errors="%i" ' % self.errors)
-        logfile.write('failures="%i" ' % self.failed)
-        logfile.write('skips="%i" ' % self.skipped)
-        logfile.write('tests="%i" ' % numtests)
-        logfile.write('time="%.3f"' % suite_time_delta)
-        logfile.write(' >')
-        logfile.writelines(self.test_logs)
-        logfile.write('</testsuite>')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -399,6 +410,7 @@
         self._notfound = []
         self._initialpaths = set()
         self._initialparts = []
+        self.items = items = []
         for arg in args:
             parts = self._parsearg(arg)
             self._initialparts.append(parts)
@@ -414,7 +426,6 @@
         if not genitems:
             return rep.result
         else:
-            self.items = items = []
             if rep.passed:
                 for node in rep.result:
                     self.items.extend(self.genitems(node))
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "<CallSpec id=%r param=%r funcargs=%r>" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,71 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        if not indirect:
+            #XXX should we also check for the opposite case?
+            for arg in argnames:
+                if arg not in self.funcargnames:
+                    raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +678,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +769,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +781,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +830,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +956,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         <ExceptionInfo ...>
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         <ExceptionInfo ...>
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,60 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
-
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
+def pytest_sessionfinish(session):
+    session._setupstate.teardown_all()
 
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +76,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +93,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +114,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +143,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +166,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +188,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +208,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "<TestReport %r when=%r outcome=%r>" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +234,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +259,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +319,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -117,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
@@ -169,21 +192,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -289,10 +282,18 @@
         # we take care to leave out Instances aka ()
         # because later versions are going to get rid of them anyway
         if self.config.option.verbose < 0:
-            for item in items:
-                nodeid = item.nodeid
-                nodeid = nodeid.replace("::()::", "::")
-                self._tw.line(nodeid)
+            if self.config.option.verbose < -1:
+                counts = {}
+                for item in items:
+                    name = item.nodeid.split('::', 1)[0]
+                    counts[name] = counts.get(name, 0) + 1
+                for name, count in sorted(counts.items()):
+                    self._tw.line("%s: %d" % (name, count))
+            else:
+                for item in items:
+                    nodeid = item.nodeid
+                    nodeid = nodeid.replace("::()::", "::")
+                    self._tw.line(nodeid)
             return
         stack = []
         indent = ""
@@ -318,12 +319,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +394,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +412,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +431,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +445,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -83,28 +91,37 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +137,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib-python/modified-2.7/test/test_set.py b/lib-python/modified-2.7/test/test_set.py
--- a/lib-python/modified-2.7/test/test_set.py
+++ b/lib-python/modified-2.7/test/test_set.py
@@ -1568,7 +1568,7 @@
             for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint):
                 for g in (G, I, Ig, L, R):
                     expected = meth(data)
-                    actual = meth(G(data))
+                    actual = meth(g(data))
                     if isinstance(expected, bool):
                         self.assertEqual(actual, expected)
                     else:
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.4.dev1'
+__version__ = '1.4.7'
 
 from py import _apipkg
 
@@ -70,6 +70,11 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
+        '_AssertionError'   : '._code.assertion:AssertionError',
+        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
+        '_reinterpret'      : '._code.assertion:reinterpret',
+        '_reprcompare'      : '._code.assertion:_reprcompare',
+        '_format_explanation' : '._code.assertion:_format_explanation',
     },
 
     # backports and additions of builtins
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -113,9 +113,12 @@
 
     # some backward compatibility helpers
     _basestring = str
-    def _totext(obj, encoding=None):
+    def _totext(obj, encoding=None, errors=None):
         if isinstance(obj, bytes):
-            obj = obj.decode(encoding)
+            if errors is None:
+                obj = obj.decode(encoding)
+            else:
+                obj = obj.decode(encoding, errors)
         elif not isinstance(obj, str):
             obj = str(obj)
         return obj
@@ -142,7 +145,7 @@
             del back
         elif locs is None:
             locs = globs
-        fp = open(fn, "rb")
+        fp = open(fn, "r")
         try:
             source = fp.read()
         finally:
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionnew.py
@@ -0,0 +1,339 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace _assertionold.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from py._code.assertion import _format_explanation, BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(failure):
+    explanation = _format_explanation(failure.explanation)
+    value = failure.cause[1]
+    if str(value):
+        lines = explanation.splitlines()
+        if not lines:
+            lines.append("")
+        lines[0] += " << %s" % (value,)
+        explanation = "\n".join(lines)
+    text = "%s: %s" % (failure.cause[0].__name__, explanation)
+    if text.startswith("AssertionError: assert "):
+        text = text[16:]
+    return text
+
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = False
+        if not local:
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not result:
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        rcomp = py.code._reprcompare
+        if rcomp:
+            res = rcomp(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = True
+        if from_instance:
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        if test_explanation.startswith("False\n{False =") and \
+                test_explanation.endswith("\n"):
+            test_explanation = test_explanation[15:-2]
+        explanation = "assert %s" % (test_explanation,)
+        if not test_result:
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionold.py
@@ -0,0 +1,555 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from py._code.assertion import BuiltinAssertionError, _format_explanation
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = '<eval>'
+            self.__obj__.filename = '<eval>'
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = '<run>'
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return _format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # simplify 'assert False where False = ...'
+        if (test.explanation.startswith('False\n{False = ') and
+            test.explanation.endswith('\n}')):
+            test.explanation = test.explanation[15:-2]
+        # print the result as  'assert <explanation>'
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = '<run>'
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
new file mode 100644
--- /dev/null
+++ b/py/_code/assertion.py
@@ -0,0 +1,94 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+_reprcompare = None # if set, will be called by assert reinterp for comparison ops
+
+def _format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "<run>".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = "<could not determine information>"
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from py._code._assertionold import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from py._code._assertionnew import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,6 +145,17 @@
         return self.frame.f_locals
     locals = property(getlocals, None, None, "locals of underlaying frame")
 
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        if self.exprinfo is None:
+            source = str(self.statement).strip()
+            x = py.code._reinterpret(source, self.frame, should_fail=True)
+            if not isinstance(x, str):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
     def getfirstlinesource(self):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
@@ -158,13 +169,12 @@
         end = self.lineno
         try:
             _, end = source.getstatementrange(end)
-        except IndexError:
+        except (IndexError, ValueError):
             end = self.lineno + 1
         # heuristic to stop displaying source on e.g.
         #   if something:  # assume this causes a NameError
         #      # _this_ lines and the one
                #        below we don't want from entry.getsource()
-        end = min(end, len(source))
         for i in range(self.lineno, end):
             if source[i].rstrip().endswith(':'):
                 end = i + 1
@@ -273,7 +283,11 @@
         """
         cache = {}
         for i, entry in enumerate(self):
-            key = entry.frame.code.path, entry.lineno
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
             #print "checking for recursion at", key
             l = cache.setdefault(key, [])
             if l:
@@ -308,7 +322,7 @@
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
         self.type, self.value, tb = self._excinfo
-        self.typename = getattr(self.type, "__name__", "???")
+        self.typename = self.type.__name__
         self.traceback = py.code.Traceback(tb)
 
     def __repr__(self):
@@ -347,14 +361,16 @@
             showlocals: show locals per traceback entry
             style: long|short|no|native traceback style
             tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
         """
         if style == 'native':
-            import traceback
-            return ''.join(traceback.format_exception(
-                self.type,
-                self.value,
-                self.traceback[0]._rawentry,
-                ))
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
 
         fmt = FormattedExcinfo(showlocals=showlocals, style=style,
             abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
@@ -452,7 +468,7 @@
     def repr_locals(self, locals):
         if self.showlocals:
             lines = []
-            keys = list(locals)
+            keys = [loc for loc in locals if loc[0] != "@"]
             keys.sort()
             for name in keys:
                 value = locals[name]
@@ -506,7 +522,10 @@
 
     def _makepath(self, path):
         if not self.abspath:
-            np = py.path.local().bestrelpath(path)
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
             if len(np) < len(str(path)):
                 path = np
         return path
@@ -595,6 +614,19 @@
         if self.extraline:
             tw.line(self.extraline)
 
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
 class ReprEntry(TerminalRepr):
     localssep = "_ "
 
@@ -680,19 +712,26 @@
 
 oldbuiltins = {}
 
-def patch_builtins(compile=True):
-    """ put compile builtins to Python's builtins. """
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from py._code import assertion
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = assertion.AssertionError
     if compile:
         l = oldbuiltins.setdefault('compile', [])
         l.append(py.builtin.builtins.compile)
         py.builtin.builtins.compile = py.code.compile
 
-def unpatch_builtins(compile=True):
+def unpatch_builtins(assertion=True, compile=True):
     """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
     if compile:
         py.builtin.builtins.compile = oldbuiltins['compile'].pop()
 
-def getrawcode(obj):
+def getrawcode(obj, trycall=True):
     """ return code object for given function. """
     try:
         return obj.__code__
@@ -701,5 +740,10 @@
         obj = getattr(obj, 'func_code', obj)
         obj = getattr(obj, 'f_code', obj)
         obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
         return obj
 
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,6 +108,7 @@
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
+            raise an IndexError if no such statementrange can be found.
         """
         # XXX there must be a better than these heuristic ways ...
         # XXX there may even be better heuristics :-)
@@ -116,6 +117,7 @@
 
         # 1. find the start of the statement
         from codeop import compile_command
+        end = None
         for start in range(lineno, -1, -1):
             if assertion:
                 line = self.lines[start]
@@ -139,7 +141,9 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, len(self)
+        if end is None:
+            raise IndexError("no valid source range around line %d " % (lineno,))
+        return start, end
 
     def getblockend(self, lineno):
         # XXX
@@ -257,23 +261,29 @@
 
 
 def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
     try:
         code = py.code.Code(obj)
     except TypeError:
-        # fallback to
-        fn = (py.std.inspect.getsourcefile(obj) or
-              py.std.inspect.getfile(obj))
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
         fspath = fn and py.path.local(fn) or None
+        lineno = -1
         if fspath:
             try:
                 _, lineno = findsource(obj)
             except IOError:
-                lineno = None
-        else:
-            lineno = None
+                pass
     else:
         fspath = code.path
         lineno = code.firstlineno
+    assert isinstance(lineno, int)
     return fspath, lineno
 
 #
@@ -286,7 +296,7 @@
     except py.builtin._sysex:
         raise
     except:
-        return None, None
+        return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
diff --git a/py/_error.py b/py/_error.py
--- a/py/_error.py
+++ b/py/_error.py
@@ -23,6 +23,7 @@
     2: errno.ENOENT,
     3: errno.ENOENT,
     17: errno.EEXIST,
+    13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -103,6 +103,7 @@
     def _parseline(self, line, lineno):
         # comments
         line = line.split('#')[0].rstrip()
+        line = line.split(';')[0].rstrip()
         # blank lines
         if not line:
             return None, None
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -12,7 +12,7 @@
     class TextIO(StringIO):
         def write(self, data):
             if not isinstance(data, unicode):
-                data = unicode(data, getattr(self, '_encoding', 'UTF-8'))
+                data = unicode(data, getattr(self, '_encoding', 'UTF-8'), 'replace')
             StringIO.write(self, data)
 else:
     TextIO = StringIO
@@ -258,6 +258,9 @@
                 f = getattr(self, name).tmpfile
                 f.seek(0)
                 res = f.read()
+                enc = getattr(f, 'encoding', None)
+                if enc:
+                    res = py.builtin._totext(res, enc, 'replace')
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -105,6 +105,8 @@
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
+    _newline = None   # the last line printed
+
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -112,11 +114,9 @@
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-                if hasattr(file, 'encoding'):
-                    encoding = file.encoding
         elif hasattr(file, '__call__'):
             file = WriteFile(file, encoding=encoding)
-        self.encoding = encoding
+        self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
@@ -182,8 +182,31 @@
         return s
 
     def line(self, s='', **kw):
+        if self._newline == False:
+            self.write("\n")
         self.write(s, **kw)
         self.write('\n')
+        self._newline = True
+
+    def reline(self, line, **opts):
+        if not self.hasmarkup:
+            raise ValueError("cannot use rewrite-line without terminal")
+        if not self._newline:
+            self.write("\r")
+        self.write(line, **opts)
+        # see if we need to fill up some spaces at the end
+        # xxx have a more exact lastlinelen working from self.write?
+        lenline = len(line)
+        try:
+            lastlen = self._lastlinelen
+        except AttributeError:
+            pass
+        else:
+            if lenline < lastlen:
+                self.write(" " * (lastlen - lenline + 1))
+        self._lastlinelen = lenline
+        self._newline = False
+
 
 class Win32ConsoleWriter(TerminalWriter):
     def write(self, s, **kw):
@@ -280,10 +303,10 @@
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
     SetConsoleTextAttribute.argtypes = [wintypes.HANDLE, wintypes.WORD]
     SetConsoleTextAttribute.restype = wintypes.BOOL
-        
+
     _GetConsoleScreenBufferInfo = \
         ctypes.windll.kernel32.GetConsoleScreenBufferInfo
-    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, 
+    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE,
                                 ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
     _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
     def GetConsoleInfo(handle):
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -64,7 +64,10 @@
                 else:
                     if bool(value) ^ bool(meth()) ^ invert:
                         return False
-            except (py.error.ENOENT, py.error.ENOTDIR):
+            except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
+                # EBUSY feels not entirely correct,
+                # but its kind of necessary since ENOMEDIUM
+                # is not accessible in python     
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
@@ -368,6 +371,5 @@
         else:
             name = str(path) # path.strpath # XXX svn?
             pattern = '*' + path.sep + pattern
-        from fnmatch import fnmatch
-        return fnmatch(name, pattern)
+        return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -157,14 +157,16 @@
         return str(self) < str(other)
 
     def samefile(self, other):
-        """ return True if 'other' references the same file as 'self'. """
-        if not iswin32:
-            return py.error.checked_call(
-                    os.path.samefile, str(self), str(other))
+        """ return True if 'other' references the same file as 'self'.
+        """
+        if not isinstance(other, py.path.local):
+            other = os.path.abspath(str(other))
         if self == other:
             return True
-        other = os.path.abspath(str(other))
-        return self == other
+        if iswin32:
+            return False # ther is no samefile
+        return py.error.checked_call(
+                os.path.samefile, str(self), str(other))
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -539,7 +541,11 @@
                 if self.basename != "__init__.py":
                     modfile = modfile[:-12]
 
-            if not self.samefile(modfile):
+            try:
+                issame = self.samefile(modfile)
+            except py.error.ENOENT:
+                issame = False
+            if not issame:
                 raise self.ImportMismatchError(modname, modfile, self)
             return mod
         else:
diff --git a/py/_path/svnurl.py b/py/_path/svnurl.py
--- a/py/_path/svnurl.py
+++ b/py/_path/svnurl.py
@@ -233,6 +233,8 @@
                 e = sys.exc_info()[1]
                 if e.err.find('non-existent in that revision') != -1:
                     raise py.error.ENOENT(self, e.err)
+                elif e.err.find("E200009:") != -1:
+                    raise py.error.ENOENT(self, e.err)
                 elif e.err.find('File not found') != -1:
                     raise py.error.ENOENT(self, e.err)
                 elif e.err.find('not part of a repository')!=-1:
diff --git a/py/_path/svnwc.py b/py/_path/svnwc.py
--- a/py/_path/svnwc.py
+++ b/py/_path/svnwc.py
@@ -482,10 +482,13 @@
         except py.process.cmdexec.Error:
             e = sys.exc_info()[1]
             strerr = e.err.lower()
-            if strerr.find('file not found') != -1:
+            if strerr.find('not found') != -1:
+                raise py.error.ENOENT(self)
+            elif strerr.find("E200009:") != -1:
                 raise py.error.ENOENT(self)
             if (strerr.find('file exists') != -1 or
                 strerr.find('file already exists') != -1 or
+                strerr.find('w150002:') != -1 or
                 strerr.find("can't create directory") != -1):
                 raise py.error.EEXIST(self)
             raise
@@ -593,7 +596,7 @@
         out = self._authsvn('lock').strip()
         if not out:
             # warning or error, raise exception
-            raise Exception(out[4:])
+            raise ValueError("unknown error in svn lock command")
 
     def unlock(self):
         """ unset a previously set lock """
@@ -1066,6 +1069,8 @@
                 modrev = '?'
                 author = '?'
                 date = ''
+            elif itemstatus == "replaced":
+                pass
             else:
                 #print entryel.toxml()
                 commitel = entryel.getElementsByTagName('commit')[0]
@@ -1148,7 +1153,11 @@
             raise  ValueError("Not a versioned resource")
             #raise ValueError, "Not a versioned resource %r" % path
         self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind']
-        self.rev = int(d['revision'])
+        try:
+            self.rev = int(d['revision'])
+        except KeyError:
+            self.rev = None
+
         self.path = py.path.local(d['path'])
         self.size = self.path.size()
         if 'lastchangedrev' in d:
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -52,7 +52,7 @@
     def unicode(self, indent=2):
         l = []
         SimpleUnicodeVisitor(l.append, indent).visit(self)
-        return "".join(l)
+        return u("").join(l)
 
     def __repr__(self):
         name = self.__class__.__name__
@@ -122,11 +122,13 @@
                 if visitmethod is not None:
                     break
             else:
-                visitmethod = self.object
+                visitmethod = self.__object
             self.cache[cls] = visitmethod
         visitmethod(node)
 
-    def object(self, obj):
+    # the default fallback handler is marked private
+    # to avoid clashes with the tag name object
+    def __object(self, obj):
         #self.write(obj)
         self.write(escape(unicode(obj)))
 
@@ -136,7 +138,8 @@
     def list(self, obj):
         assert id(obj) not in self.visited
         self.visited[id(obj)] = 1
-        map(self.visit, obj)
+        for elem in obj:
+            self.visit(elem)
 
     def Tag(self, tag):
         assert id(tag) not in self.visited
@@ -181,7 +184,11 @@
             value = getattr(attrs, name)
             if name.endswith('_'):
                 name = name[:-1]
-            return ' %s="%s"' % (name, escape(unicode(value)))
+            if isinstance(value, raw):
+                insert = value.uniobj
+            else:
+                insert = escape(unicode(value))
+            return ' %s="%s"' % (name, insert)
 
     def getstyle(self, tag):
         """ return attribute list suitable for styling. """
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -483,7 +483,7 @@
                 return a_str.strip(' ')
             elif n == 1:
                 return a_str.rstrip(' ')
-            else: 
+            else:
                 return a_str.lstrip(' ')
         s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
         assert s.no_nul
@@ -3737,6 +3737,25 @@
         s = a.build_types(f, [int])
         assert s.listdef.listitem.range_step == 0
 
+    def test_specialize_arg_memo(self):
+        @objectmodel.specialize.memo()
+        def g(n):
+            return n
+        @objectmodel.specialize.arg(0)
+        def f(i):
+            return g(i)
+        def main(i):
+            if i == 2:
+                return f(2)
+            elif i == 3:
+                return f(3)
+            else:
+                raise NotImplementedError
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(main, [int])
+        assert isinstance(s, annmodel.SomeInteger)
+
 
 def g(n):
     return [0,1,2,n]
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -158,6 +158,12 @@
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
+Note that this difference might show up indirectly in some cases.  For
+example, a generator left pending in the middle is --- again ---
+garbage-collected later in PyPy than in CPython.  You can see the
+difference if the ``yield`` keyword it is suspended at is itself
+enclosed in a ``try:`` or a ``with:`` block.
+
 Using the default GC called ``minimark``, the built-in function ``id()``
 works like it does in CPython.  With other GCs it returns numbers that
 are not real addresses (because an object can move around several times)
diff --git a/pypy/doc/discussion/win64_todo.txt b/pypy/doc/discussion/win64_todo.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/discussion/win64_todo.txt
@@ -0,0 +1,9 @@
+2011-11-04
+ll_os.py has a problem with the file rwin32.py.
+Temporarily disabled for the win64_gborg branch. This needs to be
+investigated and re-enabled.
+Resolved, enabled.
+
+2011-11-05
+test_typed.py needs explicit tests to ensure that we
+handle word sizes right.
\ No newline at end of file
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -149,6 +149,22 @@
 exported.  This would give us a one-size-fits-all generic .so file to be
 imported by any application that wants to load .so files :-)
 
+Optimising cpyext (CPython C-API compatibility layer)
+-----------------------------------------------------
+
+A lot of work has gone into PyPy's implementation of CPython's C-API over
+the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However,
+there are still many edges and corner cases where it misbehaves, and it has
+not received any substantial optimisation so far.
+
+The objective of this project is to fix bugs in cpyext and to optimise
+several performance critical parts of it, such as the reference counting
+support and other heavily used C-API functions. The net result would be to
+have CPython extensions run much faster on PyPy than they currently do, or
+to make them work at all if they currently don't. A part of this work would
+be to get cpyext into a shape where it supports running Cython generated
+extensions.
 
 .. _`issue tracker`: http://bugs.pypy.org
 .. _`mailing list`: http://mail.python.org/mailman/listinfo/pypy-dev
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -199,17 +199,11 @@
 The following features (present in some past Stackless version of PyPy)
 are for the time being not supported any more:
 
-* Tasklets and channels (currently ``stackless.py`` seems to import,
-  but you have tasklets on top of coroutines on top of greenlets on
-  top of continulets on top of stacklets, and it's probably not too
-  hard to cut two of these levels by adapting ``stackless.py`` to
-  use directly continulets)
-
 * Coroutines (could be rewritten at app-level)
 
-* Pickling and unpickling continulets (*)
-
-* Continuing execution of a continulet in a different thread (*)
+* Continuing execution of a continulet in a different thread
+  (but if it is "simple enough", you can pickle it and unpickle it
+  in the other thread).
 
 * Automatic unlimited stack (must be emulated__ so far)
 
@@ -217,15 +211,6 @@
 
 .. __: `recursion depth limit`_
 
-(*) Pickling, as well as changing threads, could be implemented by using
-a "soft" stack switching mode again.  We would get either "hard" or
-"soft" switches, similarly to Stackless Python 3rd version: you get a
-"hard" switch (like now) when the C stack contains non-trivial C frames
-to save, and a "soft" switch (like previously) when it contains only
-simple calls from Python to Python.  Soft-switched continulets would
-also consume a bit less RAM, and the switch might be a bit faster too
-(unsure about that; what is the Stackless Python experience?).
-
 
 Recursion depth limit
 +++++++++++++++++++++
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
new file mode 100644
--- /dev/null
+++ b/pypy/doc/tool/makecontributor.py
@@ -0,0 +1,133 @@
+import py
+import sys
+from collections import defaultdict
+import operator
+import re
+import mercurial.localrepo
+import mercurial.ui
+
+ROOT = py.path.local(__file__).join('..', '..', '..', '..')
+author_re = re.compile('(.*) <.*>')
+pair_programming_re = re.compile(r'^\((.*?)\)')
+excluded = set(["pypy", "convert-repo"])
+
+alias = {
+    'Anders Chrigstrom': ['arre'],
+    'Antonio Cuni': ['antocuni', 'anto'],
+    'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
+    'Maciej Fijalkowski': ['fijal'],
+    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Michael Hudson': ['mwh'],
+    'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
+    "Amaury Forgeot d'Arc": ['afa'],
+    'Alex Gaynor': ['alex', 'agaynor'],
+    'David Schneider': ['bivab', 'david'],
+    'Christian Tismer': ['chris', 'christian', 'tismer',
+                         'tismer at christia-wjtqxl.localdomain'],
+    'Benjamin Peterson': ['benjamin'],
+    'Hakan Ardo': ['hakan', 'hakanardo'],
+    'Niklaus Haldimann': ['nik'],
+    'Alexander Schremmer': ['xoraxax'],
+    'Anders Hammarquist': ['iko'],
+    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'Niko Matsakis': ['niko'],
+    'Jakub Gustak': ['jlg'],
+    'Guido Wesdorp': ['guido'],
+    'Michael Foord': ['mfoord'],
+    'Mark Pearse': ['mwp'],
+    'Toon Verwaest': ['tverwaes'],
+    'Eric van Riet Paap': ['ericvrp'],
+    'Jacob Hallen': ['jacob', 'jakob'],
+    'Anders Lehmann': ['ale', 'anders'],
+    'Bert Freudenberg': ['bert'],
+    'Boris Feigin': ['boris', 'boria'],
+    'Valentino Volonghi': ['valentino', 'dialtone'],
+    'Aurelien Campeas': ['aurelien', 'aureliene'],
+    'Adrien Di Mascio': ['adim'],
+    'Jacek Generowicz': ['Jacek', 'jacek'],
+    'Jim Hunziker': ['landtuna at gmail.com'],
+    'Kristjan Valur Jonsson': ['kristjan at kristjan-lp.ccp.ad.local'],
+    'Laura Creighton': ['lac'],
+    'Aaron Iles': ['aliles'],
+    'Ludovic Aubry': ['ludal', 'ludovic'],
+    'Lukas Diekmann': ['l.diekmann', 'ldiekmann'],
+    'Matti Picus': ['Matti Picus matti.picus at gmail.com',
+                    'matthp', 'mattip', 'mattip>'],
+    'Michael Cheng': ['mikefc'],
+    'Richard Emslie': ['rxe'],
+    'Roberto De Ioris': ['roberto at goyle'],
+    'Roberto De Ioris': ['roberto at mrspurr'],
+    'Sven Hager': ['hager'],
+    'Tomo Cocoa': ['cocoatomo'],
+    }
+
+alias_map = {}
+for name, nicks in alias.iteritems():
+    for nick in nicks:
+        alias_map[nick] = name
+
+def get_canonical_author(name):
+    match = author_re.match(name)
+    if match:
+        name = match.group(1)
+    return alias_map.get(name, name)
+
+ignored_nicknames = defaultdict(int)
+
+def get_more_authors(log):
+    match = pair_programming_re.match(log)
+    if not match:
+        return set()
+    ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
+                    'in-progress', 'bits of', 'even a little', 'floating',]
+    sep_words = ['and', ';', '+', '/', 'with special  by']
+    nicknames = match.group(1)
+    for word in ignore_words:
+        nicknames = nicknames.replace(word, '')
+    for word in sep_words:
+        nicknames = nicknames.replace(word, ',')
+    nicknames = [nick.strip().lower() for nick in nicknames.split(',')]
+    authors = set()
+    for nickname in nicknames:
+        author = alias_map.get(nickname)
+        if not author:
+            ignored_nicknames[nickname] += 1
+        else:
+            authors.add(author)
+    return authors
+
+def main(show_numbers):
+    ui = mercurial.ui.ui()
+    repo = mercurial.localrepo.localrepository(ui, str(ROOT))
+    authors_count = defaultdict(int)
+    for i in repo:
+        ctx = repo[i]
+        authors = set()
+        authors.add(get_canonical_author(ctx.user()))
+        authors.update(get_more_authors(ctx.description()))
+        for author in authors:
+            if author not in excluded:
+                authors_count[author] += 1
+
+    # uncomment the next lines to get the list of nicknamed which could not be
+    # parsed from commit logs
+    ## items = ignored_nicknames.items()
+    ## items.sort(key=operator.itemgetter(1), reverse=True)
+    ## for name, n in items:
+    ##     if show_numbers:
+    ##         print '%5d %s' % (n, name)
+    ##     else:
+    ##         print name
+                
+    items = authors_count.items()
+    items.sort(key=operator.itemgetter(1), reverse=True)
+    for name, n in items:
+        if show_numbers:
+            print '%5d %s' % (n, name)
+        else:
+            print name
+
+if __name__ == '__main__':
+    show_numbers = '-n' in sys.argv
+    main(show_numbers)
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -18,7 +18,8 @@
 Edition.  Other configurations may work as well.
 
 The translation scripts will set up the appropriate environment variables
-for the compiler.  They will attempt to locate the same compiler version that
+for the compiler, so you do not need to run vcvars before translation.  
+They will attempt to locate the same compiler version that
 was used to build the Python interpreter doing the
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
@@ -26,7 +27,7 @@
 using a 32 bit Python and vice versa.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
-will be aborted in this case.
+will fail in this case.
 
 The compiler is all you need to build pypy-c, but it will miss some
 modules that relies on third-party libraries.  See below how to get
@@ -57,7 +58,8 @@
 install third-party libraries.  We chose to install them in the parent
 directory of the pypy checkout.  For example, if you installed pypy in
 ``d:\pypy\trunk\`` (This directory contains a README file), the base
-directory is ``d:\pypy``.
+directory is ``d:\pypy``. You may choose different values by setting the
+INCLUDE, LIB and PATH (for DLLs)
 
 The Boehm garbage collector
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -126,18 +128,54 @@
 ------------------------
 
 You can compile pypy with the mingw compiler, using the --cc=mingw32 option;
-mingw.exe must be on the PATH.
+gcc.exe must be on the PATH. If the -cc flag does not begin with "ming", it should be
+the name of a valid gcc-derivative compiler, i.e. x86_64-w64-mingw32-gcc for the 64 bit
+compiler creating a 64 bit target.
 
-libffi for the mingw32 compiler
+libffi for the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To enable the _rawffi (and ctypes) module, you need to compile a mingw32
-version of libffi.  I downloaded the `libffi source files`_, and extracted
-them in the base directory.  Then run::
+To enable the _rawffi (and ctypes) module, you need to compile a mingw
+version of libffi.  Here is one way to do this, wich should allow you to try
+to build for win64 or win32:
+
+#. Download and unzip a `mingw32 build`_ or `mingw64 build`_, say into c:\mingw
+#. If you do not use cygwin, you will need msys to provide make, 
+   autoconf tools and other goodies.
+
+    #. Download and unzip a `msys for mingw`_, say into c:\msys
+    #. Edit the c:\msys\etc\fstab file to mount c:\mingw
+
+#. Download and unzip the `libffi source files`_, and extract
+   them in the base directory.  
+#. Run c:\msys\msys.bat or a cygwin shell which should make you
+   feel better since it is a shell prompt with shell tools.
+#. From inside the shell, cd to the libffi directory and do::
 
     sh ./configure
     make
     cp .libs/libffi-5.dll <somewhere on the PATH>
 
+If you can't find the dll, and the libtool issued a warning about 
+"undefined symbols not allowed", you will need to edit the libffi
+Makefile in the toplevel directory. Add the flag -no-undefined to
+the definition of libffi_la_LDFLAGS
+
+If you wish to experiment with win64, you must run configure with flags::
+
+    sh ./configure --build=x86_64-w64-mingw32 --host=x86_64-w64-mingw32
+
+or such, depending on your mingw64 download.
+
+hacking on Pypy with the mingw compiler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Since hacking on Pypy means running tests, you will need a way to specify
+the mingw compiler when hacking (as opposed to translating). As of
+March 2012, --cc is not a valid option for pytest.py. However if you set an
+environment variable CC it will allow you to choose a compiler.
+
+.. _'mingw32 build': http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Automated%20Builds
+.. _`mingw64 build`: http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Automated%20Builds
+.. _`msys for mingw`: http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/MSYS%20%2832-bit%29   
 .. _`libffi source files`: http://sourceware.org/libffi/
 .. _`RPython translation toolchain`: translation.html
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/you-want-to-help.rst
@@ -0,0 +1,86 @@
+
+You want to help with PyPy, now what?
+=====================================
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  the development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy
+  approach is great if you do TDD, not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below
+
+Architecture
+============
+
+PyPy has layers. The 100 miles view:
+
+* `RPython`_ is the language in which we write interpreters. Not the entire
+  PyPy project is written in RPython, only the parts that are compiled in
+  the translation process. The interesting point is that RPython has no parser,
+  it's compiled from the live python objects, which make it possible to do
+  all kinds of metaprogramming during import time. In short, Python is a meta
+  programming language for RPython.
+
+  The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+.. _`RPython`: coding-guide.html#RPython
+
+* The translation toolchain - this is the part that takes care about translating
+  RPython to flow graphs and then to C. There is more in the `architecture`_
+  document written about it.
+
+  It mostly lives in ``rpython``, ``annotator`` and ``objspace/flow``.
+
+.. _`architecture`: architecture.html 
+
+* Python Interpreter
+
+  xxx
+
+* Python modules
+
+  xxx
+
+* Just-in-Time Compiler (JIT): `we have a tracing JIT`_ that traces the
+  interpreter written in RPython, rather than the user program that it
+  interprets.  As a result it applies to any interpreter, i.e. any
+  language.  But getting it to work correctly is not trivial: it
+  requires a small number of precise "hints" and possibly some small
+  refactorings of the interpreter.  The JIT itself also has several
+  almost-independent parts: the tracer itself in ``jit/metainterp``, the
+  optimizer in ``jit/metainterp/optimizer`` that optimizes a list of
+  residual operations, and the backend in ``jit/backend/<machine-name>``
+  that turns it into machine code.  Writing a new backend is a
+  traditional way to get into the project.
+
+.. _`we have a tracing JIT`: jit/index.html
+
+* Garbage Collectors (GC): as you can notice if you are used to CPython's
+  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+  `Garbage collection in PyPy`_ is inserted
+  during translation.  Moreover, this is not reference counting; it is a real
+  GC written as more RPython code.  The best one we have so far is in
+  ``rpython/memory/gc/minimark.py``.
+
+.. _`Garbage collection in PyPy`: garbage_collection.html
+
+
+Toolset
+=======
+
+xxx
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -182,9 +182,11 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -199,11 +201,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -240,6 +239,26 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
+        if self.keywords is None:
+            self.keywords = keywords[:] # copy to make non-resizable
+            self.keywords_w = keywords_w[:]
+        else:
+            # looks quadratic, but the JIT should remove all of it nicely.
+            # Also, all the lists should be small
+            for key in keywords:
+                for otherkey in self.keywords:
+                    if otherkey == key:
+                        raise operationerrfmt(self.space.w_TypeError,
+                                              "got multiple values "
+                                              "for keyword argument "
+                                              "'%s'", key)
+            self.keywords = self.keywords + keywords
+            self.keywords_w = self.keywords_w + keywords_w
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -417,7 +436,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -10,16 +10,6 @@
 from pypy.interpreter.astcompiler import ast, consts
 
 
-try:
-    all
-except NameError:
-    def all(iterable):
-        for x in iterable:
-            if not x:
-                return False
-        return True
-
-
 class TestAstBuilder:
 
     def setup_class(cls):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -297,6 +297,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self.frame_trace_action = FrameTraceAction(self)
+        self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
         self.our_magic = default_magic
@@ -467,9 +468,9 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: time2 or rctime take precedence over time
+        # a bit of custom logic: rctime take precedence over time
         # XXX this could probably be done as a "requires" in the config
-        if ('time2' in modules or 'rctime' in modules) and 'time' in modules:
+        if 'rctime' in modules and 'time' in modules:
             modules.remove('time')
 
         if not self.config.objspace.nofaking:
@@ -919,6 +920,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -154,6 +154,7 @@
         #operationerr.print_detailed_traceback(self.space)
 
     def _convert_exc(self, operr):
+        # Only for the flow object space
         return operr
 
     def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
@@ -166,6 +167,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -115,6 +115,12 @@
         from pypy.interpreter.pycode import PyCode
 
         code = self.getcode() # hook for the jit
+        #
+        if (jit.we_are_jitted() and code is self.space._code_of_sys_exc_info
+                                and nargs == 0):
+            from pypy.module.sys.vm import exc_info_direct
+            return exc_info_direct(self.space, frame)
+        #
         fast_natural_arity = code.fast_natural_arity
         if nargs == fast_natural_arity:
             if nargs == 0:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -880,6 +880,12 @@
             fn.add_to_table()
         if gateway.as_classmethod:
             fn = ClassMethod(space.wrap(fn))
+        #
+        from pypy.module.sys.vm import exc_info
+        if code._bltin is exc_info:
+            assert space._code_of_sys_exc_info is None
+            space._code_of_sys_exc_info = code
+        #
         return fn
 
 
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -3,29 +3,35 @@
 from pypy.rlib.rstring import StringBuilder
 
 def parsestr(space, encoding, s):
-    # compiler.transformer.Transformer.decode_literal depends on what 
-    # might seem like minor details of this function -- changes here 
-    # must be reflected there.
+    """Parses a string or unicode literal, and return a wrapped value.
+
+    If encoding=iso8859-1, the source string is also in this encoding.
+    If encoding=None, the source string is ascii only.
+    In other cases, the source string is in utf-8 encoding.
+
+    When a bytes string is returned, it will be encoded with the
+    original encoding.
+
+    Yes, it's very inefficient.
+    Yes, CPython has very similar code.
+    """
 
     # we use ps as "pointer to s"
     # q is the virtual last char index of the string
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = True
+    unicode_literal = True
 
     # string decoration handling
-    o = ord(quote)
-    isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
-    if isalpha or quote == '_':
-        if quote == 'b' or quote == 'B':
-            ps += 1
-            quote = s[ps]
-            unicode = False
-        if quote == 'r' or quote == 'R':
-            ps += 1
-            quote = s[ps]
-            rawmode = True
+    if quote == 'b' or quote == 'B':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = False
+    if quote == 'r' or quote == 'R':
+        ps += 1
+        quote = s[ps]
+        rawmode = True
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
                              'Internal error: parser passed unquoted literal')
@@ -42,21 +48,28 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
-    if unicode: # XXX Py_UnicodeFlag is ignored for now
+    if unicode_literal: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None or encoding == "iso-8859-1":
+            # 'unicode_escape' expects latin-1 bytes, string is ready.
             buf = s
             bufp = ps
             bufq = q
             u = None
         else:
-            # "\XX" may become "\u005c\uHHLL" (12 bytes)
+            # String is utf8-encoded, but 'unicode_escape' expects
+            # latin-1; So multibyte sequences must be escaped.
             lis = [] # using a list to assemble the value
             end = q
+            # Worst case: "\XX" may become "\u005c\uHHLL" (12 bytes)
             while ps < end:
                 if s[ps] == '\\':
                     lis.append(s[ps])
                     ps += 1
                     if ord(s[ps]) & 0x80:
+                        # A multibyte sequence will follow, it will be
+                        # escaped like \u1234. To avoid confusion with
+                        # the backslash we just wrote, we emit "\u005c"
+                        # instead.
                         lis.append("u005c")
                 if ord(s[ps]) & 0x80: # XXX inefficient
                     w, ps = decode_utf8(space, s, ps, end, "utf-16-be")
@@ -82,13 +95,11 @@
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
-    # XXX add strchr like interface to rtyper
     assert 0 <= ps <= q
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
             w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
-            #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -76,7 +76,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -489,6 +492,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1797,6 +1797,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1813,6 +1814,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -27,6 +27,12 @@
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
 
+def boxlonglong(ll):
+    if longlong.is_64_bit:
+        return BoxInt(ll)
+    else:
+        return BoxFloat(ll)
+
 
 class Runner(object):
 
@@ -1623,6 +1629,11 @@
                                      [boxfloat(2.5)], t).value
         assert res == longlong2float.float2longlong(2.5)
 
+        bytes = longlong2float.float2longlong(2.5)
+        res = self.execute_operation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT,
+                                     [boxlonglong(res)], 'float').value
+        assert longlong.getrealfloat(res) == 2.5
+
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -328,6 +328,15 @@
     def produce_into(self, builder, r):
         self.put(builder, [r.choice(builder.intvars)])
 
+class CastLongLongToFloatOperation(AbstractFloatOperation):
+    def produce_into(self, builder, r):
+        if longlong.is_64_bit:
+            self.put(builder, [r.choice(builder.intvars)])
+        else:
+            if not builder.floatvars:
+                raise CannotProduceOperation
+            self.put(builder, [r.choice(builder.floatvars)])
+
 class CastFloatToIntOperation(AbstractFloatOperation):
     def produce_into(self, builder, r):
         if not builder.floatvars:
@@ -450,6 +459,7 @@
 OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT))
 OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT))
 OPERATIONS.append(CastFloatToIntOperation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG))
+OPERATIONS.append(CastLongLongToFloatOperation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT))
 
 OperationBuilder.OPERATIONS = OPERATIONS
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1251,6 +1251,15 @@
         else:
             self.mov(loc0, resloc)
 
+    def genop_convert_longlong_bytes_to_float(self, op, arglocs, resloc):
+        loc0, = arglocs
+        if longlong.is_64_bit:
+            assert isinstance(resloc, RegLoc)
+            assert isinstance(loc0, RegLoc)
+            self.mc.MOVD(resloc, loc0)
+        else:
+            self.mov(loc0, resloc)
+
     def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
         guard_opnum = guard_op.getopnum()
         self.mc.CMP(arglocs[0], imm0)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -773,10 +773,24 @@
             self.Perform(op, [loc0], loc1)
             self.xrm.possibly_free_var(op.getarg(0))
         else:
-            loc0 = self.xrm.loc(op.getarg(0))
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.loc(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
+
+    def consider_convert_longlong_bytes_to_float(self, op):
+        if longlong.is_64_bit:
+            loc0 = self.rm.make_sure_var_in_reg(op.getarg(0))
             loc1 = self.xrm.force_allocate_reg(op.result)
             self.Perform(op, [loc0], loc1)
-            self.xrm.possibly_free_var(op.getarg(0))
+            self.rm.possibly_free_var(op.getarg(0))
+        else:
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.make_sure_var_in_reg(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
 
     def _consider_llong_binop_xx(self, op):
         # must force both arguments into xmm registers, because we don't
diff --git a/pypy/jit/backend/x86/rx86.py b/pypy/jit/backend/x86/rx86.py
--- a/pypy/jit/backend/x86/rx86.py
+++ b/pypy/jit/backend/x86/rx86.py
@@ -601,7 +601,9 @@
     CVTSS2SD_xb = xmminsn('\xF3', rex_nw, '\x0F\x5A',
                           register(1, 8), stack_bp(2))
 
-    # These work on machine sized registers.
+    # These work on machine sized registers, so MOVD is actually MOVQ
+    # when running on 64 bits.  Note a bug in the Intel documentation:
+    # http://lists.gnu.org/archive/html/bug-binutils/2007-07/msg00095.html
     MOVD_rx = xmminsn('\x66', rex_w, '\x0F\x7E', register(2, 8), register(1), '\xC0')
     MOVD_xr = xmminsn('\x66', rex_w, '\x0F\x6E', register(1, 8), register(2), '\xC0')
     MOVD_xb = xmminsn('\x66', rex_w, '\x0F\x6E', register(1, 8), stack_bp(2))
diff --git a/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -182,6 +182,12 @@
         filename  = str(testdir.join(FILENAME  % methname))
         g = open(inputname, 'w')
         g.write('\x09.string "%s"\n' % BEGIN_TAG)
+        #
+        if instrname == 'MOVD' and self.WORD == 8:
+            instrname = 'MOVQ'
+            if argmodes == 'xb':
+                py.test.skip('"as" uses an undocumented alternate encoding??')
+        #
         for args in args_lists:
             suffix = ""
     ##        all = instr.as_all_suffixes
@@ -229,9 +235,6 @@
                 # movq $xxx, %rax => movl $xxx, %eax
                 suffix = 'l'
                 ops[1] = reduce_to_32bit(ops[1])
-            if instrname.lower() == 'movd':
-                ops[0] = reduce_to_32bit(ops[0])
-                ops[1] = reduce_to_32bit(ops[1])
             #
             op = '\t%s%s %s%s' % (instrname.lower(), suffix,
                                   ', '.join(ops), following)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -295,6 +295,7 @@
         return op
 
     rewrite_op_convert_float_bytes_to_longlong = _noop_rewrite
+    rewrite_op_convert_longlong_bytes_to_float = _noop_rewrite
 
     # ----------
     # Various kinds of calls
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -968,20 +968,22 @@
             int_return %i2
         """, transform=True)
 
-    def test_convert_float_bytes_to_int(self):
-        from pypy.rlib.longlong2float import float2longlong
+    def test_convert_float_bytes(self):
+        from pypy.rlib.longlong2float import float2longlong, longlong2float
         def f(x):
-            return float2longlong(x)
+            ll = float2longlong(x)
+            return longlong2float(ll)
         if longlong.is_64_bit:
-            result_var = "%i0"
-            return_op = "int_return"
+            tmp_var = "%i0"
+            result_var = "%f1"
         else:
-            result_var = "%f1"
-            return_op = "float_return"
+            tmp_var = "%f1"
+            result_var = "%f2"
         self.encoding_test(f, [25.0], """
-            convert_float_bytes_to_longlong %%f0 -> %(result_var)s
-            %(return_op)s %(result_var)s
-        """ % {"result_var": result_var, "return_op": return_op})
+            convert_float_bytes_to_longlong %%f0 -> %(tmp_var)s
+            convert_longlong_bytes_to_float %(tmp_var)s -> %(result_var)s
+            float_return %(result_var)s
+        """ % {"result_var": result_var, "tmp_var": tmp_var}, transform=True)
 
 
 def check_force_cast(FROM, TO, operations, value):
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -672,6 +672,11 @@
         a = longlong.getrealfloat(a)
         return longlong2float.float2longlong(a)
 
+    @arguments(LONGLONG_TYPECODE, returns="f")
+    def bhimpl_convert_longlong_bytes_to_float(a):
+        a = longlong2float.longlong2float(a)
+        return longlong.getfloatstorage(a)
+
     # ----------
     # control flow operations
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -224,6 +224,7 @@
                     'float_neg', 'float_abs',
                     'cast_ptr_to_int', 'cast_int_to_ptr',
                     'convert_float_bytes_to_longlong',
+                    'convert_longlong_bytes_to_float',
                     ]:
         exec py.code.Source('''
             @arguments("box")
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -420,6 +420,7 @@
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     'CONVERT_FLOAT_BYTES_TO_LONGLONG/1',
+    'CONVERT_LONGLONG_BYTES_TO_FLOAT/1',
     #
     'INT_LT/2b',
     'INT_LE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,3 +1,4 @@
+import math
 import sys
 
 import py
@@ -15,7 +16,7 @@
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
     isconstant, isvirtual, promote_string, set_param, record_known_class)
-from pypy.rlib.longlong2float import float2longlong
+from pypy.rlib.longlong2float import float2longlong, longlong2float
 from pypy.rlib.rarithmetic import ovfcheck, is_valid_int
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3795,15 +3796,15 @@
         res = self.interp_operations(g, [1])
         assert res == 3
 
-    def test_float2longlong(self):
+    def test_float_bytes(self):
         def f(n):
-            return float2longlong(n)
+            ll = float2longlong(n)
+            return longlong2float(ll)
 
         for x in [2.5, float("nan"), -2.5, float("inf")]:
             # There are tests elsewhere to verify the correctness of this.
-            expected = float2longlong(x)
             res = self.interp_operations(f, [x])
-            assert longlong.getfloatstorage(res) == expected
+            assert res == x or math.isnan(x) and math.isnan(res)
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -16,13 +16,15 @@
     appleveldefs = {}
     interpleveldefs = {}
     if sys.platform.startswith("linux"):
+        from pypy.module.__pypy__ import interp_time
         interpleveldefs["clock_gettime"] = "interp_time.clock_gettime"
         interpleveldefs["clock_getres"] = "interp_time.clock_getres"
         for name in [
             "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW",
             "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID"
         ]:
-            interpleveldefs[name] = "space.wrap(interp_time.%s)" % name
+            if getattr(interp_time, name) is not None:
+                interpleveldefs[name] = "space.wrap(interp_time.%s)" % name
 
 
 class Module(MixedModule):
diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
--- a/pypy/module/__pypy__/interp_time.py
+++ b/pypy/module/__pypy__/interp_time.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import sys
 
 from pypy.interpreter.error import exception_from_errno
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -615,7 +615,8 @@
                     raise _ssl_seterror(space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                         length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -10,7 +10,7 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.objectmodel import specialize
+from pypy.rlib.objectmodel import specialize, keepalive_until_here
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
@@ -144,18 +144,24 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(RWBuffer):
-    def __init__(self, data, bytes):
-        self.data = data
-        self.len = bytes
+    def __init__(self, array):
+        self.array = array
 
     def getlength(self):
-        return self.len
+        return self.array.len * self.array.itemsize
 
     def getitem(self, index):
-        return self.data[index]
+        array = self.array
+        data = array._charbuf_start()
+        char = data[index]
+        array._charbuf_stop()
+        return char
 
     def setitem(self, index, char):
-        self.data[index] = char
+        array = self.array
+        data = array._charbuf_start()
+        data[index] = char
+        array._charbuf_stop()
 
 
 def make_array(mytype):
@@ -277,9 +283,10 @@
             oldlen = self.len
             new = len(s) / mytype.bytes
             self.setlen(oldlen + new)
-            cbuf = self.charbuf()
+            cbuf = self._charbuf_start()
             for i in range(len(s)):
                 cbuf[oldlen * mytype.bytes + i] = s[i]
+            self._charbuf_stop()
 
         def fromlist(self, w_lst):
             s = self.len
@@ -309,8 +316,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def charbuf(self):
-            return  rffi.cast(rffi.CCHARP, self.buffer)
+        def _charbuf_start(self):
+            return rffi.cast(rffi.CCHARP, self.buffer)
+
+        def _charbuf_stop(self):
+            keepalive_until_here(self)
 
         def w_getitem(self, space, idx):
             item = self.buffer[idx]
@@ -520,8 +530,10 @@
         self.fromstring(space.bytes_w(w_s))
 
     def array_tobytes__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrapbytes(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
+        self._charbuf_stop()
+        return self.space.wrap(s)
 
     def array_tostring__Array(space, self):
         space.warn("tostring() is deprecated. Use tobytes() instead.",
@@ -602,8 +614,7 @@
     # Misc methods
 
     def buffer__Array(space, self):
-        b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes)
-        return space.wrap(b)
+        return space.wrap(ArrayBuffer(self))
 
     def array_buffer_info__Array(space, self):
         w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer))
@@ -638,7 +649,7 @@
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
-        bytes = self.charbuf()
+        bytes = self._charbuf_start()
         tmp = [bytes[0]] * mytype.bytes
         for start in range(0, self.len * mytype.bytes, mytype.bytes):
             stop = start + mytype.bytes - 1
@@ -646,6 +657,7 @@
                 tmp[i] = bytes[start + i]
             for i in range(mytype.bytes):
                 bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
 
     def repr__Array(space, self):
         if self.len == 0:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -402,7 +402,25 @@
         a = self.array('h', b'Hi')
         buf = memoryview(a)
         assert buf[1] == b'i'
-        #raises(TypeError, buf.__setitem__, 1, 'o')
+
+    def test_buffer_write(self):
+        a = self.array('b', b'hello')
+        buf = memoryview(a)
+        print(repr(buf))
+        try:
+            buf[3] = b'L'
+        except TypeError:
+            skip("memoryview(array) returns a read-only buffer on CPython")
+        assert a.tostring() == 'helLo'
+
+    def test_buffer_keepalive(self):
+        buf = memoryview(self.array('b', b'text'))
+        assert buf[2] == b'x'
+        #
+        a = self.array('b', b'foobarbaz')
+        buf = memoryview(a)
+        a.fromstring(b'some extra text')
+        assert buf[:] == b'foobarbazsome extra text'
 
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,8 +2,10 @@
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
     PyObjectFields, PyObject)
-from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
+from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
+from pypy.interpreter.error import OperationError
+from pypy.module.array.interp_array import ArrayBuffer
 
 
 PyBufferObjectStruct = lltype.ForwardReference()
@@ -41,26 +43,38 @@
         py_buf.c_b_offset = w_obj.offset
         w_obj = w_obj.buffer
 
+    # If w_obj already allocated a fixed buffer, use it, and keep a
+    # reference to w_obj.
+    # Otherwise, b_base stays NULL, and we own the b_ptr.
+
     if isinstance(w_obj, StringBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.as_str()))
+        py_buf.c_b_base = lltype.nullptr(PyObject.TO)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
+        py_buf.c_b_size = w_obj.getlength()
+    elif isinstance(w_obj, ArrayBuffer):
+        w_base = w_obj.array
+        py_buf.c_b_base = make_ref(space, w_base)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start())
         py_buf.c_b_size = w_obj.getlength()
     else:
-        raise Exception("Fail fail fail fail fail")
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "buffer flavor not supported"))
 
 
 def buffer_realize(space, py_obj):
     """
     Creates the buffer in the PyPy interpreter from a cpyext representation.
     """
-    raise Exception("realize fail fail fail")
-
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "Don't know how to realize a buffer"))
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
-    Py_DecRef(space, py_buf.c_b_base)
-    rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
+    if py_buf.c_b_base:
+        Py_DecRef(space, py_buf.c_b_base)
+    else:
+        rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -314,3 +315,65 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,14 +167,16 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -183,7 +185,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2235,24 +2235,6 @@
     """Concat two strings giving a new Unicode string."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
-def PyUnicode_Split(space, s, sep, maxsplit):
-    """Split a string giving a list of Unicode strings.  If sep is NULL, splitting
-    will be done at all whitespace substrings.  Otherwise, splits occur at the given
-    separator.  At most maxsplit splits will be done.  If negative, no limit is
-    set.  Separators are not included in the resulting list.
-
-    This function used an int type for maxsplit. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyUnicode_Splitlines(space, s, keepend):
-    """Split a Unicode string at line breaks, returning a list of Unicode strings.
-    CRLF is considered to be one line break.  If keepend is 0, the Line break
-    characters are not included in the resulting strings."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.CCHARP], PyObject)
 def PyUnicode_Translate(space, str, table, errors):
     """Translate a string by applying a character mapping table to it and return the
@@ -2269,29 +2251,6 @@
     use the default error handling."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t, error=-2)
-def PyUnicode_Find(space, str, substr, start, end, direction):
-    """Return the first position of substr in str*[*start:end] using the given
-    direction (direction == 1 means to do a forward search, direction == -1 a
-    backward search).  The return value is the index of the first match; a value of
-    -1 indicates that no match was found, and -2 indicates that an error
-    occurred and an exception has been set.
-
-    This function used an int type for start and end. This
-    might require changes in your code for properly supporting 64-bit
-    systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
-def PyUnicode_Count(space, str, substr, start, end):
-    """Return the number of non-overlapping occurrences of substr in
-    str[start:end].  Return -1 if an error occurred.
-
-    This function returned an int type and used an int
-    type for start and end. This might require changes in your code for
-    properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
 def PyUnicode_RichCompare(space, left, right, op):
     """Rich compare two unicode strings and return one of the following:
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -10,7 +10,7 @@
     return False
 
 def pytest_funcarg__space(request):
-    return gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+    return gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
 
 def pytest_funcarg__api(request):
     return request.cls.api
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -19,7 +19,8 @@
 
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
-        cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+        cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi',
+                                                        'array'])
 
         # warm up reference counts:
         # - the posix module allocates a HCRYPTPROV on Windows
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -6,7 +6,6 @@
 
 class AppTestArrayModule(AppTestCpythonExtensionBase):
     enable_leak_checking = False
-    extra_modules = ['array']
 
     def test_basic(self):
         module = self.import_module(name='array')
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -48,3 +48,17 @@
             ])
         b = module.buffer_new()
         raises(AttributeError, getattr, b, 'x')
+
+    def test_array_buffer(self):
+        module = self.import_extension('foo', [
+            ("roundtrip", "METH_O",
+             """
+                 PyBufferObject *buf = (PyBufferObject *)args;
+                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
+             """),
+            ])
+        import array
+        a = array.array('c', 'text')
+        b = buffer(a)
+        assert module.roundtrip(b) == 'text'
+        
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -35,7 +35,7 @@
 
 class AppTestApi:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
         from pypy.rlib.libffi import get_libc_name
         cls.w_libc = cls.space.wrap(get_libc_name())
 
@@ -165,11 +165,9 @@
         return leaking
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
-    extra_modules = []
     
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'] +
-                                               cls.extra_modules)
+        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
         cls.space.getbuiltinmodule("cpyext")
         from pypy.module.imp.importing import importhook
         importhook(cls.space, "os") # warm up reference counts
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
@@ -414,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -425,6 +429,36 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
+
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -492,3 +492,31 @@
         assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+
+    def test_count(self, space, api):
+        w_str = space.wrap(u"abcabdab")
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, -1) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, 2) == 1
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), -5, 30) == 2
+
+    def test_find(self, space, api):
+        w_str = space.wrap(u"abcabcd")
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, 1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, 1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
+
+    def test_split(self, space, api):
+        w_str = space.wrap(u"a\nb\nc\nd")
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
+        assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 0)))
+        assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 1)))
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -306,6 +306,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -620,7 +622,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -602,3 +602,46 @@
     else:
         return stringtype.stringendswith(str, substr, start, end)
 
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
+def PyUnicode_Count(space, w_str, w_substr, start, end):
+    """Return the number of non-overlapping occurrences of substr in
+    str[start:end].  Return -1 if an error occurred."""
+    w_count = space.call_method(w_str, "count", w_substr,
+                                space.wrap(start), space.wrap(end))
+    return space.int_w(w_count)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
+             Py_ssize_t, error=-2)
+def PyUnicode_Find(space, w_str, w_substr, start, end, direction):
+    """Return the first position of substr in str*[*start:end] using
+    the given direction (direction == 1 means to do a forward search,
+    direction == -1 a backward search).  The return value is the index
+    of the first match; a value of -1 indicates that no match was
+    found, and -2 indicates that an error occurred and an exception
+    has been set."""
+    if rffi.cast(lltype.Signed, direction) > 0:
+        w_pos = space.call_method(w_str, "find", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    else:
+        w_pos = space.call_method(w_str, "rfind", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    return space.int_w(w_pos)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
+def PyUnicode_Split(space, w_str, w_sep, maxsplit):
+    """Split a string giving a list of Unicode strings.  If sep is
+    NULL, splitting will be done at all whitespace substrings.
+    Otherwise, splits occur at the given separator.  At most maxsplit
+    splits will be done.  If negative, no limit is set.  Separators
+    are not included in the resulting list."""
+    if w_sep is None:
+        w_sep = space.w_None
+    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyUnicode_Splitlines(space, w_str, keepend):
+    """Split a Unicode string at line breaks, returning a list of
+    Unicode strings.  CRLF is considered to be one line break.  If
+    keepend is 0, the Line break characters are not included in the
+    resulting strings."""
+    return space.call_method(w_str, "splitlines", space.wrap(keepend))
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
@@ -99,9 +100,12 @@
         ("exp2", "exp2"),
         ("expm1", "expm1"),
         ("fabs", "fabs"),
+        ("fmax", "fmax"),
+        ("fmin", "fmin"),
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -122,12 +126,16 @@
         ("sinh", "sinh"),
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
+        ('square', 'square'),
         ("tan", "tan"),
         ("tanh", "tanh"),
         ('bitwise_and', 'bitwise_and'),
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
+        ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
         ('isneginf', 'isneginf'),
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -16,7 +16,7 @@
         a[i][i] = 1
     return a
 
-def sum(a,axis=None):
+def sum(a,axis=None, out=None):
     '''sum(a, axis=None)
     Sum of array elements over a given axis.
 
@@ -43,17 +43,17 @@
     # TODO: add to doc (once it's implemented): cumsum : Cumulative sum of array elements.
     if not hasattr(a, "sum"):
         a = _numpypy.array(a)
-    return a.sum(axis)
+    return a.sum(axis=axis, out=out)
 
-def min(a, axis=None):
+def min(a, axis=None, out=None):
     if not hasattr(a, "min"):
         a = _numpypy.array(a)
-    return a.min(axis)
+    return a.min(axis=axis, out=out)
 
-def max(a, axis=None):
+def max(a, axis=None, out=None):
     if not hasattr(a, "max"):
         a = _numpypy.array(a)
-    return a.max(axis)
+    return a.max(axis=axis, out=out)
 
 def arange(start, stop=None, step=1, dtype=None):
     '''arange([start], stop[, step], dtype=None)
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -62,21 +62,24 @@
         return space.wrap(dtype.itemtype.bool(self))
 
     def _binop_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                            [self, w_other, w_out])
         return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
 
     def _binop_right_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, 
+                                                            [w_other, self, w_out])
         return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
 
     def _unaryop_impl(ufunc_name):
-        def impl(self, space):
+        def impl(self, space, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                                    [self, w_out])
         return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name)
 
     descr_add = _binop_impl("add")
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -387,23 +387,6 @@
             alternate_constructors=[space.w_float],
             aliases=["float"],
         )
-        self.w_longlongdtype = W_Dtype(
-            types.Int64(),
-            num=9,
-            kind=SIGNEDLTR,
-            name='int64',
-            char='q',
-            w_box_type = space.gettypefor(interp_boxes.W_LongLongBox),
-            alternate_constructors=[space.w_int],
-        )
-        self.w_ulonglongdtype = W_Dtype(
-            types.UInt64(),
-            num=10,
-            kind=UNSIGNEDLTR,
-            name='uint64',
-            char='Q',
-            w_box_type = space.gettypefor(interp_boxes.W_ULongLongBox),
-        )
         self.w_stringdtype = W_Dtype(
             types.StringType(1),
             num=18,
@@ -436,17 +419,19 @@
             self.w_booldtype, self.w_int8dtype, self.w_uint8dtype,
             self.w_int16dtype, self.w_uint16dtype, self.w_int32dtype,
             self.w_uint32dtype, self.w_longdtype, self.w_ulongdtype,
-            self.w_longlongdtype, self.w_ulonglongdtype,
+            self.w_int64dtype, self.w_uint64dtype,
             self.w_float32dtype,
             self.w_float64dtype, self.w_stringdtype, self.w_unicodedtype,
             self.w_voiddtype,
         ]
-        self.dtypes_by_num_bytes = sorted(
+        self.float_dtypes_by_num_bytes = sorted(
             (dtype.itemtype.get_element_size(), dtype)
-            for dtype in self.builtin_dtypes
+            for dtype in [self.w_float32dtype, self.w_float64dtype]
         )
         self.dtypes_by_name = {}
-        for dtype in self.builtin_dtypes:
+        # we reverse, so the stuff with lower numbers override stuff with
+        # higher numbers
+        for dtype in reversed(self.builtin_dtypes):
             self.dtypes_by_name[dtype.name] = dtype
             can_name = dtype.kind + str(dtype.itemtype.get_element_size())
             self.dtypes_by_name[can_name] = dtype
@@ -474,7 +459,7 @@
             'LONG': self.w_longdtype,
             'UNICODE': self.w_unicodedtype,
             #'OBJECT',
-            'ULONGLONG': self.w_ulonglongdtype,
+            'ULONGLONG': self.w_uint64dtype,
             'STRING': self.w_stringdtype,
             #'CDOUBLE',
             #'DATETIME',
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -269,7 +269,7 @@
 
     def apply_transformations(self, arr, transformations):
         v = BaseIterator.apply_transformations(self, arr, transformations)
-        if len(arr.shape) == 1:
+        if len(arr.shape) == 1 and len(v.res_shape) == 1:
             return OneDimIterator(self.offset, self.strides[0],
                                   self.res_shape[0])
         return v
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -83,8 +83,9 @@
         return space.wrap(W_NDimArray(shape[:], dtype=dtype))
 
     def _unaryop_impl(ufunc_name):
-        def impl(self, space):
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
+        def impl(self, space, w_out=None):
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                                [self, w_out])
         return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name)
 
     descr_pos = _unaryop_impl("positive")
@@ -93,8 +94,9 @@
     descr_invert = _unaryop_impl("invert")
 
     def _binop_impl(ufunc_name):
-        def impl(self, space, w_other):
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+        def impl(self, space, w_other, w_out=None):
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                        [self, w_other, w_out])
         return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
 
     descr_add = _binop_impl("add")
@@ -124,12 +126,12 @@
         return space.newtuple([w_quotient, w_remainder])
 
     def _binop_right_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             w_other = scalar_w(space,
                 interp_ufuncs.find_dtype_for_scalar(space, w_other, self.find_dtype()),
                 w_other
             )
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self, w_out])
         return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
 
     descr_radd = _binop_right_impl("add")
@@ -152,13 +154,21 @@
         return space.newtuple([w_quotient, w_remainder])
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
-        def impl(self, space, w_axis=None):
+        def impl(self, space, w_axis=None, w_out=None):
             if space.is_w(w_axis, space.w_None):
                 axis = -1
             else:
                 axis = space.int_w(w_axis)
+            if space.is_w(w_out, space.w_None) or not w_out:
+                out = None
+            elif not isinstance(w_out, BaseArray):
+                raise OperationError(space.w_TypeError, space.wrap( 
+                        'output must be an array'))
+            else:
+                out = w_out
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
-                                        self, True, promote_to_largest, axis)
+                                        self, True, promote_to_largest, axis,
+                                                                   False, out)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
@@ -213,6 +223,7 @@
     def descr_dot(self, space, w_other):
         other = convert_to_array(space, w_other)
         if isinstance(other, Scalar):
+            #Note: w_out is not modified, this is numpy compliant.
             return self.descr_mul(space, other)
         elif len(self.shape) < 2 and len(other.shape) < 2:
             w_res = self.descr_mul(space, other)
@@ -502,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -514,14 +548,14 @@
             )
         return w_result
 
-    def descr_mean(self, space, w_axis=None):
+    def descr_mean(self, space, w_axis=None, w_out=None):
         if space.is_w(w_axis, space.w_None):
             w_axis = space.wrap(-1)
             w_denom = space.wrap(support.product(self.shape))
         else:
             dim = space.int_w(w_axis)
             w_denom = space.wrap(self.shape[dim])
-        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
+        return space.div(self.descr_sum_promote(space, w_axis, w_out), w_denom)
 
     def descr_var(self, space, w_axis=None):
         return get_appbridge_cache(space).call_method(space, '_var', self,
@@ -662,6 +696,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -714,11 +752,12 @@
     """
     Class for representing virtual arrays, such as binary ops or ufuncs
     """
-    def __init__(self, name, shape, res_dtype):
+    def __init__(self, name, shape, res_dtype, out_arg=None):
         BaseArray.__init__(self, shape)
         self.forced_result = None
         self.res_dtype = res_dtype
         self.name = name
+        self.res = out_arg
         self.size = support.product(self.shape) * res_dtype.get_size()
 
     def _del_sources(self):
@@ -727,13 +766,18 @@
         raise NotImplementedError
 
     def compute(self):
-        ra = ResultArray(self, self.shape, self.res_dtype)
+        ra = ResultArray(self, self.shape, self.res_dtype, self.res)
         loop.compute(ra)
+        if self.res:
+            broadcast_dims = len(self.res.shape) - len(self.shape)
+            chunks = [Chunk(0,0,0,0)] * broadcast_dims + \
+                     [Chunk(0, i, 1, i) for i in self.shape]
+            return Chunks(chunks).apply(self.res)
         return ra.left
 
     def force_if_needed(self):
         if self.forced_result is None:
-            self.forced_result = self.compute()
+            self.forced_result = self.compute().get_concrete()
             self._del_sources()
 
     def get_concrete(self):
@@ -773,8 +817,9 @@
 
 
 class Call1(VirtualArray):
-    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values):
-        VirtualArray.__init__(self, name, shape, res_dtype)
+    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values,
+                                                            out_arg=None):
+        VirtualArray.__init__(self, name, shape, res_dtype, out_arg)
         self.values = values
         self.size = values.size
         self.ufunc = ufunc
@@ -786,6 +831,12 @@
     def create_sig(self):
         if self.forced_result is not None:
             return self.forced_result.create_sig()
+        if self.shape != self.values.shape:
+            #This happens if out arg is used
+            return signature.BroadcastUfunc(self.ufunc, self.name,
+                                            self.calc_dtype,
+                                            self.values.create_sig(),
+                                            self.res.create_sig())
         return signature.Call1(self.ufunc, self.name, self.calc_dtype,
                                self.values.create_sig())
 
@@ -793,8 +844,9 @@
     """
     Intermediate class for performing binary operations.
     """
-    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
-        VirtualArray.__init__(self, name, shape, res_dtype)
+    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right,
+            out_arg=None):
+        VirtualArray.__init__(self, name, shape, res_dtype, out_arg)
         self.ufunc = ufunc
         self.left = left
         self.right = right
@@ -832,8 +884,13 @@
         Call2.__init__(self, None, 'assign', shape, dtype, dtype, res, child)
 
     def create_sig(self):
-        return signature.ResultSignature(self.res_dtype, self.left.create_sig(),
-                                         self.right.create_sig())
+        if self.left.shape != self.right.shape:
+            sig = signature.BroadcastResultSignature(self.res_dtype,
+                        self.left.create_sig(), self.right.create_sig())
+        else:
+            sig = signature.ResultSignature(self.res_dtype, 
+                        self.left.create_sig(), self.right.create_sig())
+        return sig
 
 class ToStringArray(Call1):
     def __init__(self, child):
@@ -842,9 +899,9 @@
         self.s = StringBuilder(child.size * self.item_size)
         Call1.__init__(self, None, 'tostring', child.shape, dtype, dtype,
                        child)
-        self.res = W_NDimArray([1], dtype, 'C')
-        self.res_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
-                                    self.res.storage)
+        self.res_str = W_NDimArray([1], dtype, order='C')
+        self.res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
+                                    self.res_str.storage)
 
     def create_sig(self):
         return signature.ToStringSignature(self.calc_dtype,
@@ -950,7 +1007,7 @@
 
     def setitem(self, item, value):
         self.invalidated()
-        self.dtype.setitem(self, item, value)
+        self.dtype.setitem(self, item, value.convert_to(self.dtype))
 
     def calc_strides(self, shape):
         dtype = self.find_dtype()
@@ -1231,6 +1288,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1353,9 +1435,11 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,9 +3,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype, support, loop
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
+
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
@@ -28,26 +30,38 @@
         return self.identity
 
     def descr_call(self, space, __args__):
+        from interp_numarray import BaseArray
         args_w, kwds_w = __args__.unpack()
         # it occurs to me that we don't support any datatypes that
         # require casting, change it later when we do
         kwds_w.pop('casting', None)
         w_subok = kwds_w.pop('subok', None)
         w_out = kwds_w.pop('out', space.w_None)
-        if ((w_subok is not None and space.is_true(w_subok)) or
-            not space.is_w(w_out, space.w_None)):
+        # Setup a default value for out
+        if space.is_w(w_out, space.w_None):
+            out = None
+        else:
+            out = w_out
+        if (w_subok is not None and space.is_true(w_subok)):
             raise OperationError(space.w_NotImplementedError,
                                  space.wrap("parameters unsupported"))
         if kwds_w or len(args_w) < self.argcount:
             raise OperationError(space.w_ValueError,
                 space.wrap("invalid number of arguments")
             )
-        elif len(args_w) > self.argcount:
-            # The extra arguments should actually be the output array, but we
-            # don't support that yet.
+        elif (len(args_w) > self.argcount and out is not None) or \
+             (len(args_w) > self.argcount + 1):
             raise OperationError(space.w_TypeError,
                 space.wrap("invalid number of arguments")
             )
+        # Override the default out value, if it has been provided in w_wargs
+        if len(args_w) > self.argcount:
+            out = args_w[-1]
+        else:
+            args_w = args_w[:] + [out]
+        if out is not None and not isinstance(out, BaseArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                                            'output must be an array'))
         return self.call(space, args_w)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
@@ -105,28 +119,33 @@
         array([[ 1,  5],
                [ 9, 13]])
         """
-        if not space.is_w(w_out, space.w_None):
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                "out not supported"))
+        from pypy.module.micronumpy.interp_numarray import BaseArray
         if w_axis is None:
             axis = 0
         elif space.is_w(w_axis, space.w_None):
             axis = -1
         else:
             axis = space.int_w(w_axis)
-        return self.reduce(space, w_obj, False, False, axis, keepdims)
+        if space.is_w(w_out, space.w_None):
+            out = None
+        elif not isinstance(w_out, BaseArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                                                'output must be an array'))
+        else:
+            out = w_out
+        return self.reduce(space, w_obj, False, False, axis, keepdims, out)
 
-    def reduce(self, space, w_obj, multidim, promote_to_largest, dim,
-               keepdims=False):
+    def reduce(self, space, w_obj, multidim, promote_to_largest, axis,
+               keepdims=False, out=None):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, \
-                                                           Scalar, ReduceArray
+                                             Scalar, ReduceArray, W_NDimArray
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
         assert isinstance(self, W_Ufunc2)
         obj = convert_to_array(space, w_obj)
-        if dim >= len(obj.shape):
-            raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim))
+        if axis >= len(obj.shape):
+            raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % axis))
         if isinstance(obj, Scalar):
             raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
                 "on a scalar"))
@@ -144,21 +163,55 @@
         if self.identity is None and size == 0:
             raise operationerrfmt(space.w_ValueError, "zero-size array to "
                     "%s.reduce without identity", self.name)
-        if shapelen > 1 and dim >= 0:
-            return self.do_axis_reduce(obj, dtype, dim, keepdims)
-        arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
-        return loop.compute(arr)
+        if shapelen > 1 and axis >= 0:
+            if keepdims:
+                shape = obj.shape[:axis] + [1] + obj.shape[axis + 1:]
+            else:
+                shape = obj.shape[:axis] + obj.shape[axis + 1:]
+            if out:
+                #Test for shape agreement
+                if len(out.shape) > len(shape):
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter for reduction operation %s' +
+                        ' has too many dimensions', self.name)
+                elif len(out.shape) < len(shape):
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter for reduction operation %s' +
+                        ' does not have enough dimensions', self.name)
+                elif out.shape != shape:
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter shape mismatch, expecting [%s]' +
+                        ' , got [%s]',
+                        ",".join([str(x) for x in shape]),
+                        ",".join([str(x) for x in out.shape]),
+                        )
+                #Test for dtype agreement, perhaps create an itermediate
+                #if out.dtype != dtype:
+                #    raise OperationError(space.w_TypeError, space.wrap(
+                #        "mismatched  dtypes"))
+                return self.do_axis_reduce(obj, out.find_dtype(), axis, out)
+            else:
+                result = W_NDimArray(shape, dtype)
+                return self.do_axis_reduce(obj, dtype, axis, result)
+        if out:
+            if len(out.shape)>0:
+                raise operationerrfmt(space.w_ValueError, "output parameter "
+                              "for reduction operation %s has too many"
+                              " dimensions",self.name)
+            arr = ReduceArray(self.func, self.name, self.identity, obj,
+                                                            out.find_dtype())
+            val = loop.compute(arr)
+            assert isinstance(out, Scalar)
+            out.value = val
+        else:
+            arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
+            val = loop.compute(arr)
+        return val
 
-    def do_axis_reduce(self, obj, dtype, dim, keepdims):
-        from pypy.module.micronumpy.interp_numarray import AxisReduce,\
-             W_NDimArray
-        if keepdims:
-            shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
-        else:
-            shape = obj.shape[:dim] + obj.shape[dim + 1:]
-        result = W_NDimArray(shape, dtype)
+    def do_axis_reduce(self, obj, dtype, axis, result):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce
         arr = AxisReduce(self.func, self.name, self.identity, obj.shape, dtype,
-                         result, obj, dim)
+                         result, obj, axis)
         loop.compute(arr)
         return arr.left
 
@@ -176,24 +229,55 @@
         self.bool_result = bool_result
 
     def call(self, space, args_w):
-        from pypy.module.micronumpy.interp_numarray import (Call1,
-            convert_to_array, Scalar)
-
-        [w_obj] = args_w
+        from pypy.module.micronumpy.interp_numarray import (Call1, BaseArray,
+            convert_to_array, Scalar, shape_agreement)
+        if len(args_w)<2:
+            [w_obj] = args_w
+            out = None
+        else:
+            [w_obj, out] = args_w
+            if space.is_w(out, space.w_None):
+                out = None
         w_obj = convert_to_array(space, w_obj)
         calc_dtype = find_unaryop_result_dtype(space,
                                   w_obj.find_dtype(),
                                   promote_to_float=self.promote_to_float,
                                   promote_bools=self.promote_bools)
-        if self.bool_result:
+        if out:
+            if not isinstance(out, BaseArray):
+                raise OperationError(space.w_TypeError, space.wrap(
+                                                'output must be an array'))
+            res_dtype = out.find_dtype()
+        elif self.bool_result:
             res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
         if isinstance(w_obj, Scalar):
-            return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype)))
-
-        w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype,
-                      w_obj)
+            arr = self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))
+            if isinstance(out,Scalar):
+                out.value = arr
+            elif isinstance(out, BaseArray):
+                out.fill(space, arr)
+            else:
+                out = arr
+            return space.wrap(out)
+        if out:
+            assert isinstance(out, BaseArray) # For translation
+            broadcast_shape =  shape_agreement(space, w_obj.shape, out.shape)
+            if not broadcast_shape or broadcast_shape != out.shape:
+                raise operationerrfmt(space.w_ValueError,
+                    'output parameter shape mismatch, could not broadcast [%s]' +
+                    ' to [%s]',
+                    ",".join([str(x) for x in w_obj.shape]),
+                    ",".join([str(x) for x in out.shape]),
+                    )
+            w_res = Call1(self.func, self.name, out.shape, calc_dtype,
+                                         res_dtype, w_obj, out)
+            #Force it immediately
+            w_res.get_concrete()
+        else:
+            w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
+                                         res_dtype, w_obj)
         w_obj.add_invalidates(w_res)
         return w_res
 
@@ -210,34 +294,63 @@
         self.func = func
         self.comparison_func = comparison_func
 
+    @jit.unroll_safe
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
-            convert_to_array, Scalar, shape_agreement)
-
-        [w_lhs, w_rhs] = args_w
+            convert_to_array, Scalar, shape_agreement, BaseArray)
+        if len(args_w) > 2:
+            [w_lhs, w_rhs, w_out] = args_w
+        else:
+            [w_lhs, w_rhs] = args_w
+            w_out = None
         w_lhs = convert_to_array(space, w_lhs)
         w_rhs = convert_to_array(space, w_rhs)
-        calc_dtype = find_binop_result_dtype(space,
-            w_lhs.find_dtype(), w_rhs.find_dtype(),
-            int_only=self.int_only,
-            promote_to_float=self.promote_to_float,
-            promote_bools=self.promote_bools,
-        )
+        if space.is_w(w_out, space.w_None) or w_out is None:
+            out = None
+            calc_dtype = find_binop_result_dtype(space,
+                w_lhs.find_dtype(), w_rhs.find_dtype(),
+                int_only=self.int_only,
+                promote_to_float=self.promote_to_float,
+                promote_bools=self.promote_bools,
+            )
+        elif not isinstance(w_out, BaseArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                    'output must be an array'))
+        else:
+            out = w_out
+            calc_dtype = out.find_dtype()
         if self.comparison_func:
             res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
         if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar):
-            return space.wrap(self.func(calc_dtype,
+            arr = self.func(calc_dtype,
                 w_lhs.value.convert_to(calc_dtype),
                 w_rhs.value.convert_to(calc_dtype)
-            ))
+            )
+            if isinstance(out,Scalar):
+                out.value = arr
+            elif isinstance(out, BaseArray):
+                out.fill(space, arr)
+            else:
+                out = arr
+            return space.wrap(out)
         new_shape = shape_agreement(space, w_lhs.shape, w_rhs.shape)
+        # Test correctness of out.shape
+        if out and out.shape != shape_agreement(space, new_shape, out.shape):
+            raise operationerrfmt(space.w_ValueError,
+                'output parameter shape mismatch, could not broadcast [%s]' +
+                ' to [%s]',
+                ",".join([str(x) for x in new_shape]),
+                ",".join([str(x) for x in out.shape]),
+                )
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
-                      res_dtype, w_lhs, w_rhs)
+                      res_dtype, w_lhs, w_rhs, out)
         w_lhs.add_invalidates(w_res)
         w_rhs.add_invalidates(w_res)
+        if out:
+            w_res.get_concrete()
         return w_res
 
 
@@ -314,7 +427,7 @@
             return dt
         if dt.num >= 5:
             return interp_dtype.get_dtype_cache(space).w_float64dtype
-        for bytes, dtype in interp_dtype.get_dtype_cache(space).dtypes_by_num_bytes:
+        for bytes, dtype in interp_dtype.get_dtype_cache(space).float_dtypes_by_num_bytes:
             if (dtype.kind == interp_dtype.FLOATINGLTR and
                 dtype.itemtype.get_element_size() > dt.itemtype.get_element_size()):
                 return dtype
@@ -423,14 +536,18 @@
             ("reciprocal", "reciprocal", 1),
 
             ("fabs", "fabs", 1, {"promote_to_float": True}),
+            ("fmax", "fmax", 2, {"promote_to_float": True}),
+            ("fmin", "fmin", 2, {"promote_to_float": True}),
             ("fmod", "fmod", 2, {"promote_to_float": True}),
             ("floor", "floor", 1, {"promote_to_float": True}),
             ("ceil", "ceil", 1, {"promote_to_float": True}),
+            ("trunc", "trunc", 1, {"promote_to_float": True}),
             ("exp", "exp", 1, {"promote_to_float": True}),
             ("exp2", "exp2", 1, {"promote_to_float": True}),
             ("expm1", "expm1", 1, {"promote_to_float": True}),
 
             ('sqrt', 'sqrt', 1, {'promote_to_float': True}),
+            ('square', 'square', 1, {'promote_to_float': True}),
 
             ("sin", "sin", 1, {"promote_to_float": True}),
             ("cos", "cos", 1, {"promote_to_float": True}),
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -107,6 +107,10 @@
         arr.compute_first_step(self, f)
         return f
 
+    def debug_repr(self):
+        # should be overridden, but in case it isn't, provide a default
+        return str(self)
+
 class ConcreteSignature(Signature):
     _immutable_fields_ = ['dtype']
 
@@ -207,7 +211,7 @@
     def _create_iter(self, iterlist, arraylist, arr, transforms):
         from pypy.module.micronumpy.interp_numarray import VirtualSlice
         assert isinstance(arr, VirtualSlice)
-        transforms = transforms + [ViewTransform(arr.chunks)]
+        transforms = [ViewTransform(arr.chunks)] + transforms 
         self.child._create_iter(iterlist, arraylist, arr.child, transforms)
 
     def eval(self, frame, arr):
@@ -215,14 +219,18 @@
         assert isinstance(arr, VirtualSlice)
         return self.child.eval(frame, arr.child)
 
+    def debug_repr(self):
+        return 'VirtualSlice(%s)' % self.child.debug_repr()
+
 class Call1(Signature):
-    _immutable_fields_ = ['unfunc', 'name', 'child', 'dtype']
+    _immutable_fields_ = ['unfunc', 'name', 'child', 'res', 'dtype']
 
-    def __init__(self, func, name, dtype, child):
+    def __init__(self, func, name, dtype, child, res=None):
         self.unfunc = func
         self.child = child
         self.name = name
         self.dtype = dtype
+        self.res  = res
 
     def hash(self):
         return compute_hash(self.name) ^ intmask(self.child.hash() << 1)
@@ -256,6 +264,29 @@
         v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype)
         return self.unfunc(arr.calc_dtype, v)
 
+
+class BroadcastUfunc(Call1):
+    def _invent_numbering(self, cache, allnumbers):
+        self.res._invent_numbering(cache, allnumbers)
+        self.child._invent_numbering(new_cache(), allnumbers)
+
+    def debug_repr(self):
+        return 'BroadcastUfunc(%s, %s)' % (self.name, self.child.debug_repr())
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import Call1
+
+        assert isinstance(arr, Call1)
+        vtransforms = [BroadcastTransform(arr.values.shape)] + transforms
+        self.child._create_iter(iterlist, arraylist, arr.values, vtransforms)
+        self.res._create_iter(iterlist, arraylist, arr.res, transforms)
+
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_numarray import Call1
+        assert isinstance(arr, Call1)
+        v = self.child.eval(frame, arr.values).convert_to(arr.calc_dtype)
+        return self.unfunc(arr.calc_dtype, v)
+
 class Call2(Signature):
     _immutable_fields_ = ['binfunc', 'name', 'calc_dtype', 'left', 'right']
 
@@ -316,7 +347,17 @@
 
         assert isinstance(arr, ResultArray)
         offset = frame.get_final_iter().offset
-        arr.left.setitem(offset, self.right.eval(frame, arr.right))
+        val = self.right.eval(frame, arr.right)
+        arr.left.setitem(offset, val)
+
+class BroadcastResultSignature(ResultSignature):
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_numarray import ResultArray
+
+        assert isinstance(arr, ResultArray)
+        rtransforms = [BroadcastTransform(arr.left.shape)] + transforms
+        self.left._create_iter(iterlist, arraylist, arr.left, transforms)
+        self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
 class ToStringSignature(Call1):
     def __init__(self, dtype, child):
@@ -327,10 +368,10 @@
         from pypy.module.micronumpy.interp_numarray import ToStringArray
 
         assert isinstance(arr, ToStringArray)
-        arr.res.setitem(0, self.child.eval(frame, arr.values).convert_to(
+        arr.res_str.setitem(0, self.child.eval(frame, arr.values).convert_to(
             self.dtype))
         for i in range(arr.item_size):
-            arr.s.append(arr.res_casted[i])
+            arr.s.append(arr.res_str_casted[i])
 
 class BroadcastLeft(Call2):
     def _invent_numbering(self, cache, allnumbers):
@@ -341,7 +382,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, transforms)
 
@@ -354,7 +395,7 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -367,8 +408,8 @@
         from pypy.module.micronumpy.interp_numarray import Call2
 
         assert isinstance(arr, Call2)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
-        ltransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
+        ltransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, ltransforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -390,7 +431,7 @@
         frame.cur_value = self.binfunc(self.calc_dtype, frame.cur_value, rval)
 
     def debug_repr(self):
-        return 'ReduceSig(%s)' % (self.name, self.right.debug_repr())
+        return 'ReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
 
 class SliceloopSignature(Call2):
     def eval(self, frame, arr):
@@ -414,7 +455,7 @@
         from pypy.module.micronumpy.interp_numarray import SliceArray
 
         assert isinstance(arr, SliceArray)
-        rtransforms = transforms + [BroadcastTransform(arr.shape)]
+        rtransforms = [BroadcastTransform(arr.shape)] + transforms
         self.left._create_iter(iterlist, arraylist, arr.left, transforms)
         self.right._create_iter(iterlist, arraylist, arr.right, rtransforms)
 
@@ -455,6 +496,5 @@
             cur = arr.left.getitem(iterator.offset)
             value = self.binfunc(self.calc_dtype, cur, v)
         arr.left.setitem(iterator.offset, value)
-    
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py
--- a/pypy/module/micronumpy/strides.py
+++ b/pypy/module/micronumpy/strides.py
@@ -1,6 +1,7 @@
 from pypy.rlib import jit
 from pypy.interpreter.error import OperationError
 
+ at jit.look_inside_iff(lambda chunks: jit.isconstant(len(chunks)))
 def enumerate_chunks(chunks):
     result = []
     i = -1
@@ -85,9 +86,9 @@
         space.isinstance_w(w_item_or_slice, space.w_slice)):
         raise OperationError(space.w_IndexError,
                              space.wrap('unsupported iterator index'))
-            
+
     start, stop, step, lngth = space.decode_index4(w_item_or_slice, size)
-        
+
     coords = [0] * len(shape)
     i = start
     if order == 'C':
diff --git a/pypy/module/micronumpy/support.py b/pypy/module/micronumpy/support.py
--- a/pypy/module/micronumpy/support.py
+++ b/pypy/module/micronumpy/support.py
@@ -1,5 +1,9 @@
+from pypy.rlib import jit
+
+
+ at jit.unroll_safe
 def product(s):
     i = 1
     for x in s:
         i *= x
-    return i
\ No newline at end of file
+    return i
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -301,6 +301,7 @@
         else:
             raises(OverflowError, numpy.int32, 2147483648)
             raises(OverflowError, numpy.int32, '2147483648')
+        assert numpy.dtype('int32') is numpy.dtype(numpy.int32)
 
     def test_uint32(self):
         import sys
@@ -332,15 +333,11 @@
         assert numpy.dtype(numpy.int64).type is numpy.int64
         assert numpy.int64(3) == 3
 
-        if sys.maxsize >= 2 ** 63 - 1:
-            assert numpy.int64(9223372036854775807) == 9223372036854775807
-            assert numpy.int64('9223372036854775807') == 9223372036854775807
-        else:
-            raises(OverflowError, numpy.int64, 9223372036854775807)
-            raises(OverflowError, numpy.int64, '9223372036854775807')
+        assert numpy.int64(9223372036854775807) == 9223372036854775807
+        assert numpy.int64(9223372036854775807) == 9223372036854775807
 
         raises(OverflowError, numpy.int64, 9223372036854775808)
-        raises(OverflowError, numpy.int64, '9223372036854775808')
+        raises(OverflowError, numpy.int64, 9223372036854775808L)
 
     def test_uint64(self):
         import sys
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -395,11 +395,19 @@
         assert a[3] == 0.
 
     def test_newaxis(self):
-        from _numpypy import array
+        import math
+        from _numpypy import array, cos, zeros
         from numpypy.core.numeric import newaxis
         a = array(range(5))
         b = array([range(5)])
         assert (a[newaxis] == b).all()
+        a = array(range(3))
+        b = array([1, 3])
+        expected = zeros((3, 2))
+        for x in range(3):
+            for y in range(2):
+                expected[x, y] = math.cos(a[x]) * math.cos(b[y])
+        assert ((cos(a)[:,newaxis] * cos(b).T) == expected).all()
 
     def test_newaxis_slice(self):
         from _numpypy import array
@@ -995,6 +1003,10 @@
         assert a.sum() == 5
 
         raises(TypeError, 'a.sum(2, 3)')
+        d = array(0.)
+        b = a.sum(out=d)
+        assert b == d
+        assert isinstance(b, float)
 
     def test_reduce_nd(self):
         from numpypy import arange, array, multiply
@@ -1334,6 +1346,10 @@
         dims_disagree = raises(ValueError, concatenate, (a1, b1), axis=0)
         assert str(dims_disagree.value) == \
             "array dimensions must agree except for axis being concatenated"
+        a = array([1, 2, 3, 4, 5, 6])
+        a = (a + a)[::2]
+        b = concatenate((a[:3], a[-3:]))
+        assert (b == [2, 6, 10, 2, 6, 10]).all()
 
     def test_std(self):
         from _numpypy import array
@@ -1383,7 +1399,46 @@
         assert (ones(1) + ones(1)).nbytes == 8
         assert array(3.0).nbytes == 8
 
+    def test_repeat(self):
+        from _numpypy import repeat, array
+        assert (repeat([[1, 2], [3, 4]], 3) == [1, 1, 1, 2, 2, 2,
+                                                3, 3, 3, 4, 4, 4]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=0) == [[1, 2], [1, 2], [3, 4],
+                                                        [3, 4]]).all()
+        assert (repeat([[1, 2], [3, 4]], 2, axis=1) == [[1, 1, 2, 2], [3, 3,
+                                                        4, 4]]).all()
+        assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
+
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
@@ -1495,8 +1550,6 @@
         a = array([[1, 2], [3, 4], [5, 6], [7, 8],
                    [9, 10], [11, 12], [13, 14]])
         b = a[::2]
-        print a
-        print b
         assert (b == [[1, 2], [5, 6], [9, 10], [13, 14]]).all()
         c = b + b
         assert c[1][1] == 12
diff --git a/pypy/module/micronumpy/test/test_outarg.py b/pypy/module/micronumpy/test/test_outarg.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_outarg.py
@@ -0,0 +1,126 @@
+import py
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestOutArg(BaseNumpyAppTest):
+    def test_reduce_out(self):
+        from numpypy import arange, zeros, array
+        a = arange(15).reshape(5, 3)
+        b = arange(12).reshape(4,3)
+        c = a.sum(0, out=b[1])
+        assert (c == [30, 35, 40]).all()
+        assert (c == b[1]).all()
+        raises(ValueError, 'a.prod(0, out=arange(10))')
+        a=arange(12).reshape(3,2,2)
+        raises(ValueError, 'a.sum(0, out=arange(12).reshape(3,2,2))')
+        raises(ValueError, 'a.sum(0, out=arange(3))')
+        c = array([-1, 0, 1]).sum(out=zeros([], dtype=bool))
+        #You could argue that this should product False, but
+        # that would require an itermediate result. Cpython numpy
+        # gives True.
+        assert c == True
+        a = array([[-1, 0, 1], [1, 0, -1]])
+        c = a.sum(0, out=zeros((3,), dtype=bool))
+        assert (c == [True, False, True]).all()
+        c = a.sum(1, out=zeros((2,), dtype=bool))
+        assert (c == [True, True]).all()
+
+    def test_reduce_intermediary(self):
+        from numpypy import arange, array
+        a = arange(15).reshape(5, 3)
+        b = array(range(3), dtype=bool)
+        c = a.prod(0, out=b)
+        assert(b == [False,  True,  True]).all()
+
+    def test_ufunc_out(self):
+        from _numpypy import array, negative, zeros, sin
+        from math import sin as msin
+        a = array([[1, 2], [3, 4]])
+        c = zeros((2,2,2))
+        b = negative(a + a, out=c[1])
+        #test for view, and also test that forcing out also forces b
+        assert (c[:, :, 1] == [[0, 0], [-4, -8]]).all()
+        assert (b == [[-2, -4], [-6, -8]]).all()
+        #Test broadcast, type promotion
+        b = negative(3, out=a)
+        assert (a == -3).all()
+        c = zeros((2, 2), dtype=float)
+        b = negative(3, out=c)
+        assert b.dtype.kind == c.dtype.kind
+        assert b.shape == c.shape
+        a = array([1, 2])
+        b = sin(a, out=c)
+        assert(c == [[msin(1), msin(2)]] * 2).all()
+        b = sin(a, out=c+c)
+        assert (c == b).all()
+
+        #Test shape agreement
+        a = zeros((3,4))
+        b = zeros((3,5))
+        raises(ValueError, 'negative(a, out=b)')
+        b = zeros((1,4))
+        raises(ValueError, 'negative(a, out=b)')
+
+    def test_binfunc_out(self):
+        from _numpypy import array, add
+        a = array([[1, 2], [3, 4]])
+        out = array([[1, 2], [3, 4]])
+        c = add(a, a, out=out)
+        assert (c == out).all()
+        assert c.shape == a.shape
+        assert c.dtype is a.dtype
+        c[0,0] = 100
+        assert out[0, 0] == 100
+        out[:] = 100
+        raises(ValueError, 'c = add(a, a, out=out[1])')
+        c = add(a[0], a[1], out=out[1])
+        assert (c == out[1]).all()
+        assert (c == [4, 6]).all()
+        assert (out[0] == 100).all()
+        c = add(a[0], a[1], out=out)
+        assert (c == out[1]).all()
+        assert (c == out[0]).all()
+        out = array(16, dtype=int)
+        b = add(10, 10, out=out)
+        assert b==out
+        assert b.dtype == out.dtype
+        
+    def test_applevel(self):
+        from _numpypy import array, sum, max, min
+        a = array([[1, 2], [3, 4]])
+        out = array([[0, 0], [0, 0]])
+        c = sum(a, axis=0, out=out[0])
+        assert (c == [4, 6]).all()
+        assert (c == out[0]).all()
+        assert (c != out[1]).all()
+        c = max(a, axis=1, out=out[0])
+        assert (c == [2, 4]).all()
+        assert (c == out[0]).all()
+        assert (c != out[1]).all()
+        
+    def test_ufunc_cast(self):
+        from _numpypy import array, negative, add, sum
+        a = array(16, dtype = int)
+        c = array(0, dtype = float)
+        b = negative(a, out=c)
+        assert b == c
+        b = add(a, a, out=c)
+        assert b == c
+        d = array([16, 16], dtype=int)
+        b = sum(d, out=c)
+        assert b == c
+        try:
+            from _numpypy import version
+            v = version.version.split('.')
+        except:
+            v = ['1', '6', '0'] # numpypy is api compatable to what version?
+        if v[0]<'2':
+            b = negative(c, out=a)
+            assert b == a
+            b = add(c, c, out=a)
+            assert b == a
+            b = sum(array([16, 16], dtype=float), out=a)
+            assert b == a
+        else:
+            cast_error = raises(TypeError, negative, c, a)
+            assert str(cast_error.value) == \
+            "Cannot cast ufunc negative output from dtype('float64') to dtype('int64') with casting rule 'same_kind'"
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -135,6 +135,38 @@
         assert fabs(float('-inf')) == float('inf')
         assert isnan(fabs(float('nan')))
 
+    def test_fmax(self):
+        from _numpypy import fmax
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmax(a, [ninf]*5) == a).all()
+        assert (fmax(a, [inf]*5) == [inf]*5).all()
+        assert (fmax(a, [1]*5) == [1, 1, 1, 5, inf]).all()
+        assert math.isnan(fmax(nan, 0))
+        assert math.isnan(fmax(0, nan))
+        assert math.isnan(fmax(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+
+    def test_fmin(self):
+        from _numpypy import fmin
+        import math
+
+        nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf')
+
+        a = [ninf, -5, 0, 5, inf]
+        assert (fmin(a, [ninf]*5) == [ninf]*5).all()
+        assert (fmin(a, [inf]*5) == a).all()
+        assert (fmin(a, [1]*5) == [ninf, -5, 0, 1, 1]).all()
+        assert math.isnan(fmin(nan, 0))
+        assert math.isnan(fmin(0, nan))
+        assert math.isnan(fmin(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+
     def test_fmod(self):
         from _numpypy import fmod
         import math
@@ -221,24 +253,17 @@
         for i in range(3):
             assert c[i] == a[i] - b[i]
 
-    def test_floorceil(self):
-        from _numpypy import array, floor, ceil
+    def test_floorceiltrunc(self):
+        from _numpypy import array, floor, ceil, trunc
         import math
-        reference = [-2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = floor(a)
-        for i in range(5):
-            assert b[i] == reference[i]
-        reference = [-1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0]
-        a = array([-1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5])
-        b = ceil(a)
-        assert (reference == b).all()
-        inf = float("inf")
-        data = [1.5, 2.9999, -1.999, inf]
-        results = [math.floor(x) for x in data]
-        assert (floor(data) == results).all()
-        results = [math.ceil(x) for x in data]
-        assert (ceil(data) == results).all()
+        ninf, inf = float("-inf"), float("inf")
+        a = array([ninf, -1.4, -1.5, -1.0, 0.0, 1.0, 1.4, 0.5, inf])
+        assert ([ninf, -2.0, -2.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == floor(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 2.0, 1.0, inf] == ceil(a)).all()
+        assert ([ninf, -1.0, -1.0, -1.0, 0.0, 1.0, 1.0, 0.0, inf] == trunc(a)).all()
+        assert all([math.isnan(f(float("nan"))) for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("nan"))) == 1 for f in floor, ceil, trunc])
+        assert all([math.copysign(1, f(float("-nan"))) == -1 for f in floor, ceil, trunc])
 
     def test_copysign(self):
         from _numpypy import array, copysign
@@ -455,6 +480,19 @@
         assert math.isnan(sqrt(-1))
         assert math.isnan(sqrt(nan))
 
+    def test_square(self):
+        import math
+        from _numpypy import square
+
+        nan, inf, ninf = float("nan"), float("inf"), float("-inf")
+
+        assert math.isnan(square(nan))
+        assert math.isinf(square(inf))
+        assert math.isinf(square(ninf))
+        assert square(ninf) > 0
+        assert [square(x) for x in range(-5, 5)] == [x*x for x in range(-5, 5)]
+        assert math.isinf(square(1e300))
+
     def test_radians(self):
         import math
         from _numpypy import radians, array
@@ -546,10 +584,17 @@
         raises(TypeError, 'array([1.0]) & 1')
 
     def test_unary_bitops(self):
-        from _numpypy import bitwise_not, array
+        from _numpypy import bitwise_not, invert, array
         a = array([1, 2, 3, 4])
         assert (~a == [-2, -3, -4, -5]).all()
         assert (bitwise_not(a) == ~a).all()
+        assert (invert(a) == ~a).all()
+
+    def test_shift(self):
+        from _numpypy import left_shift, right_shift
+
+        assert (left_shift([5, 1], [2, 13]) == [20, 2**13]).all()
+        assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
 
     def test_comparisons(self):
         import operator
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -131,7 +131,7 @@
         #            bogus. We need to improve the situation somehow.
         self.check_simple_loop({'getinteriorfield_raw': 2,
                                 'setinteriorfield_raw': 1,
-                                'arraylen_gc': 1,
+                                'arraylen_gc': 2,
                                 'guard_true': 1,
                                 'int_lt': 1,
                                 'jump': 1,
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -500,6 +500,19 @@
     BoxType = interp_boxes.W_ULongBox
     format_code = "L"
 
+def _int64_coerce(self, space, w_item):
+    try:
+        return self._base_coerce(space, w_item)
+    except OperationError, e:
+        if not e.match(space, space.w_OverflowError):
+            raise
+    bigint = space.bigint_w(w_item)
+    try:
+        value = bigint.tolonglong()
+    except OverflowError:
+        raise OperationError(space.w_OverflowError, space.w_None)
+    return self.box(value)
+
 class Int64(BaseType, Integer):
     _attrs_ = ()
 
@@ -507,6 +520,8 @@
     BoxType = interp_boxes.W_Int64Box
     format_code = "q"
 
+    _coerce = func_with_new_name(_int64_coerce, '_coerce')
+
 class NonNativeInt64(BaseType, NonNativeInteger):
     _attrs_ = ()
 
@@ -514,6 +529,8 @@
     BoxType = interp_boxes.W_Int64Box
     format_code = "q"    
 
+    _coerce = func_with_new_name(_int64_coerce, '_coerce')
+
 def _uint64_coerce(self, space, w_item):
     try:
         return self._base_coerce(space, w_item)
@@ -614,6 +631,22 @@
         return math.fabs(v)
 
     @simple_binary_op
+    def fmax(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return max(v1, v2)
+
+    @simple_binary_op
+    def fmin(self, v1, v2):
+        if math.isnan(v1):
+            return v1
+        elif math.isnan(v2):
+            return v2
+        return min(v1, v2)
+
+    @simple_binary_op
     def fmod(self, v1, v2):
         try:
             return math.fmod(v1, v2)
@@ -635,6 +668,13 @@
         return math.ceil(v)
 
     @simple_unary_op
+    def trunc(self, v):
+        if v < 0:
+            return math.ceil(v)
+        else:
+            return math.floor(v)
+
+    @simple_unary_op
     def exp(self, v):
         try:
             return math.exp(v)
@@ -724,6 +764,10 @@
         except ValueError:
             return rfloat.NAN
 
+    @simple_unary_op
+    def square(self, v):
+        return v*v
+
     @raw_unary_op
     def isnan(self, v):
         return rfloat.isnan(v)
diff --git a/pypy/module/pyexpat/test/__init__.py b/pypy/module/pyexpat/test/__init__.py
new file mode 100644
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -128,3 +128,82 @@
         loop, = log.loops_by_filename(self.filepath)
         ops = loop.ops_by_id('look')
         assert 'call' not in log.opnames(ops)
+
+    #XXX the following tests only work with strategies enabled
+
+    def test_should_not_create_intobject_with_sets(self):
+        def main(n):
+            i = 0
+            s = set()
+            while i < n:
+                s.add(i)
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_should_not_create_stringobject_with_sets(self):
+        def main(n):
+            i = 0
+            s = set()
+            while i < n:
+                s.add(str(i))
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_should_not_create_intobject_with_lists(self):
+        def main(n):
+            i = 0
+            l = []
+            while i < n:
+                l.append(i)
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_should_not_create_stringobject_with_lists(self):
+        def main(n):
+            i = 0
+            l = []
+            while i < n:
+                l.append(str(i))
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_optimized_create_list_from_string(self):
+        def main(n):
+            i = 0
+            l = []
+            while i < n:
+                l = list("abc" * i)
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
+
+    def test_optimized_create_set_from_list(self):
+        def main(n):
+            i = 0
+            while i < n:
+                s = set([1,2,3])
+                i += 1
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        opnames = log.opnames(loop.allops())
+        assert opnames.count('new_with_vtable') == 0
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -212,7 +212,7 @@
             i19 = int_add(i12, 1)
             setfield_gc(p9, i19, descr=<FieldS .*W_AbstractSeqIterObject.inst_index .*>)
             guard_nonnull_class(p17, 146982464, descr=...)
-            i21 = getfield_gc(p17, descr=<FieldS .*W_ArrayTypei.inst_len .*>)
+            i21 = getfield_gc(p17, descr=<FieldS .*W_Array.*.inst_len .*>)
             i23 = int_lt(0, i21)
             guard_true(i23, descr=...)
             i24 = getfield_gc(p17, descr=<FieldU .*W_ArrayTypei.inst_buffer .*>)
@@ -351,3 +351,23 @@
         # the following assertion fails if the loop was cancelled due
         # to "abort: vable escape"
         assert len(log.loops_by_id("eval")) == 1
+
+    def test_sys_exc_info(self):
+        def main():
+            i = 1
+            lst = [i]
+            while i < 1000:
+                try:
+                    return lst[i]
+                except:
+                    e = sys.exc_info()[1]    # ID: exc_info
+                    if not isinstance(e, IndexError):
+                        raise
+                i += 1
+            return 42
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("exc_info")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -198,3 +198,37 @@
             i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>)
             guard_value(i49, 1, descr=...)
             ''')
+
+    def test_remove_duplicate_method_calls(self):
+        def main(n):
+            lst = []
+            for i in range(n):
+                s = 'Hello %d' % i
+                t = s.lower()   # ID: callone
+                u = s.lower()   # ID: calltwo
+                lst.append(t)
+                lst.append(u)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '''
+            p114 = call(ConstClass(ll_lower__rpy_stringPtr), p113, descr=<Callr . r EF=3>)
+            guard_no_exception(descr=...)
+            ''')
+        loop.match_by_id('calltwo', '')    # nothing
+
+    def test_move_method_call_out_of_loop(self):
+        def main(n):
+            lst = []
+            s = 'Hello %d' % n
+            for i in range(n):
+                t = s.lower()   # ID: callone
+                lst.append(t)
+            return len(','.join(lst))
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        loop, = loops
+        loop.match_by_id('callone', '')    # nothing
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -538,3 +538,124 @@
         s4 = s3.swapcase()
         assert intern(s4) is s2
 
+
+class AppTestSysExcInfoDirect:
+
+    def setup_method(self, meth):
+        self.checking = not option.runappdirect
+        if self.checking:
+            self.seen = []
+            from pypy.module.sys import vm
+            def exc_info_with_tb(*args):
+                self.seen.append("n")     # not optimized
+                return self.old[0](*args)
+            def exc_info_without_tb(*args):
+                self.seen.append("y")     # optimized
+                return self.old[1](*args)
+            self.old = [vm.exc_info_with_tb, vm.exc_info_without_tb]
+            vm.exc_info_with_tb = exc_info_with_tb
+            vm.exc_info_without_tb = exc_info_without_tb
+            #
+            from pypy.rlib import jit
+            self.old2 = [jit.we_are_jitted]
+            jit.we_are_jitted = lambda: True
+
+    def teardown_method(self, meth):
+        if self.checking:
+            from pypy.module.sys import vm
+            from pypy.rlib import jit
+            vm.exc_info_with_tb = self.old[0]
+            vm.exc_info_without_tb = self.old[1]
+            jit.we_are_jitted = self.old2[0]
+            #
+            assert ''.join(self.seen) == meth.expected
+
+    def test_returns_none(self):
+        import sys
+        assert sys.exc_info() == (None, None, None)
+        assert sys.exc_info()[0] is None
+        assert sys.exc_info()[1] is None
+        assert sys.exc_info()[2] is None
+        assert sys.exc_info()[:2] == (None, None)
+        assert sys.exc_info()[:3] == (None, None, None)
+        assert sys.exc_info()[0:2] == (None, None)
+        assert sys.exc_info()[2:4] == (None,)
+    test_returns_none.expected = 'nnnnnnnn'
+
+    def test_returns_subscr(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert sys.exc_info()[0] is KeyError  # y
+            assert sys.exc_info()[1] is e         # y
+            assert sys.exc_info()[2] is not None  # n
+            assert sys.exc_info()[-3] is KeyError # y
+            assert sys.exc_info()[-2] is e        # y
+            assert sys.exc_info()[-1] is not None # n
+    test_returns_subscr.expected = 'yynyyn'
+
+    def test_returns_slice_2(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                  # n
+            assert sys.exc_info()[:0] == ()       # y
+            assert sys.exc_info()[:1] == foo[:1]  # y
+            assert sys.exc_info()[:2] == foo[:2]  # y
+            assert sys.exc_info()[:3] == foo      # n
+            assert sys.exc_info()[:4] == foo      # n
+            assert sys.exc_info()[:-1] == foo[:2] # y
+            assert sys.exc_info()[:-2] == foo[:1] # y
+            assert sys.exc_info()[:-3] == ()      # y
+    test_returns_slice_2.expected = 'nyyynnyyy'
+
+    def test_returns_slice_3(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            foo = sys.exc_info()                   # n
+            assert sys.exc_info()[2:2] == ()       # y
+            assert sys.exc_info()[0:1] == foo[:1]  # y
+            assert sys.exc_info()[1:2] == foo[1:2] # y
+            assert sys.exc_info()[0:3] == foo      # n
+            assert sys.exc_info()[2:4] == foo[2:]  # n
+            assert sys.exc_info()[0:-1] == foo[:2] # y
+            assert sys.exc_info()[0:-2] == foo[:1] # y
+            assert sys.exc_info()[5:-3] == ()      # y
+    test_returns_slice_3.expected = 'nyyynnyyy'
+
+    def test_strange_invocation(self):
+        import sys
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            a = []; k = {}
+            assert sys.exc_info(*a)[:0] == ()
+            assert sys.exc_info(**k)[:0] == ()
+    test_strange_invocation.expected = 'nn'
+
+    def test_call_in_subfunction(self):
+        import sys
+        def g():
+            # this case is not optimized, because we need to search the
+            # frame chain.  it's probably not worth the complications
+            return sys.exc_info()[1]
+        e = KeyError("boom")
+        try:
+            raise e
+        except:
+            assert g() is e
+    test_call_in_subfunction.expected = 'n'
+
+
+class AppTestSysExcInfoDirectCallMethod(AppTestSysExcInfoDirect):
+    def setup_class(cls):
+        cls.space = gettestobjspace(**{"objspace.opcodes.CALL_METHOD": True})
+>>>>>>> other
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (3, 2, 2, "final", 0)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 8, 1, "dev", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 9, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -89,6 +89,9 @@
     """Return the (type, value, traceback) of the most recent exception
 caught by an except clause in the current stack frame or in an older stack
 frame."""
+    return exc_info_with_tb(space)    # indirection for the tests
+
+def exc_info_with_tb(space):
     operror = space.getexecutioncontext().sys_exc_info()
     if operror is None:
         return space.newtuple([space.w_None,space.w_None,space.w_None])
@@ -96,6 +99,68 @@
         return space.newtuple([operror.w_type, operror.get_w_value(space),
                                space.wrap(operror.get_traceback())])
 
+def exc_info_without_tb(space, frame):
+    operror = frame.last_exception
+    return space.newtuple([operror.w_type, operror.get_w_value(space),
+                           space.w_None])
+
+def exc_info_direct(space, frame):
+    from pypy.tool import stdlib_opcode
+    # In order to make the JIT happy, we try to return (exc, val, None)
+    # instead of (exc, val, tb).  We can do that only if we recognize
+    # the following pattern in the bytecode:
+    #       CALL_FUNCTION/CALL_METHOD         <-- invoking me
+    #       LOAD_CONST 0, 1, -2 or -3
+    #       BINARY_SUBSCR
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST <=2
+    #       SLICE_2
+    # or:
+    #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST any integer
+    #       LOAD_CONST <=2
+    #       SLICE_3
+    need_all_three_args = True
+    co = frame.getcode().co_code
+    p = frame.last_instr
+    if (ord(co[p]) == stdlib_opcode.CALL_FUNCTION or
+        ord(co[p]) == stdlib_opcode.CALL_METHOD):
+        if ord(co[p+3]) == stdlib_opcode.LOAD_CONST:
+            lo = ord(co[p+4])
+            hi = ord(co[p+5])
+            w_constant = frame.getconstant_w((hi * 256) | lo)
+            if space.isinstance_w(w_constant, space.w_int):
+                constant = space.int_w(w_constant)
+                if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+                    if -3 <= constant <= 1 and constant != -1:
+                        need_all_three_args = False
+                elif ord(co[p+6]) == stdlib_opcode.SLICE+2:
+                    if constant <= 2:
+                        need_all_three_args = False
+                elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
+                      ord(co[p+9]) == stdlib_opcode.SLICE+3):
+                    lo = ord(co[p+7])
+                    hi = ord(co[p+8])
+                    w_constant = frame.getconstant_w((hi * 256) | lo)
+                    if space.isinstance_w(w_constant, space.w_int):
+                        if space.int_w(w_constant) <= 2:
+                            need_all_three_args = False
+    #
+    if need_all_three_args or frame.last_exception is None or frame.hide():
+        return exc_info_with_tb(space)
+    else:
+        return exc_info_without_tb(space, frame)
+
+def exc_clear(space):
+    """Clear global information on the current exception.  Subsequent calls
+to exc_info() will return (None,None,None) until another exception is
+raised and caught in the current thread or the execution stack returns to a
+frame where another exception is being handled."""
+    operror = space.getexecutioncontext().sys_exc_info()
+    if operror is not None:
+        operror.clear(space)
+
 def settrace(space, w_func):
     """Set the global debug tracing function.  It will be called on each
 function call.  See the debugger chapter in the library manual."""
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -141,4 +141,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py
--- a/pypy/module/thread/test/test_ll_thread.py
+++ b/pypy/module/thread/test/test_ll_thread.py
@@ -66,7 +66,6 @@
     def test_gc_locking(self):
         import time
         from pypy.rlib.objectmodel import invoke_around_extcall
-        from pypy.rlib.objectmodel import we_are_translated
         from pypy.rlib.debug import ll_assert
 
         class State:
@@ -129,8 +128,6 @@
             state.finished = 0
             # the next line installs before_extcall() and after_extcall()
             # to be called automatically around external function calls.
-            # When not translated it does not work around time.sleep(),
-            # so we have to call them manually for this test.
             invoke_around_extcall(before_extcall, after_extcall)
 
             g(10, 1)
@@ -142,13 +139,9 @@
                 willing_to_wait_more -= 1
                 done = len(state.answers) == expected
 
-                if not we_are_translated(): before_extcall()
                 time.sleep(0.01)
-                if not we_are_translated(): after_extcall()
 
-            if not we_are_translated(): before_extcall()
             time.sleep(0.1)
-            if not we_are_translated(): after_extcall()
 
             return len(state.answers)
 
@@ -160,12 +153,11 @@
         answers = fn()
         assert answers == expected
 
-class TestRunDirectly(AbstractThreadTests):
-    def getcompiled(self, f, argtypes):
-        return f
-
-    def test_start_new_thread(self):
-        py.test.skip("deadlocks occasionally -- why???")
+#class TestRunDirectly(AbstractThreadTests):
+#    def getcompiled(self, f, argtypes):
+#        return f
+# These are disabled because they crash occasionally for bad reasons
+# related to the fact that ll2ctypes is not at all thread-safe
 
 class TestUsingBoehm(AbstractThreadTests):
     gcpolicy = 'boehm'
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -42,6 +42,13 @@
     return w_eq
 type_eq._annspecialcase_ = 'specialize:memo'
 
+def list_iter(space):
+    "Utility that returns the app-level descriptor list.__iter__."
+    w_src, w_iter = space.lookup_in_type_where(space.w_list,
+                                               '__iter__')
+    return w_iter
+list_iter._annspecialcase_ = 'specialize:memo'
+
 def raiseattrerror(space, w_obj, name, w_descr=None):
     w_type = space.type(w_obj)
     typename = w_type.getname(space)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -86,6 +86,7 @@
         return s_None
 
     def specialize_call(self, hop):
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)
 
 # ____________________________________________________________
@@ -109,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -434,6 +434,13 @@
         self.lastblock = block
         self.pushvalue(w_result)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # This opcode was added with pypy-1.8.  Here is a simpler
+        # version, enough for annotation.
+        last_val = self.popvalue()
+        self.pushvalue(self.space.newlist([]))
+        self.pushvalue(last_val)
+
     # XXX Unimplemented 2.7 opcodes ----------------
 
     # Set literals, set comprehensions
diff --git a/pypy/objspace/flow/model.py b/pypy/objspace/flow/model.py
--- a/pypy/objspace/flow/model.py
+++ b/pypy/objspace/flow/model.py
@@ -7,8 +7,7 @@
 from pypy.tool.uid import uid, Hashable
 from pypy.tool.descriptor import roproperty
 from pypy.tool.sourcetools import PY_IDENTIFIER, nice_repr_for_func
-from pypy.tool.identity_dict import identity_dict
-from pypy.rlib.rarithmetic import is_valid_int, r_longlong, r_ulonglong
+from pypy.rlib.rarithmetic import is_valid_int, r_longlong, r_ulonglong, r_uint
 
 
 """
@@ -546,7 +545,7 @@
                     for n in cases[:len(cases)-has_default]:
                         if is_valid_int(n):
                             continue
-                        if isinstance(n, (r_longlong, r_ulonglong)):
+                        if isinstance(n, (r_longlong, r_ulonglong, r_uint)):
                             continue
                         if isinstance(n, (str, unicode)) and len(n) == 1:
                             continue
diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py
--- a/pypy/objspace/flow/test/test_objspace.py
+++ b/pypy/objspace/flow/test/test_objspace.py
@@ -1,6 +1,6 @@
 from __future__ import with_statement
 import new
-import py
+import py, sys
 from pypy.objspace.flow.model import Constant, Block, Link, Variable
 from pypy.objspace.flow.model import mkentrymap, c_last_exception
 from pypy.interpreter.argument import Arguments
@@ -893,6 +893,8 @@
         """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG
         bytecode
         """
+        if sys.version_info < (2, 7):
+            py.test.skip("2.7 only test")
         self.patch_opcodes('BUILD_LIST_FROM_ARG')
         try:
             def f():
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -66,9 +66,15 @@
     length = len(data)
     start, stop, step, slicelength = w_slice.indices4(space, length)
     assert slicelength >= 0
-    newdata = [data[start + i*step] for i in range(slicelength)]
+    if step == 1 and 0 <= start <= stop:
+        newdata = data[start:stop]
+    else:
+        newdata = _getitem_slice_multistep(data, start, step, slicelength)
     return W_BytearrayObject(newdata)
 
+def _getitem_slice_multistep(data, start, step, slicelength):
+    return [data[start + i*step] for i in range(slicelength)]
+
 def contains__Bytearray_Int(space, w_bytearray, w_char):
     char = space.int_w(w_char)
     if not 0 <= char < 256:
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -127,10 +127,10 @@
     def iter(self, w_dict):
         return ModuleDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
+    def w_keys(self, w_dict):
         space = self.space
-        iterator = self.unerase(w_dict.dstorage).iteritems
-        return [space.wrap(key) for key, cell in iterator()]
+        l = self.unerase(w_dict.dstorage).keys()
+        return space.newlist_str(l)
 
     def values(self, w_dict):
         iterator = self.unerase(w_dict.dstorage).itervalues
@@ -163,7 +163,8 @@
 
 class ModuleDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         dict_w = strategy.unerase(dictimplementation.dstorage)
         self.iterator = dict_w.iteritems()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,8 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, classofinstance=None,
-                                   strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -47,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -90,9 +93,10 @@
 def _add_indirections():
     dict_methods = "setitem setitem_str getitem \
                     getitem_str delitem length \
-                    clear keys values \
+                    clear w_keys values \
                     items iter setdefault \
-                    popitem".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -113,7 +117,7 @@
     def get_empty_storage(self):
         raise NotImplementedError
 
-    def keys(self, w_dict):
+    def w_keys(self, w_dict):
         iterator = self.iter(w_dict)
         result = []
         while 1:
@@ -121,7 +125,7 @@
             if w_key is not None:
                 result.append(w_key)
             else:
-                return result
+                return self.space.newlist(result)
 
     def values(self, w_dict):
         iterator = self.iter(w_dict)
@@ -160,6 +164,14 @@
         w_dict.strategy = strategy
         w_dict.dstorage = storage
 
+    def listview_str(self, w_dict):
+        return None
+
+    def listview_int(self, w_dict):
+        return None
+
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
 
 class EmptyDictStrategy(DictStrategy):
 
@@ -244,7 +256,7 @@
         return 0
 
     def iter(self, w_dict):
-        return EmptyIteratorImplementation(self.space, w_dict)
+        return EmptyIteratorImplementation(self.space, self, w_dict)
 
     def clear(self, w_dict):
         return
@@ -252,6 +264,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
@@ -260,8 +275,9 @@
 # Iterator Implementation base classes
 
 class IteratorImplementation(object):
-    def __init__(self, space, implementation):
+    def __init__(self, space, strategy, implementation):
         self.space = space
+        self.strategy = strategy
         self.dictimplementation = implementation
         self.len = implementation.length()
         self.pos = 0
@@ -277,7 +293,20 @@
         if self.pos < self.len:
             result = self.next_entry()
             self.pos += 1
-            return result
+            if self.strategy is self.dictimplementation.strategy:
+                return result      # common case
+            else:
+                # waaa, obscure case: the strategy changed, but not the
+                # length of the dict.  The (key, value) pair in 'result'
+                # might be out-of-date.  We try to explicitly look up
+                # the key in the dict.
+                w_key = result[0]
+                w_value = self.dictimplementation.getitem(w_key)
+                if w_value is None:
+                    self.len = -1   # Make this error state sticky
+                    raise OperationError(self.space.w_RuntimeError,
+                        self.space.wrap("dictionary changed during iteration"))
+                return (w_key, w_value)
         # no more entries
         self.dictimplementation = None
         return None, None
@@ -373,8 +402,9 @@
             self.switch_to_object_strategy(w_dict)
             return w_dict.getitem(w_key)
 
-    def keys(self, w_dict):
-        return [self.wrap(key) for key in self.unerase(w_dict.dstorage).iterkeys()]
+    def w_keys(self, w_dict):
+        l = [self.wrap(key) for key in self.unerase(w_dict.dstorage).iterkeys()]
+        return self.space.newlist(l)
 
     def values(self, w_dict):
         return self.unerase(w_dict.dstorage).values()
@@ -427,8 +457,8 @@
     def iter(self, w_dict):
         return ObjectIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class StringDictStrategy(AbstractTypedStrategy, DictStrategy):
@@ -471,15 +501,21 @@
         assert key is not None
         return self.unerase(w_dict.dstorage).get(key, None)
 
+    def listview_str(self, w_dict):
+        return self.unerase(w_dict.dstorage).keys()
+
     def iter(self, w_dict):
         return StrIteratorImplementation(self.space, self, w_dict)
 
+    def w_keys(self, w_dict):
+        return self.space.newlist_str(self.listview_str(w_dict))
+
 
 class _WrappedIteratorMixin(object):
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -493,7 +529,7 @@
     _mixin_ = True
 
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
         self.iterator = strategy.unerase(dictimplementation.dstorage).iteritems()
 
     def next_entry(self):
@@ -536,6 +572,11 @@
     def iter(self, w_dict):
         return IntIteratorImplementation(self.space, self, w_dict)
 
+    def listview_int(self, w_dict):
+        return self.unerase(w_dict.dstorage).keys()
+
+    # XXX there is no space.newlist_int yet to implement w_keys more efficiently
+
 class IntIteratorImplementation(_WrappedIteratorMixin, IteratorImplementation):
     pass
 
@@ -690,6 +731,9 @@
 
 def dict_keys__DictMulti(space, w_self):
     return W_DictViewKeysObject(space, w_self)
+    # XXX on default this is a fast path when keys are string, do we want to
+    # port it to py3k?
+    # return w_self.w_keys()
 
 def dict_values__DictMulti(space, w_self):
     return W_DictViewValuesObject(space, w_self)
diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -76,7 +76,7 @@
 
     def keys(self, w_dict):
         space = self.space
-        return [space.wrap(key) for key in self.unerase(w_dict.dstorage).dict_w.iterkeys()]
+        return space.newlist_str(self.unerase(w_dict.dstorage).dict_w.keys())
 
     def values(self, w_dict):
         return [unwrap_cell(self.space, w_value) for w_value in self.unerase(w_dict.dstorage).dict_w.itervalues()]
@@ -98,7 +98,8 @@
 
 class DictProxyIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_type = strategy.unerase(dictimplementation.dstorage)
         self.iterator = w_type.dict_w.iteritems()
 
diff --git a/pypy/objspace/std/dicttype.py b/pypy/objspace/std/dicttype.py
--- a/pypy/objspace/std/dicttype.py
+++ b/pypy/objspace/std/dicttype.py
@@ -53,8 +53,14 @@
         w_fill = space.w_None
     if space.is_w(w_type, space.w_dict):
         w_dict = W_DictMultiObject.allocate_and_init_instance(space, w_type)
-        for w_key in space.listview(w_keys):
-            w_dict.setitem(w_key, w_fill)
+
+        strlist = space.listview_str(w_keys)
+        if strlist is not None:
+            for key in strlist:
+                w_dict.setitem_str(key, w_fill)
+        else:
+            for w_key in space.listview(w_keys):
+                w_dict.setitem(w_key, w_fill)
     else:
         w_dict = space.call_function(w_type)
         for w_key in space.listview(w_keys):
diff --git a/pypy/objspace/std/frozensettype.py b/pypy/objspace/std/frozensettype.py
--- a/pypy/objspace/std/frozensettype.py
+++ b/pypy/objspace/std/frozensettype.py
@@ -39,13 +39,11 @@
 def descr__frozenset__new__(space, w_frozensettype,
                             w_iterable=gateway.NoneNotWrapped):
     from pypy.objspace.std.setobject import W_FrozensetObject
-    from pypy.objspace.std.setobject import make_setdata_from_w_iterable
     if (space.is_w(w_frozensettype, space.w_frozenset) and
         w_iterable is not None and type(w_iterable) is W_FrozensetObject):
         return w_iterable
     w_obj = space.allocate_instance(W_FrozensetObject, w_frozensettype)
-    data = make_setdata_from_w_iterable(space, w_iterable)
-    W_FrozensetObject.__init__(w_obj, space, data)
+    W_FrozensetObject.__init__(w_obj, space, w_iterable)
     return w_obj
 
 frozenset_typedef = StdTypeDef("frozenset",
diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py
--- a/pypy/objspace/std/identitydict.py
+++ b/pypy/objspace/std/identitydict.py
@@ -1,5 +1,5 @@
 ## ----------------------------------------------------------------------------
-## dict strategy (see dict_multiobject.py)
+## dict strategy (see dictmultiobject.py)
 
 from pypy.rlib import rerased
 from pypy.rlib.debug import mark_dict_non_null
@@ -80,8 +80,8 @@
     def iter(self, w_dict):
         return IdentityDictIteratorImplementation(self.space, self, w_dict)
 
-    def keys(self, w_dict):
-        return self.unerase(w_dict.dstorage).keys()
+    def w_keys(self, w_dict):
+        return self.space.newlist(self.unerase(w_dict.dstorage).keys())
 
 
 class IdentityDictIteratorImplementation(_UnwrappedIteratorMixin, IteratorImplementation):
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -29,9 +29,8 @@
 class W_SeqIterObject(W_AbstractSeqIterObject):
     """Sequence iterator implementation for general sequences."""
 
-class W_FastListIterObject(W_AbstractSeqIterObject):
-    """Sequence iterator specialized for lists, accessing directly their
-    RPython-level list of wrapped objects.
+class W_FastListIterObject(W_AbstractSeqIterObject): # XXX still needed
+    """Sequence iterator specialized for lists.
     """
 
 class W_FastTupleIterObject(W_AbstractSeqIterObject):
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -139,6 +139,16 @@
         new erased object as storage"""
         self.strategy.init_from_list_w(self, list_w)
 
+    def clear(self, space):
+        """Initializes (or overrides) the listobject as empty."""
+        self.space = space
+        if space.config.objspace.std.withliststrategies:
+            strategy = space.fromcache(EmptyListStrategy)
+        else:
+            strategy = space.fromcache(ObjectListStrategy)
+        self.strategy = strategy
+        strategy.clear(self)
+
     def clone(self):
         """Returns a clone by creating a new listobject
         with the same strategy and a copy of the storage"""
@@ -200,6 +210,11 @@
         """ Return the items in the list as unwrapped strings. If the list does
         not use the list strategy, return None. """
         return self.strategy.getitems_str(self)
+
+    def getitems_int(self):
+        """ Return the items in the list as unwrapped ints. If the list does
+        not use the list strategy, return None. """
+        return self.strategy.getitems_int(self)
     # ___________________________________________________
 
 
@@ -300,6 +315,9 @@
     def getitems_str(self, w_list):
         return None
 
+    def getitems_int(self, w_list):
+        return None
+
     def getstorage_copy(self, w_list):
         raise NotImplementedError
 
@@ -358,6 +376,9 @@
         assert len(list_w) == 0
         w_list.lstorage = self.erase(None)
 
+    def clear(self, w_list):
+        w_list.lstorage = self.erase(None)
+
     erase, unerase = rerased.new_erasing_pair("empty")
     erase = staticmethod(erase)
     unerase = staticmethod(unerase)
@@ -516,6 +537,9 @@
             raise IndexError
         return start + i * step
 
+    def getitems_int(self, w_list):
+        return self._getitems_range(w_list, False)
+
     def getitem(self, w_list, i):
         return self.wrap(self._getitem_unwrapped(w_list, i))
 
@@ -696,6 +720,7 @@
             for i in l:
                 if i == obj:
                     return True
+            return False
         return ListStrategy.contains(self, w_list, w_obj)
 
     def length(self, w_list):
@@ -937,6 +962,9 @@
     def init_from_list_w(self, w_list, list_w):
         w_list.lstorage = self.erase(list_w)
 
+    def clear(self, w_list):
+        w_list.lstorage = self.erase([])
+
     def contains(self, w_list, w_obj):
         return ListStrategy.contains(self, w_list, w_obj)
 
@@ -970,6 +998,9 @@
         if reverse:
             l.reverse()
 
+    def getitems_int(self, w_list):
+        return self.unerase(w_list.lstorage)
+
 class FloatListStrategy(AbstractUnwrappedStrategy, ListStrategy):
     _none_value = 0.0
     _applevel_repr = "float"
@@ -1027,37 +1058,49 @@
     def getitems_str(self, w_list):
         return self.unerase(w_list.lstorage)
 
-
 # _______________________________________________________
 
 init_signature = Signature(['sequence'], None, None)
 init_defaults = [None]
 
 def init__List(space, w_list, __args__):
-    from pypy.objspace.std.tupleobject import W_TupleObject
+    from pypy.objspace.std.tupleobject import W_AbstractTupleObject
     # this is on the silly side
     w_iterable, = __args__.parse_obj(
             None, 'list', init_signature, init_defaults)
-    w_list.__init__(space, [])
+    w_list.clear(space)
     if w_iterable is not None:
-        # unfortunately this is duplicating space.unpackiterable to avoid
-        # assigning a new RPython list to 'wrappeditems', which defeats the
-        # W_FastListIterObject optimization.
-        if isinstance(w_iterable, W_ListObject):
-            w_list.extend(w_iterable)
-        elif isinstance(w_iterable, W_TupleObject):
-            w_list.extend(W_ListObject(space, w_iterable.wrappeditems[:]))
-        else:
-            _init_from_iterable(space, w_list, w_iterable)
+        if type(w_iterable) is W_ListObject:
+            w_iterable.copy_into(w_list)
+            return
+        elif isinstance(w_iterable, W_AbstractTupleObject):
+            w_list.__init__(space, w_iterable.getitems_copy())
+            return
+
+        intlist = space.listview_int(w_iterable)
+        if intlist is not None:
+            w_list.strategy = strategy = space.fromcache(IntegerListStrategy)
+             # need to copy because intlist can share with w_iterable
+            w_list.lstorage = strategy.erase(intlist[:])
+            return
+
+        strlist = space.listview_str(w_iterable)
+        if strlist is not None:
+            w_list.strategy = strategy = space.fromcache(StringListStrategy)
+             # need to copy because intlist can share with w_iterable
+            w_list.lstorage = strategy.erase(strlist[:])
+            return
+
+        # xxx special hack for speed
+        from pypy.interpreter.generator import GeneratorIterator
+        if isinstance(w_iterable, GeneratorIterator):
+            w_iterable.unpack_into_w(w_list)
+            return
+        # /xxx
+        _init_from_iterable(space, w_list, w_iterable)
 
 def _init_from_iterable(space, w_list, w_iterable):
     # in its own function to make the JIT look into init__List
-    # xxx special hack for speed
-    from pypy.interpreter.generator import GeneratorIterator
-    if isinstance(w_iterable, GeneratorIterator):
-        w_iterable.unpack_into_w(w_list)
-        return
-    # /xxx
     w_iterator = space.iter(w_iterable)
     while True:
         try:
diff --git a/pypy/objspace/std/listtype.py b/pypy/objspace/std/listtype.py
--- a/pypy/objspace/std/listtype.py
+++ b/pypy/objspace/std/listtype.py
@@ -43,7 +43,7 @@
 def descr__new__(space, w_listtype, __args__):
     from pypy.objspace.std.listobject import W_ListObject
     w_obj = space.allocate_instance(W_ListObject, w_listtype)
-    W_ListObject.__init__(w_obj, space, [])
+    w_obj.clear(space)
     return w_obj
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -692,6 +692,8 @@
         self.delitem(w_dict, w_key)
         return (w_key, w_value)
 
+    # XXX could implement a more efficient w_keys based on space.newlist_str
+
 def materialize_r_dict(space, obj, dict_w):
     map = obj._get_mapdict_map()
     new_obj = map.materialize_r_dict(space, obj, dict_w)
@@ -699,7 +701,8 @@
 
 class MapDictIteratorImplementation(IteratorImplementation):
     def __init__(self, space, strategy, dictimplementation):
-        IteratorImplementation.__init__(self, space, dictimplementation)
+        IteratorImplementation.__init__(
+            self, space, strategy, dictimplementation)
         w_obj = strategy.unerase(dictimplementation.dstorage)
         self.w_obj = w_obj
         self.orig_map = self.curr_map = w_obj._get_mapdict_map()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -234,10 +234,7 @@
             return W_ComplexObject(x.real, x.imag)
 
         if isinstance(x, set):
-            rdict_w = r_dict(self.eq_w, self.hash_w)
-            for item in x:
-                rdict_w[self.wrap(item)] = None
-            res = W_SetObject(self, rdict_w)
+            res = W_SetObject(self, self.newlist([self.wrap(item) for item in x]))
             return res
 
         if isinstance(x, frozenset):
@@ -323,16 +320,15 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False, classofinstance=None,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                classofinstance=classofinstance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
-        return W_SetObject(self, newset(self))
+        return W_SetObject(self, None)
 
     def newslice(self, w_start, w_end, w_step):
         return W_SliceObject(w_start, w_end, w_step)
@@ -410,7 +406,7 @@
     def unpackiterable(self, w_obj, expected_length=-1):
         if isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
-        elif isinstance(w_obj, W_ListObject):
+        elif type(w_obj) is W_ListObject:
             t = w_obj.getitems_copy()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
@@ -424,7 +420,7 @@
         """
         if isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.tolist()
-        elif isinstance(w_obj, W_ListObject):
+        elif type(w_obj) is W_ListObject:
             if unroll:
                 t = w_obj.getitems_unroll()
             else:
@@ -445,10 +441,12 @@
         return self.fixedview(w_obj, expected_length, unroll=True)
 
     def listview(self, w_obj, expected_length=-1):
-        if isinstance(w_obj, W_ListObject):
+        if type(w_obj) is W_ListObject:
             t = w_obj.getitems()
         elif isinstance(w_obj, W_AbstractTupleObject):
             t = w_obj.getitems_copy()
+        elif isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            t = w_obj.getitems()
         else:
             return ObjSpace.unpackiterable(self, w_obj, expected_length)
         if expected_length != -1 and len(t) != expected_length:
@@ -456,10 +454,40 @@
         return t
 
     def listview_str(self, w_obj):
-        if isinstance(w_obj, W_ListObject):
+        # note: uses exact type checking for objects with strategies,
+        # and isinstance() for others.  See test_listobject.test_uses_custom...
+        if type(w_obj) is W_ListObject:
+            return w_obj.getitems_str()
+        if type(w_obj) is W_DictMultiObject:
+            return w_obj.listview_str()
+        if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
+            return w_obj.listview_str()
+        if isinstance(w_obj, W_StringObject):
+            return w_obj.listview_str()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
             return w_obj.getitems_str()
         return None
 
+    def listview_int(self, w_obj):
+        if type(w_obj) is W_ListObject:
+            return w_obj.getitems_int()
+        if type(w_obj) is W_DictMultiObject:
+            return w_obj.listview_int()
+        if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
+            return w_obj.listview_int()
+        if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
+            return w_obj.getitems_int()
+        return None
+
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
+    def _uses_list_iter(self, w_obj):
+        from pypy.objspace.descroperation import list_iter
+        return self.lookup(w_obj, '__iter__') is list_iter(self)
+
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -7,6 +7,12 @@
 from pypy.interpreter.argument import Signature
 from pypy.objspace.std.settype import set_typedef as settypedef
 from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef
+from pypy.rlib import rerased
+from pypy.rlib.objectmodel import instantiate
+from pypy.interpreter.generator import GeneratorIterator
+from pypy.objspace.std.listobject import W_ListObject
+from pypy.objspace.std.intobject import W_IntObject
+from pypy.objspace.std.stringobject import W_StringObject
 
 class W_BaseSetObject(W_Object):
     typedef = None
@@ -20,78 +26,860 @@
             return True
         return False
 
-
-    def __init__(w_self, space, setdata):
+    def __init__(w_self, space, w_iterable=None):
         """Initialize the set by taking ownership of 'setdata'."""
-        assert setdata is not None
-        w_self.setdata = setdata
+        w_self.space = space
+        set_strategy_and_setdata(space, w_self, w_iterable)
 
     def __repr__(w_self):
         """representation for debugging purposes"""
-        reprlist = [repr(w_item) for w_item in w_self.setdata.keys()]
+        reprlist = [repr(w_item) for w_item in w_self.getkeys()]
         return "<%s(%s)>" % (w_self.__class__.__name__, ', '.join(reprlist))
 
+    def from_storage_and_strategy(w_self, storage, strategy):
+        obj = w_self._newobj(w_self.space, None)
+        assert isinstance(obj, W_BaseSetObject)
+        obj.strategy = strategy
+        obj.sstorage = storage
+        return obj
+
     _lifeline_ = None
     def getweakref(self):
         return self._lifeline_
+
     def setweakref(self, space, weakreflifeline):
         self._lifeline_ = weakreflifeline
     def delweakref(self):
         self._lifeline_ = None
 
+    def switch_to_object_strategy(self, space):
+        d = self.strategy.getdict_w(self)
+        self.strategy = strategy = space.fromcache(ObjectSetStrategy)
+        self.sstorage = strategy.erase(d)
+
+    def switch_to_empty_strategy(self):
+        self.strategy = strategy = self.space.fromcache(EmptySetStrategy)
+        self.sstorage = strategy.get_empty_storage()
+
+    # _____________ strategy methods ________________
+
+
+    def clear(self):
+        """ Removes all elements from the set. """
+        self.strategy.clear(self)
+
+    def copy_real(self):
+        """ Returns a clone of the set. Frozensets storages are also copied."""
+        return self.strategy.copy_real(self)
+
+    def length(self):
+        """ Returns the number of items inside the set. """
+        return self.strategy.length(self)
+
+    def add(self, w_key):
+        """ Adds an element to the set. The element must be wrapped. """
+        self.strategy.add(self, w_key)
+
+    def remove(self, w_item):
+        """ Removes the given element from the set. Element must be wrapped. """
+        return self.strategy.remove(self, w_item)
+
+    def getdict_w(self):
+        """ Returns a dict with all elements of the set. Needed only for switching to ObjectSetStrategy. """
+        return self.strategy.getdict_w(self)
+
+    def listview_str(self):
+        """ If this is a string set return its contents as a list of uwnrapped strings. Otherwise return None. """
+        return self.strategy.listview_str(self)
+
+    def listview_int(self):
+        """ If this is an int set return its contents as a list of uwnrapped ints. Otherwise return None. """
+        return self.strategy.listview_int(self)
+
+    def get_storage_copy(self):
+        """ Returns a copy of the storage. Needed when we want to clone all elements from one set and
+        put them into another. """
+        return self.strategy.get_storage_copy(self)
+
+    def getkeys(self):
+        """ Returns a list of all elements inside the set. Only used in __repr__. Use as less as possible."""
+        return self.strategy.getkeys(self)
+
+    def difference(self, w_other):
+        """ Returns a set with all items that are in this set, but not in w_other. W_other must be a set."""
+        return self.strategy.difference(self, w_other)
+
+    def difference_update(self, w_other):
+        """ As difference but overwrites the sets content with the result. W_other must be a set."""
+        self.strategy.difference_update(self, w_other)
+
+    def symmetric_difference(self, w_other):
+        """ Returns a set with all items that are either in this set or in w_other, but not in both. W_other must be a set. """
+        return self.strategy.symmetric_difference(self, w_other)
+
+    def symmetric_difference_update(self, w_other):
+        """ As symmetric_difference but overwrites the content of the set with the result. W_other must be a set."""
+        self.strategy.symmetric_difference_update(self, w_other)
+
+    def intersect(self, w_other):
+        """ Returns a set with all items that exists in both sets, this set and in w_other. W_other must be a set. """
+        return self.strategy.intersect(self, w_other)
+
+    def intersect_update(self, w_other):
+        """ Keeps only those elements found in both sets, removing all other elements. W_other must be a set."""
+        self.strategy.intersect_update(self, w_other)
+
+    def issubset(self, w_other):
+        """ Checks wether this set is a subset of w_other. W_other must be a set. """
+        return self.strategy.issubset(self, w_other)
+
+    def isdisjoint(self, w_other):
+        """ Checks wether this set and the w_other are completly different, i.e. have no equal elements. W_other must be a set."""
+        return self.strategy.isdisjoint(self, w_other)
+
+    def update(self, w_other):
+        """ Appends all elements from the given set to this set. W_other must be a set."""
+        self.strategy.update(self, w_other)
+
+    def has_key(self, w_key):
+        """ Checks wether this set contains the given wrapped key."""
+        return self.strategy.has_key(self, w_key)
+
+    def equals(self, w_other):
+        """ Checks wether this set and the given set are equal, i.e. contain the same elements. W_other must be a set."""
+        return self.strategy.equals(self, w_other)
+
+    def iter(self):
+        """ Returns an iterator of the elements from this set. """
+        return self.strategy.iter(self)
+
+    def popitem(self):
+        """ Removes an arbitrary element from the set. May raise KeyError if set is empty."""
+        return self.strategy.popitem(self)
+
 class W_SetObject(W_BaseSetObject):
     from pypy.objspace.std.settype import set_typedef as typedef
 
-    def _newobj(w_self, space, rdict_w):
-        """Make a new set by taking ownership of 'rdict_w'."""
-        return W_SetObject(space, rdict_w)
+    def _newobj(w_self, space, w_iterable):
+        """Make a new set by taking ownership of 'w_iterable'."""
+        return W_SetObject(space, w_iterable)
 
 class W_FrozensetObject(W_BaseSetObject):
     from pypy.objspace.std.frozensettype import frozenset_typedef as typedef
     hash = 0
 
-    def _newobj(w_self, space, rdict_w):
-        """Make a new frozenset by taking ownership of 'rdict_w'."""
-        return W_FrozensetObject(space, rdict_w)
+    def _newobj(w_self, space, w_iterable):
+        """Make a new frozenset by taking ownership of 'w_iterable'.""" 
+        return W_FrozensetObject(space, w_iterable)
 
 registerimplementation(W_BaseSetObject)
 registerimplementation(W_SetObject)
 registerimplementation(W_FrozensetObject)
 
-class W_SetIterObject(W_Object):
-    from pypy.objspace.std.settype import setiter_typedef as typedef
+class SetStrategy(object):
+    def __init__(self, space):
+        self.space = space
 
-    def __init__(w_self, setdata):
-        w_self.content = content = setdata
-        w_self.len = len(content)
-        w_self.pos = 0
-        w_self.iterator = w_self.content.iterkeys()
+    def get_empty_dict(self):
+        """ Returns an empty dictionary depending on the strategy. Used to initalize a new storage. """
+        raise NotImplementedError
 
-    def next_entry(w_self):
-        for w_key in w_self.iterator:
+    def get_empty_storage(self):
+        """ Returns an empty storage (erased) object. Used to initialize an empty set."""
+        raise NotImplementedError
+
+    def listview_str(self, w_set):
+        return None
+
+    def listview_int(self, w_set):
+        return None
+
+    #def erase(self, storage):
+    #    raise NotImplementedError
+
+    #def unerase(self, storage):
+    #    raise NotImplementedError
+
+    # __________________ methods called on W_SetObject _________________
+
+    def clear(self, w_set):
+        raise NotImplementedError
+
+    def copy_real(self, w_set):
+        raise NotImplementedError
+
+    def length(self, w_set):
+        raise NotImplementedError
+
+    def add(self, w_set, w_key):
+        raise NotImplementedError
+
+    def remove(self, w_set, w_item):
+        raise NotImplementedError
+
+    def getdict_w(self, w_set):
+        raise NotImplementedError
+
+    def get_storage_copy(self, w_set):
+        raise NotImplementedError
+
+    def getkeys(self, w_set):
+        raise NotImplementedError
+
+    def difference(self, w_set, w_other):
+        raise NotImplementedError
+
+    def difference_update(self, w_set, w_other):
+        raise NotImplementedError
+
+    def symmetric_difference(self, w_set, w_other):
+        raise NotImplementedError
+
+    def symmetric_difference_update(self, w_set, w_other):
+        raise NotImplementedError
+
+    def intersect(self, w_set, w_other):
+        raise NotImplementedError
+
+    def intersect_update(self, w_set, w_other):
+        raise NotImplementedError
+
+    def issubset(self, w_set, w_other):
+        raise NotImplementedError
+
+    def isdisjoint(self, w_set, w_other):
+        raise NotImplementedError
+
+    def update(self, w_set, w_other):
+        raise NotImplementedError
+
+    def has_key(self, w_set, w_key):
+        raise NotImplementedError
+
+    def equals(self, w_set, w_other):
+        raise NotImplementedError
+
+    def iter(self, w_set):
+        raise NotImplementedError
+
+    def popitem(self, w_set):
+        raise NotImplementedError
+
+class EmptySetStrategy(SetStrategy):
+
+    erase, unerase = rerased.new_erasing_pair("empty")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def get_empty_storage(self):
+        return self.erase(None)
+
+    def is_correct_type(self, w_key):
+        return False
+
+    def length(self, w_set):
+        return 0
+
+    def clear(self, w_set):
+        pass
+
+    def copy_real(self, w_set):
+        storage = self.erase(None)
+        clone = w_set.from_storage_and_strategy(storage, self)
+        return clone
+
+    def add(self, w_set, w_key):
+        if type(w_key) is W_IntObject:
+            strategy = self.space.fromcache(IntegerSetStrategy)
+        elif type(w_key) is W_StringObject:
+            strategy = self.space.fromcache(StringSetStrategy)
+        else:
+            strategy = self.space.fromcache(ObjectSetStrategy)
+        w_set.strategy = strategy
+        w_set.sstorage = strategy.get_empty_storage()
+        w_set.add(w_key)
+
+    def remove(self, w_set, w_item):
+        return False
+
+    def getdict_w(self, w_set):
+        return newset(self.space)
+
+    def get_storage_copy(self, w_set):
+        return w_set.sstorage
+
+    def getkeys(self, w_set):
+        return []
+
+    def has_key(self, w_set, w_key):
+        return False
+
+    def equals(self, w_set, w_other):
+        if w_other.strategy is self or w_other.length() == 0:
+            return True
+        return False
+
+    def difference(self, w_set, w_other):
+        return w_set.copy_real()
+
+    def difference_update(self, w_set, w_other):
+        pass
+
+    def intersect(self, w_set, w_other):
+        return w_set.copy_real()
+
+    def intersect_update(self, w_set, w_other):
+        pass
+
+    def isdisjoint(self, w_set, w_other):
+        return True
+
+    def issubset(self, w_set, w_other):
+        return True
+
+    def symmetric_difference(self, w_set, w_other):
+        return w_other.copy_real()
+
+    def symmetric_difference_update(self, w_set, w_other):
+        w_set.strategy = w_other.strategy
+        w_set.sstorage = w_other.get_storage_copy()
+
+    def update(self, w_set, w_other):
+        w_set.strategy = w_other.strategy
+        w_set.sstorage = w_other.get_storage_copy()
+
+    def iter(self, w_set):
+        return EmptyIteratorImplementation(self.space, self, w_set)
+
+    def popitem(self, w_set):
+        raise OperationError(self.space.w_KeyError,
+                                self.space.wrap('pop from an empty set'))
+
+class AbstractUnwrappedSetStrategy(object):
+    _mixin_ = True
+
+    def is_correct_type(self, w_key):
+        """ Checks wether the given wrapped key fits this strategy."""
+        raise NotImplementedError
+
+    def unwrap(self, w_item):
+        """ Returns the unwrapped value of the given wrapped item."""
+        raise NotImplementedError
+
+    def wrap(self, item):
+        """ Returns a wrapped version of the given unwrapped item. """
+        raise NotImplementedError
+
+    def get_storage_from_list(self, list_w):
+        setdata = self.get_empty_dict()
+        for w_item in list_w:
+            setdata[self.unwrap(w_item)] = None
+        return self.erase(setdata)
+
+    def get_storage_from_unwrapped_list(self, items):
+        setdata = self.get_empty_dict()
+        for item in items:
+            setdata[item] = None
+        return self.erase(setdata)
+
+    def length(self, w_set):
+        return len(self.unerase(w_set.sstorage))
+
+    def clear(self, w_set):
+        w_set.switch_to_empty_strategy()
+
+    def copy_real(self, w_set):
+        # may be used internally on frozen sets, although frozenset().copy()
+        # returns self in frozenset_copy__Frozenset.
+        strategy = w_set.strategy
+        d = self.unerase(w_set.sstorage)
+        storage = self.erase(d.copy())
+        clone = w_set.from_storage_and_strategy(storage, strategy)
+        return clone
+
+    def add(self, w_set, w_key):
+        if self.is_correct_type(w_key):
+            d = self.unerase(w_set.sstorage)
+            d[self.unwrap(w_key)] = None
+        else:
+            w_set.switch_to_object_strategy(self.space)
+            w_set.add(w_key)
+
+    def remove(self, w_set, w_item):
+        from pypy.objspace.std.dictmultiobject import _never_equal_to_string
+        d = self.unerase(w_set.sstorage)
+        if not self.is_correct_type(w_item):
+            #XXX check type of w_item and immediately return False in some cases
+            w_set.switch_to_object_strategy(self.space)
+            return w_set.remove(w_item)
+
+        key = self.unwrap(w_item)
+        try:
+            del d[key]
+            return True
+        except KeyError:
+            return False
+
+    def getdict_w(self, w_set):
+        result = newset(self.space)
+        keys = self.unerase(w_set.sstorage).keys()
+        for key in keys:
+            result[self.wrap(key)] = None
+        return result
+
+    def get_storage_copy(self, w_set):
+        d = self.unerase(w_set.sstorage)
+        copy = self.erase(d.copy())
+        return copy
+
+    def getkeys(self, w_set):
+        keys = self.unerase(w_set.sstorage).keys()
+        keys_w = [self.wrap(key) for key in keys]
+        return keys_w
+
+    def has_key(self, w_set, w_key):
+        from pypy.objspace.std.dictmultiobject import _never_equal_to_string
+        if not self.is_correct_type(w_key):
+            #XXX check type of w_item and immediately return False in some cases
+            w_set.switch_to_object_strategy(self.space)
+            return w_set.has_key(w_key)
+        d = self.unerase(w_set.sstorage)
+        return self.unwrap(w_key) in d
+
+    def equals(self, w_set, w_other):
+        if w_set.length() != w_other.length():
+            return False
+        items = self.unerase(w_set.sstorage).keys()
+        for key in items:
+            if not w_other.has_key(self.wrap(key)):
+                return False
+        return True
+
+    def _difference_wrapped(self, w_set, w_other):
+        strategy = self.space.fromcache(ObjectSetStrategy)
+
+        d_new = strategy.get_empty_dict()
+        for obj in self.unerase(w_set.sstorage):
+            w_item = self.wrap(obj)
+            if not w_other.has_key(w_item):
+                d_new[w_item] = None
+
+        return strategy.erase(d_new)
+
+    def _difference_unwrapped(self, w_set, w_other):
+        iterator = self.unerase(w_set.sstorage).iterkeys()
+        other_dict = self.unerase(w_other.sstorage)
+        result_dict = self.get_empty_dict()
+        for key in iterator:
+            if key not in other_dict:
+                result_dict[key] = None
+        return self.erase(result_dict)
+
+    def _difference_base(self, w_set, w_other):
+        if self is w_other.strategy:
+            strategy = w_set.strategy
+            storage = self._difference_unwrapped(w_set, w_other)
+        elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
+            strategy = w_set.strategy
+            storage = w_set.sstorage
+        else:
+            strategy = self.space.fromcache(ObjectSetStrategy)
+            storage = self._difference_wrapped(w_set, w_other)
+        return storage, strategy
+
+    def difference(self, w_set, w_other):
+        storage, strategy = self._difference_base(w_set, w_other)
+        w_newset = w_set.from_storage_and_strategy(storage, strategy)
+        return w_newset
+
+    def difference_update(self, w_set, w_other):
+        storage, strategy = self._difference_base(w_set, w_other)
+        w_set.strategy = strategy
+        w_set.sstorage = storage
+
+    def _symmetric_difference_unwrapped(self, w_set, w_other):
+        d_new = self.get_empty_dict()
+        d_this = self.unerase(w_set.sstorage)
+        d_other = self.unerase(w_other.sstorage)
+        for key in d_other.keys():
+            if not key in d_this:
+                d_new[key] = None
+        for key in d_this.keys():
+            if not key in d_other:
+                d_new[key] = None
+
+        storage = self.erase(d_new)
+        return storage
+
+    def _symmetric_difference_wrapped(self, w_set, w_other):
+        newsetdata = newset(self.space)
+        for obj in self.unerase(w_set.sstorage):
+            w_item = self.wrap(obj)
+            if not w_other.has_key(w_item):
+                newsetdata[w_item] = None
+
+        w_iterator = w_other.iter()
+        while True:
+            w_item = w_iterator.next_entry()
+            if w_item is None:
+                break
+            if not w_set.has_key(w_item):
+                newsetdata[w_item] = None
+
+        strategy = self.space.fromcache(ObjectSetStrategy)
+        return strategy.erase(newsetdata)
+
+    def _symmetric_difference_base(self, w_set, w_other):
+        if self is w_other.strategy:
+            strategy = w_set.strategy
+            storage = self._symmetric_difference_unwrapped(w_set, w_other)
+        else:
+            strategy = self.space.fromcache(ObjectSetStrategy)
+            storage = self._symmetric_difference_wrapped(w_set, w_other)
+        return storage, strategy
+
+    def symmetric_difference(self, w_set, w_other):
+        storage, strategy = self._symmetric_difference_base(w_set, w_other)
+        return w_set.from_storage_and_strategy(storage, strategy)
+
+    def symmetric_difference_update(self, w_set, w_other):
+        storage, strategy = self._symmetric_difference_base(w_set, w_other)
+        w_set.strategy = strategy
+        w_set.sstorage = storage
+
+    def _intersect_base(self, w_set, w_other):
+        if self is w_other.strategy:
+            strategy = w_set.strategy
+            storage = strategy._intersect_unwrapped(w_set, w_other)
+        elif not w_set.strategy.may_contain_equal_elements(w_other.strategy):
+            strategy = self.space.fromcache(EmptySetStrategy)
+            storage = strategy.get_empty_storage()
+        else:
+            strategy = self.space.fromcache(ObjectSetStrategy)
+            storage = self._intersect_wrapped(w_set, w_other)
+        return storage, strategy
+
+    def _intersect_wrapped(self, w_set, w_other):
+        result = newset(self.space)
+        for key in self.unerase(w_set.sstorage):
+            w_key = self.wrap(key)
+            if w_other.has_key(w_key):
+                result[w_key] = None
+
+        strategy = self.space.fromcache(ObjectSetStrategy)
+        return strategy.erase(result)
+
+    def _intersect_unwrapped(self, w_set, w_other):
+        result = self.get_empty_dict()
+        d_this = self.unerase(w_set.sstorage)
+        d_other = self.unerase(w_other.sstorage)
+        for key in d_this:
+            if key in d_other:
+                result[key] = None
+        return self.erase(result)
+
+    def intersect(self, w_set, w_other):
+        if w_set.length() > w_other.length():
+            return w_other.intersect(w_set)
+
+        storage, strategy = self._intersect_base(w_set, w_other)
+        return w_set.from_storage_and_strategy(storage, strategy)
+
+    def intersect_update(self, w_set, w_other):
+        if w_set.length() > w_other.length():
+            w_intersection = w_other.intersect(w_set)
+            strategy = w_intersection.strategy
+            storage = w_intersection.sstorage
+        else:
+            storage, strategy = self._intersect_base(w_set, w_other)
+        w_set.strategy = strategy
+        w_set.sstorage = storage
+
+    def _issubset_unwrapped(self, w_set, w_other):
+        d_other = self.unerase(w_other.sstorage)
+        for item in self.unerase(w_set.sstorage):
+            if not item in d_other:
+                return False

From noreply at buildbot.pypy.org  Mon Apr 16 14:01:23 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 14:01:23 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix some of the results of the merge.
 StringListStrategy is called
Message-ID: <20120416120123.B1BF8820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54410:6c3be7bcfb8b
Date: 2012-04-16 13:57 +0200
http://bitbucket.org/pypy/pypy/changeset/6c3be7bcfb8b/

Log:	fix some of the results of the merge. StringListStrategy is called
	UnicodeListStrategy on py3k, and withrangelist has been removed (but
	we still keep the tests, just because)

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1086,7 +1086,7 @@
 
         strlist = space.listview_str(w_iterable)
         if strlist is not None:
-            w_list.strategy = strategy = space.fromcache(StringListStrategy)
+            w_list.strategy = strategy = space.fromcache(UnicodeListStrategy)
              # need to copy because intlist can share with w_iterable
             w_list.lstorage = strategy.erase(strlist[:])
             return
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1145,7 +1145,7 @@
                 (set, []),  (set,  [5]), (set,  ['x']), (set, [X]),
                 (dict, []), (dict, [(5,6)]), (dict, [('x',7)]), (dict, [(X,8)]),
                 ]:
-            print base, arg
+            print(base, arg)
             class SubClass(base):
                 def __iter__(self):
                     return iter("foobar")
@@ -1159,9 +1159,9 @@
 
 class AppTestForRangeLists(AppTestW_ListObject):
 
-    def setup_class(cls):
-        cls.space = gettestobjspace(**{"objspace.std.withrangelist" :
-                                       True})
+    ## def setup_class(cls):
+    ##     cls.space = gettestobjspace(**{"objspace.std.withrangelist" :
+    ##                                    True})
 
     def test_range_simple_backwards(self):
         x = range(5,1)

From noreply at buildbot.pypy.org  Mon Apr 16 15:11:43 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:11:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Linux: linking openssl into portable
 binaries is **A SERIOUS MESS** and I'm
Message-ID: <20120416131143.4A28A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54411:b777f530457f
Date: 2012-04-16 15:09 +0200
http://bitbucket.org/pypy/pypy/changeset/b777f530457f/

Log:	Linux: linking openssl into portable binaries is **A SERIOUS MESS**
	and I'm carefully choosing my words. Give up and use static
	linking.

diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,9 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +18,17 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +40,7 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
     includes = includes,
     export_symbols = [],
     post_include_bits = [

From noreply at buildbot.pypy.org  Mon Apr 16 15:13:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:13:41 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: Kill
Message-ID: <20120416131341.23D60820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r354:d979f90d5df8
Date: 2012-04-16 15:13 +0200
http://bitbucket.org/pypy/pypy.org/changeset/d979f90d5df8/

Log:	Kill

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -42,8 +42,8 @@
 (This is the official release 1.8;
 for the most up-to-date version see below.)
 
-* `Linux binary (32bit)`__ (`openssl0.9.8 notes`_)
-* `Linux binary (64bit)`__ (`openssl0.9.8 notes`_)
+* `Linux binary (32bit)`__
+* `Linux binary (64bit)`__
 * `Mac OS/X binary (64bit)`__
 * `Windows binary (32bit)`__ (you need the `VS 2010 runtime libraries`_)
   *note: the zip file contains the wrong version, msvcrt90.dll :-(*
@@ -105,21 +105,6 @@
 a symlink to it, otherwise it will not find its libraries.
 
 
-.. _`openssl0.9.8 notes`:
-
-Linux libraries
----------------
-
-Linux binaries require a system with the correct set of libraries.
-Right now one problem is OpenSSL, which some distributions distribute in
-version 0.9.8 and others in version 1.0.0.  The binaries require 0.9.8.
-If you can't install it in your distribution, pick it the latest
-`openssl-0.9.8*.tar.gz from sources`__ and install it with ``./config
-shared; make; make install``.
-
-.. __: http://www.openssl.org/source/
-
-
 .. _translate:
 
 Building from source

From noreply at buildbot.pypy.org  Mon Apr 16 15:30:41 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 15:30:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k fixes
Message-ID: <20120416133041.6BFE2820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54412:34b63669e7da
Date: 2012-04-16 15:19 +0200
http://bitbucket.org/pypy/pypy/changeset/34b63669e7da/

Log:	py3k fixes

diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -124,6 +124,7 @@
         intstr.get_storage_from_list = tmp_func
 
     def test_listview_str_int_on_set(self):
+        py.test.skip("listview_str not supported for py3k strings (i.e., unicode)")
         w = self.space.wrap
 
         w_a = W_SetObject(self.space)
@@ -259,7 +260,7 @@
         raises(KeyError, "b.pop()")
 
         a = set([1,2,3,4,5])
-        for i in xrange(5):
+        for i in range(5):
             a.pop()
         assert a == set()
         raises(KeyError, "a.pop()")
@@ -911,7 +912,7 @@
         it = iter(s)
         s.add(7)
         # 's' is now length 4
-        raises(RuntimeError, it.next)
+        raises(RuntimeError, it.__next__)
 
     def test_iter_set_strategy_only_change_1(self):
         s = set([1, 3, 5])
@@ -919,6 +920,8 @@
         class Foo(object):
             def __eq__(self, other):
                 return False
+            def __hash__(self):
+                return 0
         assert Foo() not in s      # this changes the strategy of 'd'
         lst = list(s)  # but iterating still works
         assert sorted(lst) == [1, 3, 5]

From noreply at buildbot.pypy.org  Mon Apr 16 15:30:42 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 15:30:42 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: bah, kill leftover from the merge
Message-ID: <20120416133042.A6904820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54413:801e0d2f2fce
Date: 2012-04-16 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/801e0d2f2fce/

Log:	bah, kill leftover from the merge

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -658,4 +658,3 @@
 class AppTestSysExcInfoDirectCallMethod(AppTestSysExcInfoDirect):
     def setup_class(cls):
         cls.space = gettestobjspace(**{"objspace.opcodes.CALL_METHOD": True})
->>>>>>> other

From noreply at buildbot.pypy.org  Mon Apr 16 15:51:26 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:51:26 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Transaction fixes.
Message-ID: <20120416135126.73887820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54414:0dbdf68abd87
Date: 2012-04-15 15:59 +0200
http://bitbucket.org/pypy/pypy/changeset/0dbdf68abd87/

Log:	Transaction fixes.

diff --git a/pypy/rlib/debug.py b/pypy/rlib/debug.py
--- a/pypy/rlib/debug.py
+++ b/pypy/rlib/debug.py
@@ -6,6 +6,7 @@
 
 def ll_assert(x, msg):
     """After translation to C, this becomes an RPyAssert."""
+    assert type(x) is bool, "bad type! got %r" % (type(x),)
     assert x, msg
 
 class Entry(ExtRegistryEntry):
diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -123,6 +123,13 @@
         self.main_thread_tls = StmGCTLS(self, in_main_thread=True)
         self.main_thread_tls.start_transaction()
 
+    def setup_thread(self):
+        from pypy.rpython.memory.gc.stmtls import StmGCTLS
+        StmGCTLS(self, in_main_thread=False)
+
+    def teardown_thread(self):
+        self.get_tls().teardown_thread()
+
     @always_inline
     def get_tls(self):
         from pypy.rpython.memory.gc.stmtls import StmGCTLS
@@ -228,6 +235,7 @@
         self.get_tls().start_transaction()
 
     def commit_transaction(self):
+        raise NotImplementedError
         self.collector.commit_transaction()
 
 
@@ -325,6 +333,7 @@
                     return localobj
             #
             # Here, we need to really make a local copy
+            raise NotImplementedError
             size = self.get_size(obj)
             tls = self.collector.get_tls()
             try:
@@ -390,6 +399,7 @@
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
+        raise NotImplementedError
         obj = llmemory.cast_ptr_to_adr(gcobj)
         hdr = self.header(obj)
         #
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -85,7 +85,7 @@
     def cast_address_to_tls_object(tlsaddr):
         if we_are_translated():
             tls = llmemory.cast_adr_to_ptr(tlsaddr, base_ptr_lltype())
-            return cast_base_ptr_to_instance(tls)
+            return cast_base_ptr_to_instance(StmGCTLS, tls)
         else:
             n = rffi.cast(lltype.Signed, tlsaddr)
             return StmGCTLS.nontranslated_dict[n]
@@ -135,7 +135,8 @@
         self.nursery_top  = self.nursery_start + self.nursery_size
 
     def local_nursery_is_empty(self):
-        ll_assert(self.nursery_free, "local_nursery_is_empty: gc not running")
+        ll_assert(bool(self.nursery_free),
+                  "local_nursery_is_empty: gc not running")
         return self.nursery_free == self.nursery_start
 
     # ------------------------------------------------------------
@@ -202,7 +203,7 @@
 
         No more mallocs are allowed after this is called.
         """
-        xxx
+        raise NotImplementedError
 
     # ------------------------------------------------------------
 
@@ -242,7 +243,7 @@
         obj = self.old_objects
         self.old_objects = NULL
         while obj:
-            hdr = self.header(obj)
+            hdr = self.gc.header(obj)
             hdr.tid |= GCFLAG_GLOBAL
             obj = hdr.version
         #
@@ -266,11 +267,8 @@
 
 
     def collect_roots_from_stack(self):
-        self.gc.root_walker.walk_roots(
-            StmGCTLS._trace_drag_out1,  # stack roots of the current thread
-            None,                       # static in prebuilt non-gc
-            None,                       # static in prebuilt gc
-            self)
+        self.gc.root_walker.walk_current_stack_roots(
+            StmGCTLS._trace_drag_out1, self)
 
     def _trace_drag_out1(self, root):
         self._trace_drag_out(root, None)
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -1374,19 +1374,16 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc,
-                   arg=None):
+                   collect_static_in_prebuilt_gc):
         gcdata = self.gcdata
         gc = self.gc
-        if arg is None:
-            arg = gc
         if collect_static_in_prebuilt_nongc:
             addr = gcdata.static_root_start
             end = gcdata.static_root_nongcend
             while addr != end:
                 result = addr.address[0]
                 if gc.points_to_valid_gc_object(result):
-                    collect_static_in_prebuilt_nongc(arg, result)
+                    collect_static_in_prebuilt_nongc(gc, result)
                 addr += sizeofaddr
         if collect_static_in_prebuilt_gc:
             addr = gcdata.static_root_nongcend
@@ -1394,10 +1391,10 @@
             while addr != end:
                 result = addr.address[0]
                 if gc.points_to_valid_gc_object(result):
-                    collect_static_in_prebuilt_gc(arg, result)
+                    collect_static_in_prebuilt_gc(gc, result)
                 addr += sizeofaddr
         if collect_stack_root:
-            self.walk_stack_roots(collect_stack_root, arg)     # abstract
+            self.walk_stack_roots(collect_stack_root)     # abstract
 
     def need_stacklet_support(self):
         raise Exception("%s does not support stacklets" % (
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -1,9 +1,11 @@
 from pypy.rpython.memory.gctransform.framework import FrameworkGCTransformer
 from pypy.rpython.memory.gctransform.framework import BaseRootWalker
 from pypy.rpython.memory.gctransform.framework import sizeofaddr
-from pypy.rpython.lltypesystem import llmemory
+from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.annotation import model as annmodel
 from pypy.rlib.debug import fatalerror_notb
+from pypy.rlib.nonconst import NonConstant
+from pypy.rlib.objectmodel import specialize
 
 
 class StmFrameworkGCTransformer(FrameworkGCTransformer):
@@ -33,11 +35,11 @@
             self.gcdata.gc.stm_normalize_global,
             [annmodel.SomeAddress()], annmodel.SomeAddress())
         self.stm_enter_transactional_mode_ptr = getfn(
-            self.gcdata.gc.enter_transactional_mode.im_func
-            [s_sc], annmodel.s_None)
+            self.gcdata.gc.enter_transactional_mode.im_func,
+            [s_gc], annmodel.s_None)
         self.stm_leave_transactional_mode_ptr = getfn(
-            self.gcdata.gc.leave_transactional_mode.im_func
-            [s_sc], annmodel.s_None)
+            self.gcdata.gc.leave_transactional_mode.im_func,
+            [s_gc], annmodel.s_None)
         self.stm_start_ptr = getfn(
             self.gcdata.gc.start_transaction.im_func,
             [s_gc], annmodel.s_None)
@@ -110,7 +112,8 @@
         self.decr_stack = decr_stack
 
         root_iterator = shadowstack.get_root_iterator(gctransformer)
-        def walk_stack_root(callback, start, end):
+        @specialize.argtype(1)
+        def walk_stack_root(callback, arg, start, end):
             root_iterator.setcontext(NonConstant(llmemory.NULL))
             gc = self.gc
             addr = end
@@ -118,7 +121,7 @@
                 addr = root_iterator.nextleft(gc, start, addr)
                 if addr == llmemory.NULL:
                     return
-                callback(gc, addr)
+                callback(arg, addr)
         self.rootstackhook = walk_stack_root
 
         rsd = gctransformer.root_stack_depth
@@ -143,10 +146,18 @@
         llmemory.raw_free(base)
 
     def walk_stack_roots(self, collect_stack_root):
+        raise NotImplementedError
         # XXX only to walk the main thread's shadow stack, so far
         stackgcdata = self.stackgcdata
         if self.gcdata.main_thread_stack_base != stackgcdata.root_stack_base:
             fatalerror_notb("XXX not implemented: walk_stack_roots in thread")
-        self.rootstackhook(collect_stack_root,
+        self.rootstackhook(collect_stack_root, self.gcdata.gc,
                            stackgcdata.root_stack_base,
                            stackgcdata.root_stack_top)
+
+    @specialize.argtype(2)
+    def walk_current_stack_roots(self, collect_stack_root, arg):
+        stackgcdata = self.stackgcdata
+        self.rootstackhook(collect_stack_root, arg,
+                           stackgcdata.root_stack_base,
+                           stackgcdata.root_stack_top)
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -433,7 +433,11 @@
             # local data structure.  We assume that objects are stored
             # only temporarily there, so it is always cleared at the point
             # where we collect the roots.
-            assert TYPE._name == 'ExcData'
+            if TYPE._name == 'ExcData':
+                return
+            # Some other thread-local data don't have any GC pointers
+            # themselves.  These are fine.
+            assert list(gc_pointers_inside(value, adr)) == []
             return
         else:
             appendto = self.addresses_of_static_ptrs_in_nongc

From noreply at buildbot.pypy.org  Mon Apr 16 15:51:27 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:51:27 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix.
Message-ID: <20120416135127.CF6E8820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54415:3252ee877473
Date: 2012-04-15 16:08 +0200
http://bitbucket.org/pypy/pypy/changeset/3252ee877473/

Log:	Fix.

diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -25,22 +25,19 @@
     pass
 
 class FakeRootWalker:
-    def walk_roots(self, f1, f2, f3, arg):
-        if f1 is not None:
-            A = lltype.Array(llmemory.Address)
-            roots = lltype.malloc(A, len(self.current_stack), flavor='raw')
-            for i in range(len(self.current_stack)):
-                roots[i] = llmemory.cast_ptr_to_adr(self.current_stack[i])
-            for i in range(len(self.current_stack)):
-                root = lltype.direct_ptradd(lltype.direct_arrayitems(roots), i)
-                root = llmemory.cast_ptr_to_adr(root)
-                f1(arg, root)
-            for i in range(len(self.current_stack)):
-                P = lltype.typeOf(self.current_stack[i])
-                self.current_stack[i] = llmemory.cast_adr_to_ptr(roots[i], P)
-            lltype.free(roots, flavor='raw')
-        assert f2 is None
-        assert f3 is None
+    def walk_current_stack_roots(self, callback, arg):
+        A = lltype.Array(llmemory.Address)
+        roots = lltype.malloc(A, len(self.current_stack), flavor='raw')
+        for i in range(len(self.current_stack)):
+            roots[i] = llmemory.cast_ptr_to_adr(self.current_stack[i])
+        for i in range(len(self.current_stack)):
+            root = lltype.direct_ptradd(lltype.direct_arrayitems(roots), i)
+            root = llmemory.cast_ptr_to_adr(root)
+            callback(arg, root)
+        for i in range(len(self.current_stack)):
+            P = lltype.typeOf(self.current_stack[i])
+            self.current_stack[i] = llmemory.cast_adr_to_ptr(roots[i], P)
+        lltype.free(roots, flavor='raw')
 
 class FakeGC:
     from pypy.rpython.memory.support import AddressDict, null_address_dict

From noreply at buildbot.pypy.org  Mon Apr 16 15:51:29 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:51:29 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Merge the usages of GCFLAG_MOVED and
	GCFLAG_VISITED.
Message-ID: <20120416135129.331E4820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54416:6a70fa336a29
Date: 2012-04-15 16:46 +0200
http://bitbucket.org/pypy/pypy/changeset/6a70fa336a29/

Log:	Merge the usages of GCFLAG_MOVED and GCFLAG_VISITED.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -44,7 +44,6 @@
 GCFLAG_FIXED_HASH = first_gcflag << 3
 GCFLAG_WEAKREF    = first_gcflag << 4
 GCFLAG_VISITED    = first_gcflag << 5
-GCFLAG_MOVED      = first_gcflag << 6
 
 
 def always_inline(fn):
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -8,7 +8,6 @@
 from pypy.rpython.memory.gc.stmgc import WORD, NULL
 from pypy.rpython.memory.gc.stmgc import always_inline, dont_inline
 from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED
-from pypy.rpython.memory.gc.stmgc import GCFLAG_MOVED
 
 
 class StmGCTLS(object):
@@ -290,7 +289,7 @@
             return
         #
         # If 'obj' was already forwarded, change it to its forwarding address.
-        if hdr.tid & GCFLAG_MOVED:
+        if hdr.tid & GCFLAG_VISITED:
             root.address[0] = hdr.version
             return
         #
@@ -308,9 +307,9 @@
                              newobj - size_gc_header,
                              totalsize)
         #
-        # Set the YOUNG copy's GCFLAG_MOVED and set its version to
+        # Set the YOUNG copy's GCFLAG_VISITED and set its version to
         # point to the OLD copy.
-        hdr.tid |= GCFLAG_MOVED
+        hdr.tid |= GCFLAG_VISITED
         hdr.version = newobj
         #
         # Change the original pointer to this object.

From noreply at buildbot.pypy.org  Mon Apr 16 15:51:30 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:51:30 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fixes.
Message-ID: <20120416135130.84611820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54417:e55dfbece637
Date: 2012-04-15 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/e55dfbece637/

Log:	Fixes.

diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -46,7 +46,7 @@
 
     def free_object(self, adr2):
         adr1 = adr2 - self.gc.gcheaderbuilder.size_gc_header
-        llarena.arena_free(adr1)
+        llarena.arena_free(llarena.getfakearenaaddress(adr1))
 
     def replace_special_stack(self, new_special_stack):
         self.special_stack.delete()
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -307,6 +307,10 @@
                              newobj - size_gc_header,
                              totalsize)
         #
+        # Register the object here, not before the memcopy() that would
+        # overwrite its 'version' field
+        self._register_newly_malloced_obj(newobj)
+        #
         # Set the YOUNG copy's GCFLAG_VISITED and set its version to
         # point to the OLD copy.
         hdr.tid |= GCFLAG_VISITED
@@ -322,9 +326,10 @@
         self.pending.append(newobj)
 
     def _malloc_out_of_nursery(self, totalsize):
-        obj = self.sharedarea_tls.malloc_object(totalsize)
+        return self.sharedarea_tls.malloc_object(totalsize)
+
+    def _register_newly_malloced_obj(self, obj):
         self.sharedarea_tls.add_regular(obj)
-        return obj
 
     def collect_roots_from_tldict(self):
         pass  # XXX
@@ -335,6 +340,7 @@
         while self.pending.non_empty():
             obj = self.pending.pop()
             self.gc.trace(obj, self._trace_drag_out, None)
+        self.pending.delete()
 
     def mass_free_old_local(self, previous_sharedarea_tls):
         obj = previous_sharedarea_tls.chained_list
@@ -344,6 +350,7 @@
             next = hdr.version
             if hdr.tid & GCFLAG_VISITED:
                 # survives: relink in the new sharedarea_tls
+                hdr.tid -= GCFLAG_VISITED
                 self.sharedarea_tls.add_regular(obj)
             else:
                 # dies
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -152,7 +152,8 @@
             srlist = self.stack_pop()
             sr1.sr2 = srlist
             srlist = sr1
-        for i in range(100)[::-1]:
-            assert srlist
-            srlist = srlist.sr2
-        assert not srlist
+            #
+            for j in range(i, -1, -1):
+                assert sr1
+                sr1 = sr1.sr2
+            assert not sr1

From noreply at buildbot.pypy.org  Mon Apr 16 15:51:31 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:51:31 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Tweaks. Try to restore the write barrier.
Message-ID: <20120416135131.BDF0F820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54418:6982e52364be
Date: 2012-04-15 19:53 +0200
http://bitbucket.org/pypy/pypy/changeset/6982e52364be/

Log:	Tweaks. Try to restore the write barrier.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -97,6 +97,7 @@
         self.stm_operations = stm_operations
         self.nursery_size = nursery_size
         self.sharedarea = stmshared.StmGCSharedArea(self)
+        self.transactional_mode = False
         #
         def _get_size(obj):     # indirection to hide 'self'
             return self.get_size(obj)
@@ -136,9 +137,13 @@
         return StmGCTLS.cast_address_to_tls_object(tls)
 
     def enter_transactional_mode(self):
+        ll_assert(not self.transactional_mode, "already in transactional mode")
         self.main_thread_tls.enter_transactional_mode()
+        self.transactional_mode = True
 
     def leave_transactional_mode(self):
+        ll_assert(self.transactional_mode, "already in non-transactional mode")
+        self.transactional_mode = False
         self.main_thread_tls.leave_transactional_mode()
 
     # ----------
@@ -226,9 +231,10 @@
         tls.global_stop = result + totalsize
 
 
-    def collect(self, gen=0):
-        #raise NotImplementedError
-        debug_print("XXX collect() ignored")
+    def collect(self, gen=1):
+        self.get_tls().local_collection()
+        if gen > 0:
+            debug_print("XXX not doing a global collect()")
 
     def start_transaction(self):
         self.get_tls().start_transaction()
@@ -311,11 +317,10 @@
         #
         @dont_inline
         def _stm_write_barrier_global(obj):
-            if not stm_operations.in_transaction():
-                return obj
             # we need to find or make a local copy
             hdr = self.header(obj)
             if hdr.tid & GCFLAG_WAS_COPIED == 0:
+                #
                 # in this case, we are sure that we don't have a copy
                 hdr.tid |= GCFLAG_WAS_COPIED
                 # ^^^ non-protected write, but concurrent writes should
@@ -332,11 +337,11 @@
                     return localobj
             #
             # Here, we need to really make a local copy
-            raise NotImplementedError
             size = self.get_size(obj)
-            tls = self.collector.get_tls()
+            totalsize = self.gcheaderbuilder.size_gc_header + size
+            tls = self.get_tls()
             try:
-                localobj = self._malloc_local_raw(tls, size)
+                localobj = tls.malloc_local_copy(totalsize)
             except MemoryError:
                 # XXX
                 fatalerror("MemoryError in _stm_write_barrier_global -- sorry")
diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -48,10 +48,6 @@
         adr1 = adr2 - self.gc.gcheaderbuilder.size_gc_header
         llarena.arena_free(llarena.getfakearenaaddress(adr1))
 
-    def replace_special_stack(self, new_special_stack):
-        self.special_stack.delete()
-        self.special_stack = new_special_stack
-
     def clear(self):
         obj = self.chained_list
         self.chained_list = NULL
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -39,13 +39,14 @@
         # --- the local raw-malloced objects (chained list via hdr.version)
         #self.rawmalloced_objects = NULL
         # --- the local "normal" old objects (chained list via hdr.version)
-        self.old_objects = NULL
+        #self.old_objects = NULL
         # --- the local objects with weakrefs (chained list via hdr.version)
         #self.young_objects_with_weakrefs = NULL
         #self.old_objects_with_weakrefs = NULL
         #
         # --- a thread-local allocator for the shared area
-        self.sharedarea_tls = None
+        from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
+        self.sharedarea_tls = StmGCThreadLocalAllocator(self.gc.sharedarea)
         #
         self._register_with_C_code()
 
@@ -155,7 +156,7 @@
         #
         # Move away the previous sharedarea_tls and start a new one.
         from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
-        previous_sharedarea_tls = self.sharedarea_tls    # may be None
+        previous_sharedarea_tls = self.sharedarea_tls
         self.sharedarea_tls = StmGCThreadLocalAllocator(self.gc.sharedarea)
         #
         # List of LOCAL objects pending a visit.  Note that no GLOBAL
@@ -182,8 +183,7 @@
         #
         # Visit all previous OLD objects.  Free the ones that have not been
         # visited above, and reset GCFLAG_VISITED on the others.
-        if previous_sharedarea_tls is not None:
-            self.mass_free_old_local(previous_sharedarea_tls)
+        self.mass_free_old_local(previous_sharedarea_tls)
         #
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
@@ -234,24 +234,30 @@
                   "odd-valued (i.e. tagged) pointer unexpected here")
         return self.nursery_start <= addr < self.nursery_top
 
+    def malloc_local_copy(self, totalsize):
+        """Allocate an object that will be used as a LOCAL copy of
+        some GLOBAL object."""
+        localobj = self.sharedarea_tls.malloc_object(totalsize)
+        self.sharedarea_tls.add_special(localobj)
+        return localobj
+
     # ------------------------------------------------------------
 
     def _promote_locals_to_globals(self):
         ll_assert(self.local_nursery_is_empty(), "nursery must be empty [1]")
+        ll_assert(not self.sharedarea_tls.special_stack.non_empty(),
+                  "special_stack should be empty here [1]")
         #
-        obj = self.old_objects
-        self.old_objects = NULL
+        # Promote all objects in sharedarea_tls to global
+        obj = self.sharedarea_tls.chained_list
+        self.sharedarea_tls.chained_list = NULL
+        #
         while obj:
             hdr = self.gc.header(obj)
+            obj = hdr.version
+            ll_assert(not (hdr.tid & GCFLAG_GLOBAL), "already GLOBAL [1]")
+            hdr.version = NULL
             hdr.tid |= GCFLAG_GLOBAL
-            obj = hdr.version
-        #
-##        obj = self.rawmalloced_objects
-##        self.rawmalloced_objects = NULL
-##        while obj:
-##            hdr = self.header(obj)
-##            hdr.tid |= GCFLAG_GLOBAL
-##            obj = hdr.version
 
     def _cleanup_state(self):
         #if self.rawmalloced_objects:
@@ -259,10 +265,7 @@
 
         # if we still have a StmGCThreadLocalAllocator, free the old unused
         # local objects it still contains
-        if self.sharedarea_tls is not None:
-            self.sharedarea_tls.clear()
-            self.sharedarea_tls.delete()
-            self.sharedarea_tls = None
+        self.sharedarea_tls.clear()
 
 
     def collect_roots_from_stack(self):

From noreply at buildbot.pypy.org  Mon Apr 16 15:51:33 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:51:33 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Some progress, fixing test_rstm.
Message-ID: <20120416135133.09232820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54419:53a337586265
Date: 2012-04-16 15:43 +0200
http://bitbucket.org/pypy/pypy/changeset/53a337586265/

Log:	Some progress, fixing test_rstm.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -220,8 +220,8 @@
     """The main function running one of the threads."""
     # Note that we cannot allocate any object here outside a transaction,
     # so we need to be very careful.
+    rstm.descriptor_init()
     state.lock()
-    rstm.descriptor_init()
     #
     rstm.perform_transaction(_setup_thread, AbstractPending, None)
     my_transactions_pending = state.getvalue()._transaction_pending
@@ -260,10 +260,10 @@
             state.lock()
             _add_list(my_transactions_pending)
     #
-    rstm.descriptor_done()
     if state.num_waiting_threads == 0:    # only the last thread to leave
         state.unlock_unfinished()
     state.unlock()
+    rstm.descriptor_done()
 
 
 @rgc.no_collect
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -1,12 +1,14 @@
 import os, thread, time
-from pypy.rlib.debug import debug_print, ll_assert
+from pypy.rlib.debug import debug_print, ll_assert, fatalerror
 from pypy.rlib import rstm
+from pypy.rpython.annlowlevel import llhelper
 from pypy.translator.stm.test.support import CompiledSTMTests
+from pypy.module.thread import ll_thread
 
 
 class Arg(object):
     pass
-arg = Arg()
+arg_list = [Arg() for i in range(10)]
 
 def setx(arg, retry_counter):
     debug_print(arg.x)
@@ -17,22 +19,29 @@
         assert rstm._debug_get_state() == 2
     arg.x = 42
 
-def stm_perform_transaction(initial_x=202):
-    arg.x = initial_x
+def stm_perform_transaction(done=None, i=0):
     ll_assert(rstm._debug_get_state() == -2, "bad debug_get_state (1)")
     rstm.descriptor_init()
+    arg = arg_list[i]
+    if done is None:
+        arg.x = 202
+    else:
+        arg.x = done.initial_x
     ll_assert(rstm._debug_get_state() == 0, "bad debug_get_state (2)")
     rstm.perform_transaction(setx, Arg, arg)
     ll_assert(rstm._debug_get_state() == 0, "bad debug_get_state (3)")
+    ll_assert(arg.x == 42, "bad arg.x")
+    if done is not None:
+        ll_thread.release_NOAUTO(done.finished_lock)
     rstm.descriptor_done()
     ll_assert(rstm._debug_get_state() == -2, "bad debug_get_state (4)")
-    ll_assert(arg.x == 42, "bad arg.x")
 
 def test_stm_multiple_threads():
     ok = []
     def f(i):
-        stm_perform_transaction()
+        stm_perform_transaction(i=i)
         ok.append(i)
+    rstm.enter_transactional_mode()
     for i in range(10):
         thread.start_new_thread(f, (i,))
     timeout = 10
@@ -40,6 +49,7 @@
         time.sleep(0.1)
         timeout -= 0.1
         assert timeout >= 0.0, "timeout!"
+    rstm.leave_transactional_mode()
     assert sorted(ok) == range(10)
 
 
@@ -58,22 +68,23 @@
         assert '102' in dataerr.splitlines()
 
     def build_perform_transaction(self):
-        from pypy.module.thread import ll_thread
         class Done: done = False
         done = Done()
         def g():
-            stm_perform_transaction(done.initial_x)
-            done.done = True
+            stm_perform_transaction(done)
         def f(argv):
             done.initial_x = int(argv[1])
             assert rstm._debug_get_state() == -1    # main thread
-            ll_thread.start_new_thread(g, ())
-            for i in range(20):
-                if done.done: break
-                time.sleep(0.1)
-            else:
-                print "timeout!"
-                raise Exception
+            done.finished_lock = ll_thread.allocate_ll_lock()
+            ll_thread.acquire_NOAUTO(done.finished_lock, True)
+            #
+            rstm.enter_transactional_mode()
+            #
+            llcallback = llhelper(ll_thread.CALLBACK, g)
+            ident = ll_thread.c_thread_start_NOGIL(llcallback)
+            ll_thread.acquire_NOAUTO(done.finished_lock, True)
+            #
+            rstm.leave_transactional_mode()
             return 0
         t, cbuilder = self.compile(f)
         return cbuilder
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -403,8 +403,8 @@
     'stm_become_inevitable':  LLOp(),
     'stm_descriptor_init':    LLOp(canrun=True),
     'stm_descriptor_done':    LLOp(canrun=True),
-    'stm_enter_transactional_mode': LLOp(),
-    'stm_leave_transactional_mode': LLOp(),
+    'stm_enter_transactional_mode': LLOp(canrun=True),
+    'stm_leave_transactional_mode': LLOp(canrun=True),
     'stm_writebarrier':       LLOp(sideeffects=False),
     'stm_normalize_global':   LLOp(),
     'stm_start_transaction':  LLOp(canrun=True),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -630,6 +630,12 @@
 def op_stm_commit_transaction():
     pass
 
+def op_stm_enter_transactional_mode():
+    pass
+
+def op_stm_leave_transactional_mode():
+    pass
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -240,8 +240,7 @@
         self.get_tls().start_transaction()
 
     def commit_transaction(self):
-        raise NotImplementedError
-        self.collector.commit_transaction()
+        self.get_tls().stop_transaction()
 
 
     @always_inline
@@ -317,6 +316,11 @@
         #
         @dont_inline
         def _stm_write_barrier_global(obj):
+            tls = self.get_tls()
+            if not tls.in_transaction():
+                return obj # not in transaction: only when running the code
+                           # in _run_thread(), i.e. in sub-threads outside
+                           # transactions.  xxx statically detect this case?
             # we need to find or make a local copy
             hdr = self.header(obj)
             if hdr.tid & GCFLAG_WAS_COPIED == 0:
@@ -339,7 +343,6 @@
             # Here, we need to really make a local copy
             size = self.get_size(obj)
             totalsize = self.gcheaderbuilder.size_gc_header + size
-            tls = self.get_tls()
             try:
                 localobj = tls.malloc_local_copy(totalsize)
             except MemoryError:
@@ -453,230 +456,3 @@
 
     def identityhash(self, gcobj):
         return self.id_or_identityhash(gcobj, True)
-
-# ------------------------------------------------------------
-
-
-class Collector(object):
-    """A separate frozen class.  Useful to prevent any buggy concurrent
-    access to GC data.  The methods here use the GCTLS instead for
-    storing things in a thread-local way."""
-
-    def __init__(self, gc):
-        self.gc = gc
-        self.stm_operations = gc.stm_operations
-
-    def _freeze_(self):
-        return True
-
-    def is_in_nursery(self, tls, addr):
-        ll_assert(llmemory.cast_adr_to_int(addr) & 1 == 0,
-                  "odd-valued (i.e. tagged) pointer unexpected here")
-        return tls.nursery_start <= addr < tls.nursery_top
-
-    def header(self, obj):
-        return self.gc.header(obj)
-
-
-    def start_transaction(self):
-        """Start a transaction, by clearing and resetting the tls nursery."""
-        tls = self.get_tls()
-        self.gc.reset_nursery(tls)
-
-
-    def commit_transaction(self):
-        """End of a transaction, just before its end.  No more GC
-        operations should occur afterwards!  Note that the C code that
-        does the commit runs afterwards, and may still abort."""
-        #
-        debug_start("gc-collect-commit")
-        #
-        tls = self.get_tls()
-        #
-        # Do a mark-and-move minor collection out of the tls' nursery
-        # into the main thread's global area (which is right now also
-        # called a nursery).
-        debug_print("local arena:", tls.nursery_free - tls.nursery_start,
-                    "bytes")
-        #
-        # We are starting from the tldict's local objects as roots.  At
-        # this point, these objects have GCFLAG_WAS_COPIED, and the other
-        # local objects don't.  We want to move all reachable local objects
-        # to the global area.
-        #
-        # Start from tracing the root objects
-        self.collect_roots_from_tldict(tls)
-        #
-        # Continue iteratively until we have reached all the reachable
-        # local objects
-        self.collect_from_pending_list(tls)
-        #
-        # Fix up the weakrefs that used to point to local objects
-        self.fixup_weakrefs(tls)
-        #
-        # Now, all indirectly reachable local objects have been copied into
-        # the global area, and all pointers have been fixed to point to the
-        # global copies, including in the local copy of the roots.  What
-        # remains is only overwriting of the global copy of the roots.
-        # This is done by the C code.
-        debug_stop("gc-collect-commit")
-
-
-    def collect_roots_from_tldict(self, tls):
-        tls.pending_list = NULL
-        tls.surviving_weakrefs = NULL
-        # Enumerate the roots, which are the local copies of global objects.
-        # For each root, trace it.
-        CALLBACK = self.stm_operations.CALLBACK_ENUM
-        callback = llhelper(CALLBACK, self._enum_entries)
-        # xxx hack hack hack!  Stores 'self' in a global place... but it's
-        # pointless after translation because 'self' is a Void.
-        _global_collector.collector = self
-        self.stm_operations.tldict_enum(callback)
-
-
-    @staticmethod
-    def _enum_entries(tls_addr, globalobj, localobj):
-        self = _global_collector.collector
-        tls = llmemory.cast_adr_to_ptr(tls_addr, lltype.Ptr(StmGC.GCTLS))
-        #
-        localhdr = self.header(localobj)
-        ll_assert(localhdr.version == globalobj,
-                  "in a root: localobj.version != globalobj")
-        ll_assert(localhdr.tid & GCFLAG_GLOBAL == 0,
-                  "in a root: unexpected GCFLAG_GLOBAL")
-        ll_assert(localhdr.tid & GCFLAG_WAS_COPIED != 0,
-                  "in a root: missing GCFLAG_WAS_COPIED")
-        #
-        self.trace_and_drag_out_of_nursery(tls, localobj)
-
-
-    def collect_from_pending_list(self, tls):
-        while tls.pending_list != NULL:
-            pending_obj = tls.pending_list
-            pending_hdr = self.header(pending_obj)
-            #
-            # 'pending_list' is a chained list of fresh global objects,
-            # linked together via their 'version' field.  The 'version'
-            # must be replaced with NULL after we pop the object from
-            # the linked list.
-            tls.pending_list = pending_hdr.version
-            pending_hdr.version = NULL
-            #
-            # Check the flags of pending_obj: it should be a fresh global
-            # object, without GCFLAG_WAS_COPIED
-            ll_assert(pending_hdr.tid & GCFLAG_GLOBAL != 0,
-                      "from pending list: missing GCFLAG_GLOBAL")
-            ll_assert(pending_hdr.tid & GCFLAG_WAS_COPIED == 0,
-                      "from pending list: unexpected GCFLAG_WAS_COPIED")
-            #
-            self.trace_and_drag_out_of_nursery(tls, pending_obj)
-
-
-    def trace_and_drag_out_of_nursery(self, tls, obj):
-        # This is called to fix the references inside 'obj', to ensure that
-        # they are global.  If necessary, the referenced objects are copied
-        # into the global area first.  This is called on the *local* copy of
-        # the roots, and on the fresh *global* copy of all other reached
-        # objects.
-        self.gc.trace(obj, self._trace_drag_out, tls)
-
-    def _trace_drag_out(self, root, tls):
-        obj = root.address[0]
-        hdr = self.header(obj)
-        #
-        # Figure out if the object is GLOBAL or not by looking at its
-        # address, not at its header --- to avoid cache misses and
-        # pollution for all global objects
-        if not self.is_in_nursery(tls, obj):
-            ll_assert(hdr.tid & GCFLAG_GLOBAL != 0,
-                      "trace_and_mark: non-GLOBAL obj is not in nursery")
-            return        # ignore global objects
-        #
-        ll_assert(hdr.tid & GCFLAG_GLOBAL == 0,
-                  "trace_and_mark: GLOBAL obj in nursery")
-        #
-        if hdr.tid & (GCFLAG_WAS_COPIED | GCFLAG_HAS_SHADOW) == 0:
-            # First visit to a local-only 'obj': allocate a corresponding
-            # global object
-            size = self.gc.get_size(obj)
-            globalobj = self.gc._malloc_global_raw(tls, size)
-            need_to_copy = True
-            #
-        else:
-            globalobj = hdr.version
-            if hdr.tid & GCFLAG_WAS_COPIED != 0:
-                # this local object is a root or was already marked.  Either
-                # way, its 'version' field should point to the corresponding
-                # global object. 
-                size = 0
-                need_to_copy = False
-            else:
-                # this local object has a shadow made by id_or_identityhash();
-                # and the 'version' field points to the global shadow.
-                ll_assert(hdr.tid & GCFLAG_HAS_SHADOW != 0, "uh?")
-                size = self.gc.get_size(obj)
-                need_to_copy = True
-        #
-        if need_to_copy:
-            # Copy the data of the object from the local to the global
-            llmemory.raw_memcopy(obj, globalobj, size)
-            #
-            # Initialize the header of the 'globalobj'
-            globalhdr = self.header(globalobj)
-            globalhdr.tid = hdr.tid | GCFLAG_GLOBAL
-            #
-            # Add the flags to 'localobj' to say 'has been copied now'
-            hdr.tid |= GCFLAG_WAS_COPIED
-            hdr.version = globalobj
-            #
-            # Set a temporary linked list through the globalobj's version
-            # numbers.  This is normally not allowed, but it works here
-            # because these new globalobjs are not visible to any other
-            # thread before the commit is really complete.
-            globalhdr.version = tls.pending_list
-            tls.pending_list = globalobj
-            #
-            if hdr.tid & GCFLAG_WEAKREF != 0:
-                # this was a weakref object that survives.
-                self.young_weakref_survives(tls, obj)
-        #
-        # Fix the original root.address[0] to point to the globalobj
-        root.address[0] = globalobj
-
-
-    @dont_inline
-    def young_weakref_survives(self, tls, obj):
-        # Relink it in the tls.surviving_weakrefs chained list,
-        # via the weakpointer_offset in the local copy of the object.
-        # Do it only if the weakref points to a local object.
-        offset = self.gc.weakpointer_offset(self.gc.get_type_id(obj))
-        if self.is_in_nursery(tls, (obj + offset).address[0]):
-            (obj + offset).address[0] = tls.surviving_weakrefs
-            tls.surviving_weakrefs = obj
-
-    def fixup_weakrefs(self, tls):
-        obj = tls.surviving_weakrefs
-        while obj:
-            offset = self.gc.weakpointer_offset(self.gc.get_type_id(obj))
-            #
-            hdr = self.header(obj)
-            ll_assert(hdr.tid & GCFLAG_GLOBAL == 0,
-                      "weakref: unexpectedly global")
-            globalobj = hdr.version
-            obj2 = (globalobj + offset).address[0]
-            hdr2 = self.header(obj2)
-            ll_assert(hdr2.tid & GCFLAG_GLOBAL == 0,
-                      "weakref: points to a global")
-            if hdr2.tid & GCFLAG_WAS_COPIED:
-                obj2g = hdr2.version    # obj2 survives, going there
-            else:
-                obj2g = llmemory.NULL   # obj2 dies
-            (globalobj + offset).address[0] = obj2g
-            #
-            obj = (obj + offset).address[0]
-
-
-class _GlobalCollector(object):
-    pass
-_global_collector = _GlobalCollector()
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -1,13 +1,15 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
+from pypy.rpython.annlowlevel import cast_instance_to_base_ptr, llhelper
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, base_ptr_lltype
 from pypy.rlib.objectmodel import we_are_translated, free_non_gc_object
+from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rarithmetic import r_uint
 from pypy.rlib.debug import ll_assert, debug_start, debug_stop, fatalerror
 
 from pypy.rpython.memory.gc.stmgc import WORD, NULL
 from pypy.rpython.memory.gc.stmgc import always_inline, dont_inline
 from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED
+from pypy.rpython.memory.gc.stmgc import GCFLAG_WAS_COPIED
 
 
 class StmGCTLS(object):
@@ -101,11 +103,8 @@
     def enter_transactional_mode(self):
         """Called on the main thread, just before spawning the other
         threads."""
-        self.local_collection()
-        if not self.local_nursery_is_empty():
-            self.local_collection(run_finalizers=False)
-        self._promote_locals_to_globals()
-        self._disable_mallocs()
+        self.stop_transaction()
+        self.stm_operations.enter_transactional_mode()
 
     def leave_transactional_mode(self):
         """Restart using the main thread for mallocs."""
@@ -113,10 +112,11 @@
             for key, value in StmGCTLS.nontranslated_dict.items():
                 if value is not self:
                     del StmGCTLS.nontranslated_dict[key]
+        self.stm_operations.leave_transactional_mode()
         self.start_transaction()
 
     def start_transaction(self):
-        """Enter a thread: performs any pending cleanups, and set
+        """Start a transaction: performs any pending cleanups, and set
         up a fresh state for allocating.  Called at the start of
         each transaction, and at the start of the main thread."""
         # Note that the calls to enter() and
@@ -134,11 +134,24 @@
         self.nursery_free = self.nursery_start
         self.nursery_top  = self.nursery_start + self.nursery_size
 
+    def stop_transaction(self):
+        """Stop a transaction: do a local collection to empty the
+        nursery and track which objects are still alive now, and
+        then mark all these objects as global."""
+        self.local_collection()
+        if not self.local_nursery_is_empty():
+            self.local_collection(run_finalizers=False)
+        self._promote_locals_to_globals()
+        self._disable_mallocs()
+
     def local_nursery_is_empty(self):
         ll_assert(bool(self.nursery_free),
                   "local_nursery_is_empty: gc not running")
         return self.nursery_free == self.nursery_start
 
+    def in_transaction(self):
+        return bool(self.nursery_free)
+
     # ------------------------------------------------------------
 
     def local_collection(self, run_finalizers=True):
@@ -192,18 +205,6 @@
         #
         debug_stop("gc-local")
 
-    def end_of_transaction_collection(self):
-        """Do an end-of-transaction collection.  Finds all surviving
-        non-GCFLAG_WAS_COPIED young objects and make them old.  Assumes
-        that there are no roots from the stack.  This guarantees that the
-        nursery will end up empty, apart from GCFLAG_WAS_COPIED objects.
-        To finish the commit, the C code will need to copy them over the
-        global objects (or abort in case of conflict, which is still ok).
-
-        No more mallocs are allowed after this is called.
-        """
-        raise NotImplementedError
-
     # ------------------------------------------------------------
 
     @always_inline
@@ -245,8 +246,6 @@
 
     def _promote_locals_to_globals(self):
         ll_assert(self.local_nursery_is_empty(), "nursery must be empty [1]")
-        ll_assert(not self.sharedarea_tls.special_stack.non_empty(),
-                  "special_stack should be empty here [1]")
         #
         # Promote all objects in sharedarea_tls to global
         obj = self.sharedarea_tls.chained_list
@@ -272,18 +271,31 @@
         self.gc.root_walker.walk_current_stack_roots(
             StmGCTLS._trace_drag_out1, self)
 
+    def trace_and_drag_out_of_nursery(self, obj):
+        # This is called to fix the references inside 'obj', to ensure that
+        # they are global.  If necessary, the referenced objects are copied
+        # into the global area first.  This is called on the LOCAL copy of
+        # the roots, and on the freshly OLD copy of all other reached LOCAL
+        # objects.
+        self.gc.trace(obj, self._trace_drag_out, None)
+
     def _trace_drag_out1(self, root):
         self._trace_drag_out(root, None)
 
     def _trace_drag_out(self, root, ignored):
         """Trace callback: 'root' is the address of some pointer.  If that
         pointer points to a YOUNG object, allocate an OLD copy of it and
-        fix the pointer.  Also, add the object to 'pending_list', if it was
-        not done so far.
+        fix the pointer.  Also, add the object to the 'pending' stack, if
+        it was not done so far.
         """
         obj = root.address[0]
         hdr = self.gc.header(obj)
         #
+        # If 'obj' is a LOCAL copy of a GLOBAL object, skip it
+        # (this case is handled differently in collect_roots_from_tldict)
+        if hdr.tid & GCFLAG_WAS_COPIED:
+            return
+        #
         # If 'obj' is not in the nursery, we set GCFLAG_VISITED
         if not self.is_in_nursery(obj):
             if hdr.tid & GCFLAG_VISITED == 0:
@@ -335,14 +347,32 @@
         self.sharedarea_tls.add_regular(obj)
 
     def collect_roots_from_tldict(self):
-        pass  # XXX
+        if not we_are_translated():
+            if not hasattr(self.stm_operations, 'tldict_enum'):
+                return
+        CALLBACK = self.stm_operations.CALLBACK_ENUM
+        callback = llhelper(CALLBACK, StmGCTLS._enum_entries)
+        self.stm_operations.tldict_enum(callback)
+
+    @staticmethod
+    def _enum_entries(tlsaddr, globalobj, localobj):
+        self = StmGCTLS.cast_address_to_tls_object(tlsaddr)
+        localhdr = self.gc.header(localobj)
+        ll_assert(localhdr.version == globalobj,
+                  "in a root: localobj.version != globalobj")
+        ll_assert(localhdr.tid & GCFLAG_GLOBAL == 0,
+                  "in a root: unexpected GCFLAG_GLOBAL")
+        ll_assert(localhdr.tid & GCFLAG_WAS_COPIED != 0,
+                  "in a root: missing GCFLAG_WAS_COPIED")
+        #
+        self.trace_and_drag_out_of_nursery(localobj)
 
     def collect_flush_pending(self):
         # Follow the objects in the 'pending' stack and move the
         # young objects they point to out of the nursery.
         while self.pending.non_empty():
             obj = self.pending.pop()
-            self.gc.trace(obj, self._trace_drag_out, None)
+            self.trace_and_drag_out_of_nursery(obj)
         self.pending.delete()
 
     def mass_free_old_local(self, previous_sharedarea_tls):
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -57,6 +57,14 @@
     def gct_stm_descriptor_done(self, hop):
         hop.genop("direct_call", [self.teardown_thread_ptr, self.c_const_gc])
 
+    def gct_stm_enter_transactional_mode(self, hop):
+        hop.genop("direct_call", [self.stm_enter_transactional_mode_ptr,
+                                  self.c_const_gc])
+
+    def gct_stm_leave_transactional_mode(self, hop):
+        hop.genop("direct_call", [self.stm_leave_transactional_mode_ptr,
+                                  self.c_const_gc])
+
     def gct_stm_writebarrier(self, hop):
         op = hop.spaceop
         v_adr = hop.genop('cast_ptr_to_adr',
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -143,26 +143,18 @@
 {
   owner_version_t newver = d->end_time;
   wlog_t *item;
-  /* loop in "forward" order: in this order, if there are duplicate orecs
-     then only the last one has p != -1. */
   REDOLOG_LOOP_FORWARD(d->redolog, item)
     {
       void *globalobj = item->addr;
       void *localobj = item->val;
-      owner_version_t p = item->p;
       long size = rpython_get_size(localobj);
       memcpy(((char *)globalobj) + sizeof(orec_t),
              ((char *)localobj) + sizeof(orec_t),
              size - sizeof(orec_t));
-      /* but we must only unlock the orec if it's the last time it
-         appears in the redolog list.  If it's not, then p == -1.
-         XXX I think that duplicate orecs are not possible any more. */
-      if (p != -1)
-        {
-          volatile orec_t* o = get_orec(globalobj);
-          CFENCE;
-          o->version = newver;
-        }
+      /* unlock the orec */
+      volatile orec_t* o = get_orec(globalobj);
+      CFENCE;
+      o->version = newver;
     } REDOLOG_LOOP_END;
 }
 
@@ -186,12 +178,10 @@
   wlog_t *item;
   REDOLOG_LOOP_FORWARD(d->redolog, item)
     {
-      if (item->p != -1)
-        {
-          volatile orec_t* o = get_orec(item->addr);
-          o->version = item->p;
-          item->p = -1;
-        }
+      volatile orec_t* o = get_orec(item->addr);
+      assert(item->p != -1);
+      o->version = item->p;
+      item->p = -1;
     } REDOLOG_LOOP_END;
 }
 
@@ -218,14 +208,14 @@
         if (!bool_cas(&o->version, ovt, d->my_lock_word))
           goto retry;
         // save old version to item->p.  Now we hold the lock.
-        // in case of duplicate orecs, only the last one has p != -1.
         item->p = ovt;
       }
       // else if the location is too recent...
       else if (!IS_LOCKED(ovt))
         tx_abort(0);
-      // else it is locked: if we don't hold the lock...
-      else if (ovt != d->my_lock_word) {
+      // else it is locked: check it's not by me
+      else {
+        assert(ovt != d->my_lock_word);
         // we can either abort or spinloop.  Because we are at the end of
         // the transaction we might try to spinloop, even though after the
         // lock is released the ovt will be very recent, possibly
@@ -534,7 +524,10 @@
       /*d->spinloop_counter = (unsigned int)(d->my_lock_word | 1);*/
 
       thread_descriptor = d;
-      /* active_thread_descriptor stays NULL */
+      if (in_main_thread)
+        stm_leave_transactional_mode();
+      else
+        ;   /* active_thread_descriptor stays NULL */
 
 #ifdef RPY_STM_DEBUG_PRINT
       if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE, "thread %lx starting\n",
@@ -681,6 +674,33 @@
   return result;
 }
 
+void stm_enter_transactional_mode(void)
+{
+  struct tx_descriptor *d = active_thread_descriptor;
+  assert(d != NULL);
+  assert(is_inevitable(d));
+  /* we only need a subset of a full commit */
+  acquireLocks(d);
+  commitInevitableTransaction(d);
+  common_cleanup(d);
+  active_thread_descriptor = NULL;
+}
+
+void stm_leave_transactional_mode(void)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  assert(active_thread_descriptor == NULL);
+
+  mutex_lock();
+  d->setjmp_buf = NULL;
+  d->start_time = get_global_timestamp(d);
+  assert(!(d->start_time & 1));
+  set_global_timestamp(d, d->start_time | 1);
+
+  assert(is_inevitable(d));
+  active_thread_descriptor = d;
+}
+
 void stm_try_inevitable(STM_CCHARP1(why))
 {
   /* when a transaction is inevitable, its start_time is equal to
@@ -745,13 +765,12 @@
     return -2;
   if (active_thread_descriptor == NULL)
     {
-      if (d->my_lock_word == 0)
-        return -1;
-      else
-        return 0;
+      assert(d->my_lock_word != 0);
+      return 0;
     }
   assert(d == active_thread_descriptor);
-  assert(d->my_lock_word != 0);
+  if (d->my_lock_word == 0)
+    return -1;
   if (!is_inevitable(d))
     return 1;
   else
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -40,6 +40,8 @@
 
 
 void* stm_perform_transaction(void*(*)(void*, long), void*);
+void stm_enter_transactional_mode(void);
+void stm_leave_transactional_mode(void);
 void stm_try_inevitable(STM_CCHARP1(why));
 void stm_abort_and_retry(void);
 long stm_debug_get_state(void);  /* -1: descriptor_init() was not called
diff --git a/pypy/translator/stm/stmgcintf.py b/pypy/translator/stm/stmgcintf.py
--- a/pypy/translator/stm/stmgcintf.py
+++ b/pypy/translator/stm/stmgcintf.py
@@ -56,6 +56,11 @@
     get_tls = smexternal('stm_get_tls', [], llmemory.Address)
     del_tls = smexternal('stm_del_tls', [], lltype.Void)
 
+    enter_transactional_mode = smexternal('stm_enter_transactional_mode',
+                                          [], lltype.Void)
+    leave_transactional_mode = smexternal('stm_leave_transactional_mode',
+                                          [], lltype.Void)
+
     tldict_lookup = smexternal('stm_tldict_lookup', [llmemory.Address],
                                llmemory.Address)
     tldict_add = smexternal('stm_tldict_add', [llmemory.Address] * 2,

From noreply at buildbot.pypy.org  Mon Apr 16 15:51:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 15:51:34 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fixes
Message-ID: <20120416135134.49081820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54420:f0e91e76c868
Date: 2012-04-16 15:50 +0200
http://bitbucket.org/pypy/pypy/changeset/f0e91e76c868/

Log:	Fixes

diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -100,6 +100,7 @@
 
 @rgc.no_collect     # don't use the gc as long as other threads are running
 def _run():
+    rstm.enter_transactional_mode()
     i = 0
     while i < glob.NUM_THREADS:
         glob._arg = glob._arglist[i]
@@ -111,6 +112,7 @@
     while glob.done < glob.NUM_THREADS:    # poor man's lock
         _sleep(rffi.cast(rffi.ULONG, 1))
     debug_print("done sleeping.")
+    rstm.leave_transactional_mode()
 
 
 # Posix only
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -249,7 +249,7 @@
     # one variable on which we do 'stm_writebarrier', but there are
     # also other variables that contain the same pointer, e.g. casted
     # to a different precise type.
-    from pypy.translator.stm.gcsource import COPIES_POINTER
+    from pypy.translator.stm.gcsource import COPIES_POINTER, _is_gc
     #
     def emit(op):
         for v1 in op.args:
@@ -275,7 +275,8 @@
         for op in block.operations:
             if op.opname in COPIES_POINTER:
                 assert len(op.args) == 1
-                copies[op.result] = op
+                if _is_gc(op.result) and _is_gc(op.args[0]):
+                    copies[op.result] = op
             elif (op.opname in ('getfield', 'getarrayitem',
                                 'getinteriorfield') and
                   op.result.concretetype is not lltype.Void and

From noreply at buildbot.pypy.org  Mon Apr 16 16:21:23 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 16:21:23 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fixes,
	enough to have 'targetdemo-c' pass.
Message-ID: <20120416142123.6B815820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54421:665a595261ba
Date: 2012-04-16 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/665a595261ba/

Log:	Fixes, enough to have 'targetdemo-c' pass.

diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -22,10 +22,10 @@
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
-                 translated_to_c=True):
+                 multithread=False, translated_to_c=True):
         self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
-        self.AddressStack = get_address_stack(chunk_size)
-        self.AddressDeque = get_address_deque(chunk_size)
+        self.AddressStack = get_address_stack(chunk_size, multithread)
+        self.AddressDeque = get_address_deque(chunk_size, multithread)
         self.AddressDict = AddressDict
         self.null_address_dict = null_address_dict
         self.config = config
diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -84,7 +84,7 @@
                  #small_request_threshold=5*WORD,
                  #ArenaCollectionClass=None,
                  **kwds):
-        MovingGCBase.__init__(self, config, **kwds)
+        MovingGCBase.__init__(self, config, multithread=True, **kwds)
         #
         if isinstance(stm_operations, str):
             assert stm_operations == 'use_real_one', (
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -200,7 +200,9 @@
         #
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
-        llarena.arena_reset(self.nursery_start, self.nursery_size, 2)
+        ll_assert(bool(self.nursery_free), "nursery_free is NULL")
+        size_used = self.nursery_free - self.nursery_start
+        llarena.arena_reset(self.nursery_start, size_used, 2)
         self.nursery_free = self.nursery_start
         #
         debug_stop("gc-local")
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -16,9 +16,10 @@
 DEFAULT_CHUNK_SIZE = 1019
 
 
-def get_chunk_manager(chunk_size=DEFAULT_CHUNK_SIZE, cache={}):
+def get_chunk_manager(chunk_size=DEFAULT_CHUNK_SIZE, multithread=False,
+                      cache={}):
     try:
-        return cache[chunk_size]
+        return cache[chunk_size, multithread]
     except KeyError:
         pass
 
@@ -33,10 +34,11 @@
         _alloc_flavor_ = "raw"
 
         def __init__(self):
-            self.free_list = null_chunk
+            if not multithread:
+                self.free_list = null_chunk
 
         def get(self):
-            if not self.free_list:
+            if multithread or not self.free_list:
                 # we zero-initialize the chunks to make the translation
                 # backends happy, but we don't need to do it at run-time.
                 zero = not we_are_translated()
@@ -48,7 +50,7 @@
             return result
 
         def put(self, chunk):
-            if we_are_translated():
+            if not multithread and we_are_translated():
                 chunk.next = self.free_list
                 self.free_list = chunk
             else:
@@ -58,17 +60,18 @@
                 lltype.free(chunk, flavor="raw", track_allocation=False)
 
     unused_chunks = FreeList()
-    cache[chunk_size] = unused_chunks, null_chunk
+    cache[chunk_size, multithread] = unused_chunks, null_chunk
     return unused_chunks, null_chunk
 
 
-def get_address_stack(chunk_size=DEFAULT_CHUNK_SIZE, cache={}):
+def get_address_stack(chunk_size=DEFAULT_CHUNK_SIZE, multithread=False,
+                      cache={}):
     try:
-        return cache[chunk_size]
+        return cache[chunk_size, multithread]
     except KeyError:
         pass
 
-    unused_chunks, null_chunk = get_chunk_manager(chunk_size)
+    unused_chunks, null_chunk = get_chunk_manager(chunk_size, multithread)
 
     class AddressStack(object):
         _alloc_flavor_ = "raw"
@@ -173,20 +176,21 @@
                 chunk.items[count] = got
                 got = next
 
-    cache[chunk_size] = AddressStack
+    cache[chunk_size, multithread] = AddressStack
     return AddressStack
 
 def _add_in_dict(item, d):
     d.add(item)
 
 
-def get_address_deque(chunk_size=DEFAULT_CHUNK_SIZE, cache={}):
+def get_address_deque(chunk_size=DEFAULT_CHUNK_SIZE, multithread=False,
+                      cache={}):
     try:
-        return cache[chunk_size]
+        return cache[chunk_size, multithread]
     except KeyError:
         pass
 
-    unused_chunks, null_chunk = get_chunk_manager(chunk_size)
+    unused_chunks, null_chunk = get_chunk_manager(chunk_size, multithread)
 
     class AddressDeque(object):
         _alloc_flavor_ = "raw"
@@ -261,7 +265,7 @@
                 cur = next
             free_non_gc_object(self)
 
-    cache[chunk_size] = AddressDeque
+    cache[chunk_size, multithread] = AddressDeque
     return AddressDeque
 
 # ____________________________________________________________
diff --git a/pypy/translator/c/src/debug_alloc.h b/pypy/translator/c/src/debug_alloc.h
--- a/pypy/translator/c/src/debug_alloc.h
+++ b/pypy/translator/c/src/debug_alloc.h
@@ -1,10 +1,11 @@
 /**************************************************************/
 /***  tracking raw mallocs and frees for debugging          ***/
 
-#ifndef RPY_ASSERT
+#if !defined(RPY_ASSERT) || defined(RPY_STM)   /* <= not thread-safe */
 
 #  define OP_TRACK_ALLOC_START(addr, r)   /* nothing */
 #  define OP_TRACK_ALLOC_STOP(addr, r)    /* nothing */
+#  define pypy_debug_alloc_results()      /* nothing */
 
 #else   /* ifdef RPY_ASSERT */
 
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -59,9 +59,7 @@
 
     exitcode = STANDALONE_ENTRY_POINT(argc, argv);
 
-#ifdef RPY_ASSERT
     pypy_debug_alloc_results();
-#endif
 
     if (RPyExceptionOccurred()) {
         /* print the RPython traceback */

From noreply at buildbot.pypy.org  Mon Apr 16 16:37:03 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 16 Apr 2012 16:37:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: a simple recursive finder of source
	positions
Message-ID: <20120416143703.1B55C820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54422:563a7d2fac42
Date: 2012-04-16 12:30 +0200
http://bitbucket.org/pypy/pypy/changeset/563a7d2fac42/

Log:	a simple recursive finder of source positions

diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 

From noreply at buildbot.pypy.org  Mon Apr 16 16:37:04 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 16 Apr 2012 16:37:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120416143704.5F33F820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54423:3c8d3429fe18
Date: 2012-04-16 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/3c8d3429fe18/

Log:	merge

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -171,7 +175,13 @@
         space = self.space
         keywords, values_w = space.view_as_kwargs(w_starstararg)
         if keywords is not None: # this path also taken for empty dicts
-            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
             return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
             keys_w = space.unpackiterable(w_starstararg)
@@ -229,22 +239,16 @@
     @jit.look_inside_iff(lambda self, keywords, keywords_w:
             jit.isconstant(len(keywords) and
             jit.isconstant(self.keywords)))
-    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
-        if self.keywords is None:
-            self.keywords = keywords[:] # copy to make non-resizable
-            self.keywords_w = keywords_w[:]
-        else:
-            # looks quadratic, but the JIT should remove all of it nicely.
-            # Also, all the lists should be small
-            for key in keywords:
-                for otherkey in self.keywords:
-                    if otherkey == key:
-                        raise operationerrfmt(self.space.w_TypeError,
-                                              "got multiple values "
-                                              "for keyword argument "
-                                              "'%s'", key)
-            self.keywords = self.keywords + keywords
-            self.keywords_w = self.keywords_w + keywords_w
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
 
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,9 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +18,17 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +40,7 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
     includes = includes,
     export_symbols = [],
     post_include_bits = [

From noreply at buildbot.pypy.org  Mon Apr 16 16:43:33 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 16 Apr 2012 16:43:33 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: rework get_osfhandle to call
	validate_fd first
Message-ID: <20120416144333.548B9820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54424:95133865222a
Date: 2012-04-16 17:43 +0300
http://bitbucket.org/pypy/pypy/changeset/95133865222a/

Log:	rework get_osfhandle to call validate_fd first

diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -739,9 +739,7 @@
         # assume -1 and 0 both mean invalid file descriptor
         # to 'anonymously' map memory.
         if fileno != -1 and fileno != 0:
-            if not rposix.validate_fd(fileno):
-                raise OSError(rposix.get_errno(), 'Bad file descriptor')
-            fh = rwin32._get_osfhandle(fileno)
+            fh = rwin32.get_osfhandle(fileno)
             if fh == INVALID_HANDLE:
                 errno = rposix.get_errno()
                 raise OSError(errno, os.strerror(errno))
diff --git a/pypy/rlib/rwin32.py b/pypy/rlib/rwin32.py
--- a/pypy/rlib/rwin32.py
+++ b/pypy/rlib/rwin32.py
@@ -8,6 +8,7 @@
 from pypy.translator.platform import CompilationError
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib.rarithmetic import intmask
+from pypy.rlib.rposix import validate_fd
 from pypy.rlib import jit
 import os, sys, errno
 
@@ -126,6 +127,11 @@
 
     _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE)
 
+    def get_osfhandle(fd):
+        if not validate_fd(fileno):
+            raise OSError(GetLastError(), 'Bad file descriptor')
+        return _get_osfhandle(fd)
+
     def build_winerror_to_errno():
         """Build a dictionary mapping windows error numbers to POSIX errno.
         The function returns the dict, and the default value for codes not
diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -181,8 +181,6 @@
     import errno
 
     _eci = ExternalCompilationInfo()
-    _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], rffi.LONG,
-                                     compilation_info=_eci)
     _setmode = rffi.llexternal('_setmode', [rffi.INT, rffi.INT], rffi.INT,
                                compilation_info=_eci)
     SetEndOfFile = rffi.llexternal('SetEndOfFile', [rffi.LONG], rwin32.BOOL,
@@ -200,7 +198,7 @@
             # move to the position to be truncated
             os.lseek(fd, size, 0)
             # Truncate.  Note that this may grow the file!
-            handle = _get_osfhandle(fd)
+            handle = rwin32.get_osfhandle(fd)
             if handle == -1:
                 raise OSError(errno.EBADF, "Invalid file handle")
             if not SetEndOfFile(handle):
diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -402,7 +402,7 @@
             lltype.free(data, flavor='raw')
 
     def win32_fstat_llimpl(fd):
-        handle = rwin32._get_osfhandle(fd)
+        handle = rwin32.get_osfhandle(fd)
 
         filetype = win32traits.GetFileType(handle)
         if filetype == win32traits.FILE_TYPE_CHAR:

From noreply at buildbot.pypy.org  Mon Apr 16 16:50:25 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 16 Apr 2012 16:50:25 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: experiment with llexternal
	definition
Message-ID: <20120416145025.508C0820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54425:db0e69d3f9ee
Date: 2012-04-16 17:49 +0300
http://bitbucket.org/pypy/pypy/changeset/db0e69d3f9ee/

Log:	experiment with llexternal definition

diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -107,7 +107,7 @@
     validate_fd = rffi.llexternal(
         "_PyVerify_fd", [rffi.INT], rffi.INT,
         _callable=validate_fd_emulator, compilation_info=eci,
-        _nowrapper=True, elidable_function=True, sandboxsafe=True,
+        _nowrapper=True, elidable_function=False, sandboxsafe=False,
         )
 else:
     def validate_fd(fd):

From noreply at buildbot.pypy.org  Mon Apr 16 17:24:06 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 17:24:06 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: kill this class altogheter: range() no
 longer returns a list, so most of these tests don't pass
Message-ID: <20120416152406.3F7D9820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54426:1a61563e6769
Date: 2012-04-16 17:13 +0200
http://bitbucket.org/pypy/pypy/changeset/1a61563e6769/

Log:	kill this class altogheter: range() no longer returns a list, so
	most of these tests don't pass

diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1157,95 +1157,6 @@
             s.update(Sub2(arg))
             assert s == set(base(arg))
 
-class AppTestForRangeLists(AppTestW_ListObject):
-
-    ## def setup_class(cls):
-    ##     cls.space = gettestobjspace(**{"objspace.std.withrangelist" :
-    ##                                    True})
-
-    def test_range_simple_backwards(self):
-        x = range(5,1)
-        assert x == []
-
-    def test_range_big_start(self):
-        x = range(1,10)
-        x[22:0:-1] == range(1,10)
-
-    def test_range_list_invalid_slice(self):
-        x = [1,2,3,4]
-        assert x[10:0] == []
-        assert x[10:0:None] == []
-
-        x = range(1,5)
-        assert x[10:0] == []
-        assert x[10:0:None] == []
-
-        assert x[0:22] == [1,2,3,4]
-        assert x[-1:10] == [4]
-
-        assert x[0:22:None] == [1,2,3,4]
-        assert x[-1:10:None] == [4]
-
-    def test_range_backwards(self):
-        x = range(1,10)
-        assert x[22:-10] == []
-        assert x[22:-10:-1] == [9,8,7,6,5,4,3,2,1]
-        assert x[10:3:-1] == [9,8,7,6,5]
-        assert x[10:3:-2] == [9,7,5]
-        assert x[1:5:-1] == []
-
-    def test_sort_range(self):
-        l = range(3,10,3)
-        l.sort()
-        assert l == [3, 6, 9]
-        l.sort(reverse = True)
-        assert l == [9, 6, 3]
-        l.sort(reverse = True)
-        assert l == [9, 6, 3]
-        l.sort()
-        assert l == [3, 6, 9]
-
-    def test_slice(self):
-        l = []
-        l2 = range(3)
-        l.__setslice__(0,3,l2)
-        assert l == [0,1,2]
-
-    def test_getitem(self):
-        l = range(5)
-        raises(IndexError, "l[-10]")
-
-    def test_append(self):
-        l = range(5)
-        l.append(26)
-        assert l == [0,1,2,3,4,26]
-
-        l = range(5)
-        l.append("a")
-        assert l == [0,1,2,3,4,"a"]
-
-        l = range(5)
-        l.append(5)
-        assert l == [0,1,2,3,4,5]
-
-    def test_pop(self):
-        l = range(3)
-        assert l.pop(0) == 0
-
-    def test_setitem(self):
-        l = range(3)
-        l[0] = 1
-        assert l == [1,1,2]
-
-    def test_inset(self):
-        l = range(3)
-        l.insert(1,5)
-        assert l == [0,5,1,2]
-
-    def test_reverse(self):
-        l = range(3)
-        l.reverse()
-        assert l == [2,1,0]
 
 
 class AppTestWithoutStrategies(object):

From noreply at buildbot.pypy.org  Mon Apr 16 17:24:07 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 17:24:07 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: test and fix by disabling the
 unicodeliststragegy when converting from bytes
Message-ID: <20120416152407.B7131820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54427:4deee06acd87
Date: 2012-04-16 17:23 +0200
http://bitbucket.org/pypy/pypy/changeset/4deee06acd87/

Log:	test and fix by disabling the unicodeliststragegy when converting
	from bytes

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1084,12 +1084,15 @@
             w_list.lstorage = strategy.erase(intlist[:])
             return
 
-        strlist = space.listview_str(w_iterable)
-        if strlist is not None:
-            w_list.strategy = strategy = space.fromcache(UnicodeListStrategy)
-             # need to copy because intlist can share with w_iterable
-            w_list.lstorage = strategy.erase(strlist[:])
-            return
+        # XXX: listview_str works for bytes but not for strings, and the
+        # strategy works for strings but not for bytes. Disable it for now,
+        # but we'll need to fix it
+        ## strlist = space.listview_str(w_iterable)
+        ## if strlist is not None:
+        ##     w_list.strategy = strategy = space.fromcache(UnicodeListStrategy)
+        ##      # need to copy because intlist can share with w_iterable
+        ##     w_list.lstorage = strategy.erase(strlist[:])
+        ##     return
 
         # xxx special hack for speed
         from pypy.interpreter.generator import GeneratorIterator
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -1134,6 +1134,10 @@
         assert l == []
         assert list(g) == []
 
+    def test_list_from_bytes(self):
+        b = list(b'abc')
+        assert b == [97, 98, 99]
+
     def test_uses_custom_iterator(self):
         # obscure corner case: space.listview*() must not shortcut subclasses
         # of dicts, because the OrderedDict in the stdlib relies on this.
@@ -1158,7 +1162,6 @@
             assert s == set(base(arg))
 
 
-
 class AppTestWithoutStrategies(object):
 
     def setup_class(cls):

From noreply at buildbot.pypy.org  Mon Apr 16 18:05:04 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 18:05:04 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Start to rewrite test_stmgc.py.
Message-ID: <20120416160504.A0FA2820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54428:8f3f4e2f10be
Date: 2012-04-16 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/8f3f4e2f10be/

Log:	Start to rewrite test_stmgc.py.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -190,47 +190,6 @@
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
 
-    def _malloc_local_raw(self, tls, size):
-        # for _stm_write_barrier_global(): a version of malloc that does
-        # no initialization of the malloc'ed object
-        xxxxxxxx
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        totalsize = size_gc_header + size
-        result = self._allocate_bump_pointer(tls, totalsize)
-        llarena.arena_reserve(result, totalsize)
-        obj = result + size_gc_header
-        return obj
-
-
-    def _malloc_global_raw(self, tls, size):
-        # For collection: allocates enough space for a global object from
-        # the main_tls.  The argument 'tls' is the current (local) GCTLS.
-        # We try to do it by reserving "pages" of memory from the global
-        # area at once, and subdividing here.
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        totalsize = size_gc_header + size
-        freespace = tls.global_stop - tls.global_free
-        if freespace < llmemory.raw_malloc_usage(totalsize):
-            self._malloc_global_more(tls, llmemory.raw_malloc_usage(totalsize))
-        result = tls.global_free
-        tls.global_free = result + totalsize
-        llarena.arena_reserve(result, totalsize)
-        obj = result + size_gc_header
-        return obj
-
-    @dont_inline
-    def _malloc_global_more(self, tls, totalsize):
-        xxxxxxxxxxxxxxxxx
-        if totalsize < self.tls_page_size:
-            totalsize = self.tls_page_size
-        main_tls = self.main_thread_tls
-        self.acquire(self.mutex_lock)
-        result = self._allocate_bump_pointer(main_tls, totalsize)
-        self.release(self.mutex_lock)
-        tls.global_free = result
-        tls.global_stop = result + totalsize
-
-
     def collect(self, gen=1):
         self.get_tls().local_collection()
         if gen > 0:
@@ -392,14 +351,6 @@
         self.stm_normalize_global = stm_normalize_global
 
     # ----------
-
-    def acquire(self, lock):
-        ll_thread.acquire_NOAUTO(lock, True)
-
-    def release(self, lock):
-        ll_thread.release_NOAUTO(lock)
-
-    # ----------
     # id() and identityhash() support
 
     def id_or_identityhash(self, gcobj, is_hash):
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -138,9 +138,9 @@
         """Stop a transaction: do a local collection to empty the
         nursery and track which objects are still alive now, and
         then mark all these objects as global."""
-        self.local_collection()
+        self.local_collection(end_of_transaction=1)
         if not self.local_nursery_is_empty():
-            self.local_collection(run_finalizers=False)
+            self.local_collection(end_of_transaction=2)
         self._promote_locals_to_globals()
         self._disable_mallocs()
 
@@ -154,7 +154,7 @@
 
     # ------------------------------------------------------------
 
-    def local_collection(self, run_finalizers=True):
+    def local_collection(self, end_of_transaction=0):
         """Do a local collection.  This should be equivalent to a minor
         collection only, but the GC is not generational so far, so it is
         for now the same as a full collection --- but only on LOCAL
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -132,6 +132,10 @@
 def fake_weakpointer_offset(tid):
     return llmemory.offsetof(WR, 'wadr')
 
+class FakeRootWalker:
+    def walk_current_stack_roots(self, *args):
+        pass     # no stack roots in this test file
+
 
 class StmGCTests:
     GCClass = StmGC
@@ -146,6 +150,7 @@
         self.gc.get_size = fake_get_size
         self.gc.trace = fake_trace
         self.gc.weakpointer_offset = fake_weakpointer_offset
+        self.gc.root_walker = FakeRootWalker()
         self.gc.setup()
 
     def teardown_method(self, meth):
@@ -159,18 +164,22 @@
 
     # ----------
     # test helpers
-    def malloc(self, STRUCT, weakref=False):
+    def malloc(self, STRUCT, weakref=False, globl='auto'):
         size = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT))
         tid = lltype.cast_primitive(llgroup.HALFWORD, 123)
         gcref = self.gc.malloc_fixedsize_clear(tid, size,
                                                contains_weakptr=weakref)
         realobj = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), gcref)
         addr = llmemory.cast_ptr_to_adr(realobj)
+        if globl == 'auto':
+            globl = (self.gc.stm_operations.threadnum == 0)
+        if globl:
+            self.gc.header(addr).tid |= GCFLAG_GLOBAL
         return realobj, addr
     def select_thread(self, threadnum):
         self.gc.stm_operations.threadnum = threadnum
         if threadnum not in self.gc.stm_operations._tls_dict:
-            self.gc.setup_thread(False)
+            self.gc.setup_thread()
             self.gc.start_transaction()
     def gcsize(self, S):
         return (llmemory.raw_malloc_usage(llmemory.sizeof(self.gc.HDR)) +
@@ -313,15 +322,20 @@
         assert u_adr == t_adr
 
     def test_write_barrier_main_thread(self):
-        t, t_adr = self.malloc(S)
-        obj = self.gc.stm_writebarrier(t_adr)     # main thread
+        t, t_adr = self.malloc(S, globl=False)
+        obj = self.gc.stm_writebarrier(t_adr)     # main thread, but not global
         assert obj == t_adr
 
+    def test_write_barrier_global(self):
+        t, t_adr = self.malloc(S, globl=True)
+        obj = self.gc.stm_writebarrier(t_adr)     # global, even if main thread
+        assert obj != t_adr
+
     def test_commit_transaction_empty(self):
         self.select_thread(1)
         s, s_adr = self.malloc(S)
         t, t_adr = self.malloc(S)
-        self.gc.collector.commit_transaction()    # no roots
+        self.gc.commit_transaction()    # no roots
         main_tls = self.gc.main_thread_tls
         assert main_tls.nursery_free == main_tls.nursery_start   # empty
 
@@ -339,15 +353,14 @@
         assert main_tls.nursery_free != main_tls.nursery_start  # contains s
         old_value = main_tls.nursery_free
         #
-        self.gc.collector.commit_transaction()
+        self.gc.commit_transaction()
         #
         assert main_tls.nursery_free == old_value    # no new object
         assert s.b == 12345     # not updated by the GC code
         assert t.b == 67890     # still valid
 
-    def _commit_transaction_with_one_reference(self, tls_page_size):
-        self.gc.tls_page_size = tls_page_size
-        #
+    def test_commit_transaction_with_one_reference(self):
+        py.test.skip("rewrite me")
         sr, sr_adr = self.malloc(SR)
         assert sr.s1 == lltype.nullptr(S)
         assert sr.sr2 == lltype.nullptr(SR)
@@ -361,27 +374,18 @@
         assert tr.sr2 == lltype.nullptr(SR)
         tr.s1 = t
         #
-        main_tls = self.gc.main_thread_tls
-        old_value = main_tls.nursery_free
+        tls = self.gc.get_tls()
+        old_value = tls.nursery_free
         #
-        self.gc.collector.commit_transaction()
+        tls.stop_transaction()
         #
-        consumed = main_tls.nursery_free - old_value
+        consumed = tls.nursery_free - old_value
         expected = self.gcsize(S)        # round this value up to tls_page_size
         if expected < tls_page_size: expected = tls_page_size
         assert consumed == expected
 
-    def test_commit_transaction_with_one_reference_1(self):
-        self._commit_transaction_with_one_reference(1)
-
-    def test_commit_transaction_with_one_reference_N1(self):
-        N1 = self.gcsize(S)-1
-        self._commit_transaction_with_one_reference(N1)
-
-    def test_commit_transaction_with_one_reference_128(self):
-        self._commit_transaction_with_one_reference(128)
-
     def test_commit_transaction_with_graph(self):
+        py.test.skip("rewrite me")
         self.gc.tls_page_size = 1
         sr1, sr1_adr = self.malloc(SR)
         sr2, sr2_adr = self.malloc(SR)

From noreply at buildbot.pypy.org  Mon Apr 16 18:05:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 18:05:05 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: id(), hash(), and small refactors
Message-ID: <20120416160505.E4872820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54429:48a8fcb4017c
Date: 2012-04-16 17:42 +0200
http://bitbucket.org/pypy/pypy/changeset/48a8fcb4017c/

Log:	id(), hash(), and small refactors

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -319,8 +319,8 @@
             ll_assert(hdr.tid & GCFLAGS == GCFLAGS,
                       "stm_write: bogus flags on source object")
             #
-            # Remove the GCFLAG_GLOBAL from the copy
-            localhdr.tid = hdr.tid & ~GCFLAG_GLOBAL
+            # Remove the GCFLAG_GLOBAL from the copy, and add GCFLAG_VISITED
+            localhdr.tid = hdr.tid + (GCFLAG_VISITED - GCFLAG_GLOBAL)
             #
             # Set the 'version' field of the local copy to be a pointer
             # to the global obj.  (The field is called 'version' because
@@ -357,35 +357,43 @@
         """Implement the common logic of id() and identityhash()
         of an object, given as a GCREF.
         """
-        raise NotImplementedError
         obj = llmemory.cast_ptr_to_adr(gcobj)
         hdr = self.header(obj)
-        #
-        if hdr.tid & GCFLAG_GLOBAL == 0:
+        tls = self.get_tls()
+        if tls.is_in_nursery(obj):
             #
-            # The object is a local object.  Find or allocate a corresponding
-            # global object.
-            if hdr.tid & (GCFLAG_WAS_COPIED | GCFLAG_HAS_SHADOW) == 0:
+            # The object is still in the nursery of the current TLS.
+            # (It cannot be in the nursery of a different thread, because
+            # such objects are not visible to different threads at all.)
+            #
+            ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0, "id: WAS_COPIED?")
+            #
+            if hdr.tid & GCFLAG_HAS_SHADOW == 0:
                 #
                 # We need to allocate a global object here.  We only allocate
                 # it for now; it is left completely uninitialized.
+                size_gc_header = self.gcheaderbuilder.size_gc_header
                 size = self.get_size(obj)
-                tls = self.collector.get_tls()
-                globalobj = self._malloc_global_raw(tls, size)
-                self.header(globalobj).tid = GCFLAG_GLOBAL
+                totalsize = size_gc_header + size
+                fixedobj = tls.sharedarea_tls.malloc_object(totalsize)
+                tls.sharedarea_tls.add_regular(fixedobj)
+                self.header(fixedobj).tid = 0     # GCFLAG_VISITED is off
                 #
                 # Update the header of the local 'obj'
                 hdr.tid |= GCFLAG_HAS_SHADOW
-                hdr.version = globalobj
+                hdr.version = fixedobj
                 #
             else:
-                # There is already a corresponding globalobj
-                globalobj = hdr.version
+                # There is already a corresponding fixedobj
+                fixedobj = hdr.version
             #
-            obj = globalobj
+            obj = fixedobj
+            #
+        elif hdr.tid & (GCFLAG_GLOBAL|GCFLAG_WAS_COPIED) == GCFLAG_WAS_COPIED:
+            #
+            # The object is the local copy of a LOCAL-GLOBAL pair.
+            obj = hdr.version
         #
-        ll_assert(self.header(obj).tid & GCFLAG_GLOBAL != 0,
-                  "id_or_identityhash: unexpected local object")
         i = llmemory.cast_adr_to_int(obj)
         if is_hash:
             # For identityhash(), we need a special case for some
diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -25,7 +25,6 @@
         self.gc = sharedarea.gc
         self.sharedarea = sharedarea
         self.chained_list = NULL
-        self.special_stack = self.gc.AddressStack()
 
     def malloc_object(self, totalsize):
         """Malloc.  You must also call add_regular() or add_special() later."""
@@ -40,10 +39,6 @@
         hdr.version = self.chained_list
         self.chained_list = obj
 
-    def add_special(self, obj):
-        """After malloc_object(), register the object in a separate stack."""
-        self.special_stack.append(obj)
-
     def free_object(self, adr2):
         adr1 = adr2 - self.gc.gcheaderbuilder.size_gc_header
         llarena.arena_free(llarena.getfakearenaaddress(adr1))
@@ -55,10 +50,6 @@
             next = self.gc.header(obj).version
             self.free_object(obj)
             obj = next
-        s = self.special_stack
-        while s.non_empty():
-            self.free_object(s.pop())
 
     def delete(self):
-        self.special_stack.delete()
         free_non_gc_object(self)
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -9,7 +9,7 @@
 from pypy.rpython.memory.gc.stmgc import WORD, NULL
 from pypy.rpython.memory.gc.stmgc import always_inline, dont_inline
 from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED
-from pypy.rpython.memory.gc.stmgc import GCFLAG_WAS_COPIED
+from pypy.rpython.memory.gc.stmgc import GCFLAG_WAS_COPIED, GCFLAG_HAS_SHADOW
 
 
 class StmGCTLS(object):
@@ -49,12 +49,16 @@
         # --- a thread-local allocator for the shared area
         from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
         self.sharedarea_tls = StmGCThreadLocalAllocator(self.gc.sharedarea)
+        # --- the LOCAL objects with GCFLAG_WAS_COPIED
+        self.copied_local_objects = self.AddressStack()
         #
         self._register_with_C_code()
 
     def teardown_thread(self):
         self._cleanup_state()
         self._unregister_with_C_code()
+        self.copied_local_objects.delete()
+        self.sharedarea_tls.delete()
         self._free_nursery(self.nursery_start)
         free_non_gc_object(self)
 
@@ -241,7 +245,7 @@
         """Allocate an object that will be used as a LOCAL copy of
         some GLOBAL object."""
         localobj = self.sharedarea_tls.malloc_object(totalsize)
-        self.sharedarea_tls.add_special(localobj)
+        self.copied_local_objects.append(localobj)
         return localobj
 
     # ------------------------------------------------------------
@@ -264,9 +268,18 @@
         #if self.rawmalloced_objects:
         #    xxx     # free the rawmalloced_objects still around
 
-        # if we still have a StmGCThreadLocalAllocator, free the old unused
-        # local objects it still contains
+        # free the old unused local objects still allocated in the
+        # StmGCThreadLocalAllocator
         self.sharedarea_tls.clear()
+        # free the local copies.  Note that commonly, they are leftovers
+        # from the previous transaction running in this thread.  The C code
+        # has just copied them over the corresponding GLOBAL objects at the
+        # very end of that transaction.
+        self._free_and_clear_list(self.copied_local_objects)
+
+    def _free_and_clear_list(self, lst):
+        while lst.non_empty():
+            self.sharedarea_tls.free_object(lst.pop())
 
 
     def collect_roots_from_stack(self):
@@ -293,40 +306,58 @@
         obj = root.address[0]
         hdr = self.gc.header(obj)
         #
-        # If 'obj' is a LOCAL copy of a GLOBAL object, skip it
-        # (this case is handled differently in collect_roots_from_tldict)
-        if hdr.tid & GCFLAG_WAS_COPIED:
-            return
-        #
         # If 'obj' is not in the nursery, we set GCFLAG_VISITED
         if not self.is_in_nursery(obj):
             if hdr.tid & GCFLAG_VISITED == 0:
+                ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0,
+                          "local GCFLAG_WAS_COPIED without GCFLAG_VISITED")
                 hdr.tid |= GCFLAG_VISITED
                 self.pending.append(obj)
             return
         #
         # If 'obj' was already forwarded, change it to its forwarding address.
-        if hdr.tid & GCFLAG_VISITED:
-            root.address[0] = hdr.version
-            return
-        #
-        # First visit to 'obj': we must move this YOUNG obj out of the nursery.
-        size_gc_header = self.gc.gcheaderbuilder.size_gc_header
-        size = self.gc.get_size(obj)
-        totalsize = size_gc_header + size
-        #
-        # Common case: allocate a new nonmovable location for it.
-        newobj = self._malloc_out_of_nursery(totalsize)
-        #
-        # Copy it.  Note that references to other objects in the
-        # nursery are kept unchanged in this step.
-        llmemory.raw_memcopy(obj - size_gc_header,
-                             newobj - size_gc_header,
-                             totalsize)
-        #
-        # Register the object here, not before the memcopy() that would
-        # overwrite its 'version' field
-        self._register_newly_malloced_obj(newobj)
+        # If 'obj' has already a shadow but isn't forwarded so far, use it.
+        if hdr.tid & (GCFLAG_VISITED | GCFLAG_HAS_SHADOW):
+            #
+            if hdr.tid & GCFLAG_VISITED:
+                root.address[0] = hdr.version
+                return
+            #
+            # Case of GCFLAG_HAS_SHADOW.  See comments below.
+            size_gc_header = self.gc.gcheaderbuilder.size_gc_header
+            size = self.gc.get_size(obj)
+            totalsize = size_gc_header + size
+            hdr.tid &= ~GCFLAG_HAS_SHADOW
+            newobj = hdr.version
+            newhdr = self.gc.header(newobj)
+            #
+            saved_version = newhdr.version
+            llmemory.raw_memcopy(obj - size_gc_header,
+                                 newobj - size_gc_header,
+                                 totalsize)
+            newhdr.version = saved_version
+            newhdr.tid = hdr.tid | GCFLAG_VISITED
+            #
+        else:
+            #
+            # First visit to 'obj': we must move this YOUNG obj out of the
+            # nursery.
+            size_gc_header = self.gc.gcheaderbuilder.size_gc_header
+            size = self.gc.get_size(obj)
+            totalsize = size_gc_header + size
+            #
+            # Common case: allocate a new nonmovable location for it.
+            newobj = self._malloc_out_of_nursery(totalsize)
+            #
+            # Copy it.  Note that references to other objects in the
+            # nursery are kept unchanged in this step.
+            llmemory.raw_memcopy(obj - size_gc_header,
+                                 newobj - size_gc_header,
+                                 totalsize)
+            #
+            # Register the object here, not before the memcopy() that would
+            # overwrite its 'version' field
+            self._register_newly_malloced_obj(newobj)
         #
         # Set the YOUNG copy's GCFLAG_VISITED and set its version to
         # point to the OLD copy.
@@ -366,6 +397,8 @@
                   "in a root: unexpected GCFLAG_GLOBAL")
         ll_assert(localhdr.tid & GCFLAG_WAS_COPIED != 0,
                   "in a root: missing GCFLAG_WAS_COPIED")
+        ll_assert(localhdr.tid & GCFLAG_VISITED != 0,
+                  "in a root: missing GCFLAG_VISITED")
         #
         self.trace_and_drag_out_of_nursery(localobj)
 
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -167,14 +167,21 @@
     def malloc(self, STRUCT, weakref=False, globl='auto'):
         size = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT))
         tid = lltype.cast_primitive(llgroup.HALFWORD, 123)
-        gcref = self.gc.malloc_fixedsize_clear(tid, size,
-                                               contains_weakptr=weakref)
-        realobj = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), gcref)
-        addr = llmemory.cast_ptr_to_adr(realobj)
         if globl == 'auto':
             globl = (self.gc.stm_operations.threadnum == 0)
         if globl:
-            self.gc.header(addr).tid |= GCFLAG_GLOBAL
+            totalsize = self.gc.gcheaderbuilder.size_gc_header + size
+            adr1 = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize),
+                                        1)
+            llarena.arena_reserve(adr1, totalsize)
+            addr = adr1 + self.gc.gcheaderbuilder.size_gc_header
+            self.gc.header(addr).tid = GCFLAG_GLOBAL
+            realobj = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(STRUCT))
+        else:
+            gcref = self.gc.malloc_fixedsize_clear(tid, size,
+                                                   contains_weakptr=weakref)
+            realobj = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), gcref)
+            addr = llmemory.cast_ptr_to_adr(realobj)
         return realobj, addr
     def select_thread(self, threadnum):
         self.gc.stm_operations.threadnum = threadnum
@@ -340,6 +347,7 @@
         assert main_tls.nursery_free == main_tls.nursery_start   # empty
 
     def test_commit_transaction_no_references(self):
+        py.test.skip("rewrite me")
         s, s_adr = self.malloc(S)
         s.b = 12345
         self.select_thread(1)
@@ -448,6 +456,7 @@
         s, s_adr = self.malloc(S)
         self.select_thread(1)
         t_adr = self.gc.stm_writebarrier(s_adr)   # make a local copy
+        assert t_adr != s_adr
         t = llmemory.cast_adr_to_ptr(t_adr, llmemory.GCREF)
         i = self.gc.id(t)
         assert i == llmemory.cast_adr_to_int(s_adr)
@@ -465,8 +474,11 @@
 
     def test_id_of_local_surviving(self):
         sr1, sr1_adr = self.malloc(SR)
+        assert sr1.s1 == lltype.nullptr(S)
+        assert sr1.sr2 == lltype.nullptr(SR)
         self.select_thread(1)
         t2, t2_adr = self.malloc(S)
+        t2.a = 423
         tr1_adr = self.gc.stm_writebarrier(sr1_adr)
         assert tr1_adr != sr1_adr
         tr1 = llmemory.cast_adr_to_ptr(tr1_adr, lltype.Ptr(SR))
@@ -478,7 +490,7 @@
         assert i == self.gc.id(t2)
         self.gc.commit_transaction()
         s2 = tr1.s1       # tr1 is a root, so not copied yet
-        assert s2 and s2 != t2
+        assert s2 and s2.a == 423 and s2._obj0 != t2._obj0
         assert self.gc.id(s2) == i
 
     def test_hash_of_global(self):
@@ -509,6 +521,7 @@
         sr1, sr1_adr = self.malloc(SR)
         self.select_thread(1)
         t2, t2_adr = self.malloc(S)
+        t2.a = 424
         tr1_adr = self.gc.stm_writebarrier(sr1_adr)
         assert tr1_adr != sr1_adr
         tr1 = llmemory.cast_adr_to_ptr(tr1_adr, lltype.Ptr(SR))
@@ -521,7 +534,7 @@
         assert i == self.gc.identityhash(t2)
         self.gc.commit_transaction()
         s2 = tr1.s1       # tr1 is a root, so not copied yet
-        assert s2 and s2 != t2
+        assert s2 and s2.a == 424 and s2._obj0 != t2._obj0
         assert self.gc.identityhash(s2) == i
 
     def test_weakref_to_global(self):

From noreply at buildbot.pypy.org  Mon Apr 16 18:05:07 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 18:05:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Disable a couple of extra features when
	translating with
Message-ID: <20120416160507.45996820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54430:09a1f054f29a
Date: 2012-04-16 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/09a1f054f29a/

Log:	Disable a couple of extra features when translating with
	--no-translation-rweakref.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 

From noreply at buildbot.pypy.org  Mon Apr 16 18:05:08 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 18:05:08 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Disable a couple of extra features when
	translating with
Message-ID: <20120416160508.93068820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54431:3ab81c92db19
Date: 2012-04-16 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/3ab81c92db19/

Log:	Disable a couple of extra features when translating with
	--no-translation-rweakref.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 

From noreply at buildbot.pypy.org  Mon Apr 16 18:05:09 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 18:05:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120416160509.E7EF2820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54432:b11776235fa9
Date: 2012-04-16 18:04 +0200
http://bitbucket.org/pypy/pypy/changeset/b11776235fa9/

Log:	merge heads

diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,9 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +18,17 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +40,7 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
     includes = includes,
     export_symbols = [],
     post_include_bits = [

From noreply at buildbot.pypy.org  Mon Apr 16 18:07:58 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 18:07:58 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: the usual wrap/wrapbytes confusion
Message-ID: <20120416160758.B26BB820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54433:aaf9d714037c
Date: 2012-04-16 17:32 +0200
http://bitbucket.org/pypy/pypy/changeset/aaf9d714037c/

Log:	the usual wrap/wrapbytes confusion

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -533,7 +533,7 @@
         cbuf = self._charbuf_start()
         s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
         self._charbuf_stop()
-        return self.space.wrap(s)
+        return self.space.wrapbytes(s)
 
     def array_tostring__Array(space, self):
         space.warn("tostring() is deprecated. Use tobytes() instead.",

From noreply at buildbot.pypy.org  Mon Apr 16 18:08:00 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 18:08:00 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix syntax
Message-ID: <20120416160800.0E98D820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54434:abe868f1ea64
Date: 2012-04-16 17:33 +0200
http://bitbucket.org/pypy/pypy/changeset/abe868f1ea64/

Log:	fix syntax

diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -30,7 +30,7 @@
 
         a = self.array('u')
         raises(TypeError, a.append, 7)
-        raises(TypeError, a.append, u'hi')
+        raises(TypeError, a.append, 'hi')
         a.append('h')
         assert a[0] == 'h'
         assert type(a[0]) is str

From noreply at buildbot.pypy.org  Mon Apr 16 18:08:01 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 18:08:01 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: the long type has gone
Message-ID: <20120416160801.50BC7820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54435:a00aeda79bbc
Date: 2012-04-16 18:03 +0200
http://bitbucket.org/pypy/pypy/changeset/a00aeda79bbc/

Log:	the long type has gone

diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -64,9 +64,9 @@
                            ('h', (-32768, 30535, 32767),  int),
                            ('H', (     0, 56783, 65535),  int),
                            ('i', (-32768, 30535, 32767),  int),
-                           ('I', (     0, 56783, 65535), long),
+                           ('I', (     0, 56783, 65535), int),
                            ('l', (-2 ** 32 // 2, 34, 2 ** 32 // 2 - 1),  int),
-                           ('L', (0, 3523532, 2 ** 32 - 1), long),
+                           ('L', (0, 3523532, 2 ** 32 - 1), int),
                            ):
             a = self.array(tc, ok)
             assert len(a) == len(ok)

From noreply at buildbot.pypy.org  Mon Apr 16 18:08:02 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 18:08:02 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: tweak the test to work in this branch
Message-ID: <20120416160802.961E8820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54436:65e51b061a32
Date: 2012-04-16 18:06 +0200
http://bitbucket.org/pypy/pypy/changeset/65e51b061a32/

Log:	tweak the test to work in this branch

diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py
--- a/pypy/objspace/std/test/test_setobject.py
+++ b/pypy/objspace/std/test/test_setobject.py
@@ -927,10 +927,12 @@
         assert sorted(lst) == [1, 3, 5]
 
     def test_iter_set_strategy_only_change_2(self):
-        s = set([1, 3, 5])
+        # on py3k the IntStrategy doesn't work yet. So, we use the
+        # StringSetStrategy for this test
+        s = set(['1', '3', '5'])
         it = iter(s)
-        s.add('foo')
-        s.remove(1)
+        s.add(42) # change the strategy
+        s.remove('1')
         # 's' is still length 3, but its strategy changed.  we are
         # getting a RuntimeError because iterating over the old storage
         # gives us 1, but 1 is not in the set any longer.

From noreply at buildbot.pypy.org  Mon Apr 16 18:08:03 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Mon, 16 Apr 2012 18:08:03 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: kill this test,
 it does not make sense now that we have string vs bytes
Message-ID: <20120416160803.D9E78820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54437:16ccffcad1fb
Date: 2012-04-16 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/16ccffcad1fb/

Log:	kill this test, it does not make sense now that we have string vs
	bytes

diff --git a/pypy/objspace/std/test/test_builtinshortcut.py b/pypy/objspace/std/test/test_builtinshortcut.py
--- a/pypy/objspace/std/test/test_builtinshortcut.py
+++ b/pypy/objspace/std/test/test_builtinshortcut.py
@@ -60,24 +60,6 @@
         x += 6.5
         assert x == 'foo'
         assert 5 + 6.5 == 11.5
-
-    def test_unicode_string_compares(self):
-        assert u'a' == 'a'
-        assert 'a' == u'a'
-        assert not u'a' == 'b'
-        assert not 'a'  == u'b'
-        assert u'a' != 'b'
-        assert 'a'  != u'b'
-        assert not (u'a' == 5)
-        assert u'a' != 5
-        assert u'a' < 5 or u'a' > 5
-
-        s = chr(128)
-        u = unichr(128)
-        assert not s == u # UnicodeWarning
-        assert s != u
-        assert not u == s
-        assert u != s
    
 
 class AppTestSet(test_setobject.AppTestAppSetTest):

From noreply at buildbot.pypy.org  Mon Apr 16 18:40:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 18:40:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Disable autoflushing if we are
 translated without weakref support.
Message-ID: <20120416164000.AC3FD82F4F@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54439:922cce43ab51
Date: 2012-04-16 18:37 +0200
http://bitbucket.org/pypy/pypy/changeset/922cce43ab51/

Log:	Disable autoflushing if we are translated without weakref support.

diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder

From noreply at buildbot.pypy.org  Mon Apr 16 18:39:59 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 18:39:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a way to know if the program can
	use weakrefs.
Message-ID: <20120416163959.6E62C820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54438:e36a83af4ecb
Date: 2012-04-16 18:35 +0200
http://bitbucket.org/pypy/pypy/changeset/e36a83af4ecb/

Log:	Add a way to know if the program can use weakrefs.

diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False

From noreply at buildbot.pypy.org  Mon Apr 16 20:34:45 2012
From: noreply at buildbot.pypy.org (taavi_burns)
Date: Mon, 16 Apr 2012 20:34:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge in more ufuncs
Message-ID: <20120416183445.E1F93820B3@wyvern.cs.uni-duesseldorf.de>

Author: Taavi Burns <taavi.burns at gmail.com>
Branch: 
Changeset: r54440:8ae7413e7b32
Date: 2012-04-16 14:33 -0400
http://bitbucket.org/pypy/pypy/changeset/8ae7413e7b32/

Log:	Merge in more ufuncs

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -762,6 +762,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -772,7 +774,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -785,6 +788,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -796,7 +801,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True

From noreply at buildbot.pypy.org  Mon Apr 16 21:14:06 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon, 16 Apr 2012 21:14:06 +0200 (CEST)
Subject: [pypy-commit] pypy build-external: close dead build-external branch
Message-ID: <20120416191406.E30D3820B3@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: build-external
Changeset: r54441:aefc282c703d
Date: 2012-04-16 21:13 +0200
http://bitbucket.org/pypy/pypy/changeset/aefc282c703d/

Log:	close dead build-external branch


From noreply at buildbot.pypy.org  Mon Apr 16 21:23:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 21:23:00 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: More asserts
Message-ID: <20120416192300.4A25A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54442:d5653c635122
Date: 2012-04-16 21:18 +0200
http://bitbucket.org/pypy/pypy/changeset/d5653c635122/

Log:	More asserts

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -399,6 +399,18 @@
                   "in a root: missing GCFLAG_WAS_COPIED")
         ll_assert(localhdr.tid & GCFLAG_VISITED != 0,
                   "in a root: missing GCFLAG_VISITED")
+        globalhdr = self.gc.header(globalobj)
+        ll_assert(globalhdr.tid & GCFLAG_GLOBAL != 0,
+                  "in a root: GLOBAL: missing GCFLAG_GLOBAL")
+        ll_assert(globalhdr.tid & GCFLAG_WAS_COPIED != 0,
+                  "in a root: GLOBAL: missing GCFLAG_WAS_COPIED")
+        ll_assert(globalhdr.tid & GCFLAG_VISITED == 0,
+                  "in a root: GLOBAL: unexpected GCFLAG_VISITED")
+        TL = lltype.cast_primitive(lltype.Signed,
+                                   self.gc.get_type_id(localobj))
+        TG = lltype.cast_primitive(lltype.Signed,
+                                   self.gc.get_type_id(globalobj))
+        ll_assert(TL == TG, "in a root: type(LOCAL) != type(GLOBAL)")
         #
         self.trace_and_drag_out_of_nursery(localobj)
 

From noreply at buildbot.pypy.org  Mon Apr 16 21:23:01 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 21:23:01 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: A failing test
Message-ID: <20120416192301.84E29820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54443:ddbb298b6971
Date: 2012-04-16 21:18 +0200
http://bitbucket.org/pypy/pypy/changeset/ddbb298b6971/

Log:	A failing test

diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -175,7 +175,7 @@
                                         1)
             llarena.arena_reserve(adr1, totalsize)
             addr = adr1 + self.gc.gcheaderbuilder.size_gc_header
-            self.gc.header(addr).tid = GCFLAG_GLOBAL
+            self.gc.header(addr).tid = self.gc.combine(tid, GCFLAG_GLOBAL)
             realobj = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(STRUCT))
         else:
             gcref = self.gc.malloc_fixedsize_clear(tid, size,
@@ -346,6 +346,17 @@
         main_tls = self.gc.main_thread_tls
         assert main_tls.nursery_free == main_tls.nursery_start   # empty
 
+    def test_commit_tldict_entry_with_global_references(self):
+        t, t_adr = self.malloc(S)
+        tr, tr_adr = self.malloc(SR)
+        tr.s1 = t
+        self.select_thread(1)
+        sr_adr = self.gc.stm_writebarrier(tr_adr)
+        assert sr_adr != tr_adr
+        s_adr = self.gc.stm_writebarrier(t_adr)
+        assert s_adr != t_adr
+        self.gc.commit_transaction()
+
     def test_commit_transaction_no_references(self):
         py.test.skip("rewrite me")
         s, s_adr = self.malloc(S)

From noreply at buildbot.pypy.org  Mon Apr 16 21:23:02 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 16 Apr 2012 21:23:02 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix the test.
Message-ID: <20120416192302.BBB00820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54444:39116e9494e8
Date: 2012-04-16 21:21 +0200
http://bitbucket.org/pypy/pypy/changeset/39116e9494e8/

Log:	Fix the test.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -297,6 +297,14 @@
     def _trace_drag_out1(self, root):
         self._trace_drag_out(root, None)
 
+    def _trace_drag_out_if_not_global(self, root, ignored):
+        # like _trace_drag_out(), but ignores references to GLOBAL objects.
+        # used only for the LOCAL copy of a GLOBAL object, which may still
+        # have further GLOBAL pointers.
+        obj = root.address[0]
+        if self.gc.header(obj).tid & GCFLAG_GLOBAL == 0:
+            self._trace_drag_out(root, ignored)
+
     def _trace_drag_out(self, root, ignored):
         """Trace callback: 'root' is the address of some pointer.  If that
         pointer points to a YOUNG object, allocate an OLD copy of it and
@@ -412,7 +420,7 @@
                                    self.gc.get_type_id(globalobj))
         ll_assert(TL == TG, "in a root: type(LOCAL) != type(GLOBAL)")
         #
-        self.trace_and_drag_out_of_nursery(localobj)
+        self.gc.trace(localobj, self._trace_drag_out_if_not_global, None)
 
     def collect_flush_pending(self):
         # Follow the objects in the 'pending' stack and move the

From noreply at buildbot.pypy.org  Mon Apr 16 21:35:00 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 16 Apr 2012 21:35:00 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: rework get_osfhandle,
	begin review (amaury_)
Message-ID: <20120416193500.E0032820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54445:390a3cc7e556
Date: 2012-04-16 22:34 +0300
http://bitbucket.org/pypy/pypy/changeset/390a3cc7e556/

Log:	rework get_osfhandle, begin review (amaury_)

diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -740,9 +740,6 @@
         # to 'anonymously' map memory.
         if fileno != -1 and fileno != 0:
             fh = rwin32.get_osfhandle(fileno)
-            if fh == INVALID_HANDLE:
-                errno = rposix.get_errno()
-                raise OSError(errno, os.strerror(errno))
             # Win9x appears to need us seeked to zero
             # SEEK_SET = 0
             # libc._lseek(fileno, 0, SEEK_SET)
diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -97,17 +97,9 @@
     _set_errno(rffi.cast(INT, errno))
 
 if os.name == 'nt':
-    def validate_fd_emulator(fd):
-        try:
-            os.fstat(fd)
-            return 1
-        except:
-            return 0
-
     validate_fd = rffi.llexternal(
         "_PyVerify_fd", [rffi.INT], rffi.INT,
-        _callable=validate_fd_emulator, compilation_info=eci,
-        _nowrapper=True, elidable_function=False, sandboxsafe=False,
+        compilation_info=eci,
         )
 else:
     def validate_fd(fd):
diff --git a/pypy/rlib/rwin32.py b/pypy/rlib/rwin32.py
--- a/pypy/rlib/rwin32.py
+++ b/pypy/rlib/rwin32.py
@@ -128,9 +128,12 @@
     _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE)
 
     def get_osfhandle(fd):
-        if not validate_fd(fileno):
-            raise OSError(GetLastError(), 'Bad file descriptor')
-        return _get_osfhandle(fd)
+        if not validate_fd(fd):
+            raise WindowsError(errno.EBADF, 'Bad file descriptor')
+        handle = _get_osfhandle(fd)
+        if handle == INVALID_HANDLE_VALUE:
+            raise WindowsError(errno.EBADF, "Invalid file handle")
+        return handle
 
     def build_winerror_to_errno():
         """Build a dictionary mapping windows error numbers to POSIX errno.
diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -175,15 +175,14 @@
 
 
 if sys.platform == "win32":
-    from pypy.rlib import rwin32
+    from pypy.rlib.rwin32 import BOOL, HANDLE, get_osfhandle, GetLastError
     from pypy.translator.tool.cbuild import ExternalCompilationInfo
     from pypy.rpython.lltypesystem import rffi
-    import errno
 
     _eci = ExternalCompilationInfo()
     _setmode = rffi.llexternal('_setmode', [rffi.INT, rffi.INT], rffi.INT,
                                compilation_info=_eci)
-    SetEndOfFile = rffi.llexternal('SetEndOfFile', [rffi.LONG], rwin32.BOOL,
+    SetEndOfFile = rffi.llexternal('SetEndOfFile', [HANDLE], BOOL,
                                    compilation_info=_eci)
 
     # HACK: These implementations are specific to MSVCRT and the C backend.
@@ -198,11 +197,9 @@
             # move to the position to be truncated
             os.lseek(fd, size, 0)
             # Truncate.  Note that this may grow the file!
-            handle = rwin32.get_osfhandle(fd)
-            if handle == -1:
-                raise OSError(errno.EBADF, "Invalid file handle")
+            handle = get_osfhandle(fd)
             if not SetEndOfFile(handle):
-                raise WindowsError(rwin32.GetLastError(),
+                raise WindowsError(GetLastError(),
                                    "Could not truncate file")
         finally:
             # we restore the file pointer position in any case
diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -403,7 +403,6 @@
 
     def win32_fstat_llimpl(fd):
         handle = rwin32.get_osfhandle(fd)
-
         filetype = win32traits.GetFileType(handle)
         if filetype == win32traits.FILE_TYPE_CHAR:
             # console or LPT device

From noreply at buildbot.pypy.org  Mon Apr 16 23:29:59 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 16 Apr 2012 23:29:59 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: more cleanup of validate_fd
	(amaury_)
Message-ID: <20120416212959.CE256820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54446:dba9f63cc754
Date: 2012-04-17 00:29 +0300
http://bitbucket.org/pypy/pypy/changeset/dba9f63cc754/

Log:	more cleanup of validate_fd (amaury_)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -26,7 +26,7 @@
 from pypy.module.__builtin__.interp_classobj import W_ClassObject
 from pypy.module.__builtin__.interp_memoryview import W_MemoryView
 from pypy.rlib.entrypoint import entrypoint
-from pypy.rlib.rposix import validate_fd
+from pypy.rlib.rposix import _validate_fd
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import specialize
 from pypy.rlib.exports import export_struct
@@ -90,7 +90,7 @@
 
 _fclose = rffi.llexternal('fclose', [FILEP], rffi.INT)
 def fclose(fp):
-    if not validate_fd(fileno(fp)):
+    if not _validate_fd(fileno(fp)):
         return -1
     return _fclose(fp)
 
diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -5,6 +5,7 @@
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.rarithmetic import intmask
 from pypy.rlib.objectmodel import specialize
+from pypy.rlib import jit
 
 class CConstantErrno(CConstant):
     # these accessors are used when calling get_errno() or set_errno()
@@ -97,10 +98,14 @@
     _set_errno(rffi.cast(INT, errno))
 
 if os.name == 'nt':
-    validate_fd = rffi.llexternal(
+    _validate_fd = rffi.llexternal(
         "_PyVerify_fd", [rffi.INT], rffi.INT,
         compilation_info=eci,
         )
+    @jit.dont_look_inside
+    def validate_fd(fd):
+        if not _validate_fd(fd):
+            raise OSError(get_errno(), 'Bad file descriptor')
 else:
     def validate_fd(fd):
         return 1
@@ -109,7 +114,7 @@
     # this behaves like os.closerange() from Python 2.6.
     for fd in xrange(fd_low, fd_high):
         try:
-            if validate_fd(fd):
+            if _validate_fd(fd):
                 os.close(fd)
         except OSError:
             pass
diff --git a/pypy/rlib/rwin32.py b/pypy/rlib/rwin32.py
--- a/pypy/rlib/rwin32.py
+++ b/pypy/rlib/rwin32.py
@@ -128,8 +128,7 @@
     _get_osfhandle = rffi.llexternal('_get_osfhandle', [rffi.INT], HANDLE)
 
     def get_osfhandle(fd):
-        if not validate_fd(fd):
-            raise WindowsError(errno.EBADF, 'Bad file descriptor')
+        validate_fd(fd)
         handle = _get_osfhandle(fd)
         if handle == INVALID_HANDLE_VALUE:
             raise WindowsError(errno.EBADF, "Invalid file handle")
diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -133,11 +133,11 @@
                 pass
 
     def test_validate_fd(self):
-        assert rposix.validate_fd(0) == 1
+        assert rposix._validate_fd(0) == 1
         fid = open(str(udir.join('validate_test.txt')), 'w')
         fd = fid.fileno()
-        assert rposix.validate_fd(fd) == 1
+        assert rposix._validate_fd(fd) == 1
         fid.close()
-        assert rposix.validate_fd(fd) == 0
+        assert rposix._validate_fd(fd) == 0
 
 
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -916,8 +916,7 @@
 
         def os_write_llimpl(fd, data):
             count = len(data)
-            if not rposix.validate_fd(fd):
-                raise OSError(rposix.get_errno(), 'Bad file descriptor')
+            rposix.validate_fd(fd)
             buf = rffi.get_nonmovingbuffer(data)
             try:
                 written = rffi.cast(lltype.Signed, os_write(
@@ -942,8 +941,7 @@
                                    rffi.INT, threadsafe=False)
         
         def close_llimpl(fd):
-            if not rposix.validate_fd(fd):
-                raise OSError(rposix.get_errno(), 'Bad file descriptor')
+            rposix.validate_fd(fd)
             error = rffi.cast(lltype.Signed, os_close(rffi.cast(rffi.INT, fd)))
             if error == -1:
                 raise OSError(rposix.get_errno(), "close failed")
@@ -979,8 +977,7 @@
                                    rffi.LONGLONG)
 
         def lseek_llimpl(fd, pos, how):
-            if not rposix.validate_fd(fd):
-                raise OSError(rposix.get_errno(), 'Bad file descriptor')
+            rposix.validate_fd(fd)
             how = fix_seek_arg(how)
             res = os_lseek(rffi.cast(rffi.INT,      fd),
                            rffi.cast(rffi.LONGLONG, pos),
@@ -1006,8 +1003,7 @@
                                        [rffi.INT, rffi.LONGLONG], rffi.INT)
 
         def ftruncate_llimpl(fd, length):
-            if not rposix.validate_fd(fd):
-                raise OSError(rposix.get_errno(), 'Bad file descriptor')
+            rposix.validate_fd(fd)
             res = rffi.cast(rffi.LONG,
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
@@ -1026,8 +1022,7 @@
             os_fsync = self.llexternal('_commit', [rffi.INT], rffi.INT)
 
         def fsync_llimpl(fd):
-            if not rposix.validate_fd(fd):
-                raise OSError(rposix.get_errno(), 'Bad file descriptor')
+            rposix.validate_fd(fd)
             res = rffi.cast(rffi.SIGNED, os_fsync(rffi.cast(rffi.INT, fd)))
             if res < 0:
                 raise OSError(rposix.get_errno(), "fsync failed")
@@ -1040,8 +1035,7 @@
         os_fdatasync = self.llexternal('fdatasync', [rffi.INT], rffi.INT)
 
         def fdatasync_llimpl(fd):
-            if not rposix.validate_fd(fd):
-                raise OSError(rposix.get_errno(), 'Bad file descriptor')
+            rposix.validate_fd(fd)
             res = rffi.cast(rffi.SIGNED, os_fdatasync(rffi.cast(rffi.INT, fd)))
             if res < 0:
                 raise OSError(rposix.get_errno(), "fdatasync failed")

From noreply at buildbot.pypy.org  Tue Apr 17 06:01:27 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Apr 2012 06:01:27 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: small change,
 unsure what to do about tuples and cpyext
Message-ID: <20120417040127.42F5082F4F@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54448:cf002eb38eeb
Date: 2012-04-17 00:01 -0400
http://bitbucket.org/pypy/pypy/changeset/cf002eb38eeb/

Log:	small change, unsure what to do about tuples and cpyext

diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -1,11 +1,13 @@
 from pypy.interpreter.error import OperationError
-from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL,
                                     build_type_checkers)
+from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef,
     borrow_from, make_ref, from_ref)
-from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
+from pypy.rlib.rerased_raw import INSTANCE
+from pypy.rpython.lltypesystem import rffi, lltype
+
 
 PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple")
 
@@ -29,7 +31,7 @@
     # are also other implementations of tuples.
     from pypy.objspace.std.tupletype import store_obj
     assert isinstance(w_t, W_TupleObject)
-    store_obj(space, w_t.tuplestorage, pos, w_obj)
+    store_obj(space, w_t.tuplestorage, INSTANCE, pos, w_obj)
 
 @cpython_api([PyObject, Py_ssize_t], PyObject)
 def PyTuple_GetItem(space, w_t, pos):

From noreply at buildbot.pypy.org  Tue Apr 17 06:01:26 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Apr 2012 06:01:26 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: merged default in
Message-ID: <20120417040126.06C16820B3@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54447:0a24a858507f
Date: 2012-04-16 23:57 -0400
http://bitbucket.org/pypy/pypy/changeset/0a24a858507f/

Log:	merged default in

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return pyc
+def pytest_unconfigure(config):
+    hook = config._assertstate.hook
+    if hook is not None:
+        sys.meta_path.remove(hook)
 
-def before_module_import(mod):
-    if mod.config._assertstate.mode != "on":
-        return
-    # Some deep magic: load the source, rewrite the asserts, and write a
-    # fake pyc, so that it'll be loaded when the module is imported.
-    source = mod.fspath.read()
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
-        return
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, str(mod.fspath), "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the assertion
-        # rewriting, but I don't know of a fast way to tell.
-        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
-        return
-    mod._pyc = _write_pyc(co, mod.fspath)
-    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+def pytest_collection(session):
+    # this hook is only called when test modules are collected
+    # so for example not in the master process of pytest-xdist
+    # (which does not collect test modules)
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.set_session(session)
 
-def after_module_import(mod):
-    if not hasattr(mod, "_pyc"):
-        return
-    state = mod.config._assertstate
-    try:
-        mod._pyc.remove()
-    except py.error.ENOENT:
-        state.trace("couldn't find pyc: %r" % (mod._pyc,))
-    else:
-        state.trace("removed pyc: %r" % (mod._pyc,))
+def pytest_runtest_setup(item):
+    def callbinrepr(op, left, right):
+        hook_result = item.ihook.pytest_assertrepr_compare(
+            config=item.config, op=op, left=left, right=right)
+        for new_expl in hook_result:
+            if new_expl:
+                res = '\n~'.join(new_expl)
+                if item.config.getvalue("assertmode") == "rewrite":
+                    # The result will be fed back a python % formatting
+                    # operation, which will fail if there are extraneous
+                    # '%'s in the string. Escape them here.
+                    res = res.replace("%", "%%")
+                return res
+    util._reprcompare = callbinrepr
 
-def warn_about_missing_assertion():
+def pytest_runtest_teardown(item):
+    util._reprcompare = None
+
+def pytest_sessionfinish(session):
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.session = None
+
+def _load_modules(mode):
+    """Lazily import assertion related code."""
+    global rewrite, reinterpret
+    from _pytest.assertion import reinterpret
+    if mode == "rewrite":
+        from _pytest.assertion import rewrite
+
+def warn_about_missing_assertion(mode):
     try:
         assert False
     except AssertionError:
         pass
     else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
+        if mode == "rewrite":
+            specifically = ("assertions which are not in test modules "
+                            "will be ignored")
+        else:
+            specifically = "failing tests may report as passing"
+
+        sys.stderr.write("WARNING: " + specifically +
+                        " because assert statements are not executed "
+                        "by the underlying Python interpreter "
+                        "(are you using python -O?)\n")
 
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -53,7 +53,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
+                "compute assert expression before the assert or use --assert=plain)")
 
 def run(offending_line, frame=None):
     if frame is None:
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
@@ -482,7 +481,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --nomagic)")
+                "compute assert expression before the assert or use --assert=plain)")
     else:
         return None
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,14 +1,258 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
+import errno
 import itertools
+import imp
+import marshal
+import os
+import struct
 import sys
+import types
 
 import py
 from _pytest.assertion import util
 
 
+# Windows gives ENOENT in places *nix gives ENOTDIR.
+if sys.platform.startswith("win"):
+    PATH_COMPONENT_NOT_DIR = errno.ENOENT
+else:
+    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
+
+# py.test caches rewritten pycs in __pycache__.
+if hasattr(imp, "get_tag"):
+    PYTEST_TAG = imp.get_tag() + "-PYTEST"
+else:
+    if hasattr(sys, "pypy_version_info"):
+        impl = "pypy"
+    elif sys.platform == "java":
+        impl = "jython"
+    else:
+        impl = "cpython"
+    ver = sys.version_info
+    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
+    del ver, impl
+
+PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
+
+REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+
+class AssertionRewritingHook(object):
+    """Import hook which rewrites asserts."""
+
+    def __init__(self):
+        self.session = None
+        self.modules = {}
+
+    def set_session(self, session):
+        self.fnpats = session.config.getini("python_files")
+        self.session = session
+
+    def find_module(self, name, path=None):
+        if self.session is None:
+            return None
+        sess = self.session
+        state = sess.config._assertstate
+        state.trace("find_module called for: %s" % name)
+        names = name.rsplit(".", 1)
+        lastname = names[-1]
+        pth = None
+        if path is not None and len(path) == 1:
+            pth = path[0]
+        if pth is None:
+            try:
+                fd, fn, desc = imp.find_module(lastname, path)
+            except ImportError:
+                return None
+            if fd is not None:
+                fd.close()
+            tp = desc[2]
+            if tp == imp.PY_COMPILED:
+                if hasattr(imp, "source_from_cache"):
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
+            elif tp != imp.PY_SOURCE:
+                # Don't know what this is.
+                return None
+        else:
+            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
+        fn_pypath = py.path.local(fn)
+        # Is this a test file?
+        if not sess.isinitpath(fn):
+            # We have to be very careful here because imports in this code can
+            # trigger a cycle.
+            self.session = None
+            try:
+                for pat in self.fnpats:
+                    if fn_pypath.fnmatch(pat):
+                        state.trace("matched test file %r" % (fn,))
+                        break
+                else:
+                    return None
+            finally:
+                self.session = sess
+        else:
+            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+        # The requested module looks like a test file, so rewrite it. This is
+        # the most magical part of the process: load the source, rewrite the
+        # asserts, and load the rewritten source. We also cache the rewritten
+        # module code in a special pyc. We must be aware of the possibility of
+        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # tricky race conditions, we maintain the following invariant: The
+        # cached pyc is always a complete, valid pyc. Operations on it must be
+        # atomic. POSIX's atomic rename comes in handy.
+        write = not sys.dont_write_bytecode
+        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
+        if write:
+            try:
+                os.mkdir(cache_dir)
+            except OSError:
+                e = sys.exc_info()[1].errno
+                if e == errno.EEXIST:
+                    # Either the __pycache__ directory already exists (the
+                    # common case) or it's blocked by a non-dir node. In the
+                    # latter case, we'll ignore it in _write_pyc.
+                    pass
+                elif e == PATH_COMPONENT_NOT_DIR:
+                    # One of the path components was not a directory, likely
+                    # because we're in a zip file.
+                    write = False
+                elif e == errno.EACCES:
+                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    write = False
+                else:
+                    raise
+        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
+        pyc = os.path.join(cache_dir, cache_name)
+        # Notice that even if we're in a read-only directory, I'm going to check
+        # for a cached pyc. This may not be optimal...
+        co = _read_pyc(fn_pypath, pyc)
+        if co is None:
+            state.trace("rewriting %r" % (fn,))
+            co = _rewrite_test(state, fn_pypath)
+            if co is None:
+                # Probably a SyntaxError in the test.
+                return None
+            if write:
+                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+        else:
+            state.trace("found cached rewritten pyc for %r" % (fn,))
+        self.modules[name] = co, pyc
+        return self
+
+    def load_module(self, name):
+        co, pyc = self.modules.pop(name)
+        # I wish I could just call imp.load_compiled here, but __file__ has to
+        # be set properly. In Python 3.2+, this all would be handled correctly
+        # by load_compiled.
+        mod = sys.modules[name] = imp.new_module(name)
+        try:
+            mod.__file__ = co.co_filename
+            # Normally, this attribute is 3.2+.
+            mod.__cached__ = pyc
+            py.builtin.exec_(co, mod.__dict__)
+        except:
+            del sys.modules[name]
+            raise
+        return sys.modules[name]
+
+def _write_pyc(co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as (C)Python, since
+    # these "pycs" should never be seen by builtin import. However, there's
+    # little reason deviate, and I hope sometime to be able to use
+    # imp.load_compiled to load them. (See the comment in load_module above.)
+    mtime = int(source_path.mtime())
+    try:
+        fp = open(pyc, "wb")
+    except IOError:
+        err = sys.exc_info()[1].errno
+        if err == PATH_COMPONENT_NOT_DIR:
+            # This happens when we get a EEXIST in find_module creating the
+            # __pycache__ directory and __pycache__ is by some non-dir node.
+            return False
+        raise
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return True
+
+RN = "\r\n".encode("utf-8")
+N = "\n".encode("utf-8")
+
+def _rewrite_test(state, fn):
+    """Try to read and rewrite *fn* and return the code object."""
+    try:
+        source = fn.read("rb")
+    except EnvironmentError:
+        return None
+    # On Python versions which are not 2.7 and less than or equal to 3.1, the
+    # parser expects *nix newlines.
+    if REWRITE_NEWLINES:
+        source = source.replace(RN, N) + N
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        state.trace("failed to parse: %r" % (fn,))
+        return None
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, fn.strpath, "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the
+        # assertion rewriting, but I don't know of a fast way to tell.
+        state.trace("failed to compile: %r" % (fn,))
+        return None
+    return co
+
+def _make_rewritten_pyc(state, fn, pyc, co):
+    """Try to dump rewritten code to *pyc*."""
+    if sys.platform.startswith("win"):
+        # Windows grants exclusive access to open files and doesn't have atomic
+        # rename, so just write into the final file.
+        _write_pyc(co, fn, pyc)
+    else:
+        # When not on windows, assume rename is atomic. Dump the code object
+        # into a file specific to this process and atomically replace it.
+        proc_pyc = pyc + "." + str(os.getpid())
+        if _write_pyc(co, fn, proc_pyc):
+            os.rename(proc_pyc, pyc)
+
+def _read_pyc(source, pyc):
+    """Possibly read a py.test pyc containing rewritten code.
+
+    Return rewritten code if successful or None if not.
+    """
+    try:
+        fp = open(pyc, "rb")
+    except IOError:
+        return None
+    try:
+        try:
+            mtime = int(source.mtime())
+            data = fp.read(8)
+        except EnvironmentError:
+            return None
+        # Check for invalid or out of date pyc file.
+        if (len(data) != 8 or
+            data[:4] != imp.get_magic() or
+            struct.unpack("<l", data[4:])[0] != mtime):
+            return None
+        co = marshal.load(fp)
+        if not isinstance(co, types.CodeType):
+            # That's interesting....
+            return None
+        return co
+    finally:
+        fp.close()
+
+
 def rewrite_asserts(mod):
     """Rewrite the assert statements in mod."""
     AssertionRewriter().run(mod)
@@ -17,13 +261,8 @@
 _saferepr = py.io.saferepr
 from _pytest.assertion.util import format_explanation as _format_explanation
 
-def _format_boolop(operands, explanations, is_or):
-    show_explanations = []
-    for operand, expl in zip(operands, explanations):
-        show_explanations.append(expl)
-        if operand == is_or:
-            break
-    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+def _format_boolop(explanations, is_or):
+    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -109,8 +348,8 @@
                     return
                 lineno += len(doc) - 1
                 expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +357,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +382,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +437,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +460,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +480,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +545,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,21 +25,39 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +86,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("</testcase>")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="xfail-marked test passes unexpectedly"/>')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('<failure message="test failure">%s</failure>',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<failure message="collection failure">%s</failure>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<skipped message="collection skipped">%s</skipped>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('<error message="test setup failure">%s</error>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="expected test failure">%s</skipped>',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('<skipped type="pytest.skip" '
-                           'message="%s">%s</skipped>',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +185,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n<testcase classname="pytest" name="internal">'
-            '    <error message="internal error">'
-            '%s</error></testcase>' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +203,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write('<testsuite ')
-        logfile.write('name="" ')
-        logfile.write('errors="%i" ' % self.errors)
-        logfile.write('failures="%i" ' % self.failed)
-        logfile.write('skips="%i" ' % self.skipped)
-        logfile.write('tests="%i" ' % numtests)
-        logfile.write('time="%.3f"' % suite_time_delta)
-        logfile.write(' >')
-        logfile.writelines(self.test_logs)
-        logfile.write('</testsuite>')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -399,6 +410,7 @@
         self._notfound = []
         self._initialpaths = set()
         self._initialparts = []
+        self.items = items = []
         for arg in args:
             parts = self._parsearg(arg)
             self._initialparts.append(parts)
@@ -414,7 +426,6 @@
         if not genitems:
             return rep.result
         else:
-            self.items = items = []
             if rep.passed:
                 for node in rep.result:
                     self.items.extend(self.genitems(node))
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "<CallSpec id=%r param=%r funcargs=%r>" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,71 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        if not indirect:
+            #XXX should we also check for the opposite case?
+            for arg in argnames:
+                if arg not in self.funcargnames:
+                    raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +678,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +769,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +781,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +830,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +956,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         <ExceptionInfo ...>
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         <ExceptionInfo ...>
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,60 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
-
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
+def pytest_sessionfinish(session):
+    session._setupstate.teardown_all()
 
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +76,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +93,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +114,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +143,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +166,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +188,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +208,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "<TestReport %r when=%r outcome=%r>" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +234,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +259,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +319,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -117,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
@@ -169,21 +192,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -289,10 +282,18 @@
         # we take care to leave out Instances aka ()
         # because later versions are going to get rid of them anyway
         if self.config.option.verbose < 0:
-            for item in items:
-                nodeid = item.nodeid
-                nodeid = nodeid.replace("::()::", "::")
-                self._tw.line(nodeid)
+            if self.config.option.verbose < -1:
+                counts = {}
+                for item in items:
+                    name = item.nodeid.split('::', 1)[0]
+                    counts[name] = counts.get(name, 0) + 1
+                for name, count in sorted(counts.items()):
+                    self._tw.line("%s: %d" % (name, count))
+            else:
+                for item in items:
+                    nodeid = item.nodeid
+                    nodeid = nodeid.replace("::()::", "::")
+                    self._tw.line(nodeid)
             return
         stack = []
         indent = ""
@@ -318,12 +319,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +394,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +412,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +431,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +445,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -83,28 +91,37 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +137,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.4.dev1'
+__version__ = '1.4.7'
 
 from py import _apipkg
 
@@ -70,6 +70,11 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
+        '_AssertionError'   : '._code.assertion:AssertionError',
+        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
+        '_reinterpret'      : '._code.assertion:reinterpret',
+        '_reprcompare'      : '._code.assertion:_reprcompare',
+        '_format_explanation' : '._code.assertion:_format_explanation',
     },
 
     # backports and additions of builtins
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -113,9 +113,12 @@
 
     # some backward compatibility helpers
     _basestring = str
-    def _totext(obj, encoding=None):
+    def _totext(obj, encoding=None, errors=None):
         if isinstance(obj, bytes):
-            obj = obj.decode(encoding)
+            if errors is None:
+                obj = obj.decode(encoding)
+            else:
+                obj = obj.decode(encoding, errors)
         elif not isinstance(obj, str):
             obj = str(obj)
         return obj
@@ -142,7 +145,7 @@
             del back
         elif locs is None:
             locs = globs
-        fp = open(fn, "rb")
+        fp = open(fn, "r")
         try:
             source = fp.read()
         finally:
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionnew.py
@@ -0,0 +1,339 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace _assertionold.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from py._code.assertion import _format_explanation, BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(failure):
+    explanation = _format_explanation(failure.explanation)
+    value = failure.cause[1]
+    if str(value):
+        lines = explanation.splitlines()
+        if not lines:
+            lines.append("")
+        lines[0] += " << %s" % (value,)
+        explanation = "\n".join(lines)
+    text = "%s: %s" % (failure.cause[0].__name__, explanation)
+    if text.startswith("AssertionError: assert "):
+        text = text[16:]
+    return text
+
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = False
+        if not local:
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not result:
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        rcomp = py.code._reprcompare
+        if rcomp:
+            res = rcomp(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = True
+        if from_instance:
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        if test_explanation.startswith("False\n{False =") and \
+                test_explanation.endswith("\n"):
+            test_explanation = test_explanation[15:-2]
+        explanation = "assert %s" % (test_explanation,)
+        if not test_result:
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionold.py
@@ -0,0 +1,555 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from py._code.assertion import BuiltinAssertionError, _format_explanation
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = '<eval>'
+            self.__obj__.filename = '<eval>'
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = '<run>'
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return _format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # simplify 'assert False where False = ...'
+        if (test.explanation.startswith('False\n{False = ') and
+            test.explanation.endswith('\n}')):
+            test.explanation = test.explanation[15:-2]
+        # print the result as  'assert <explanation>'
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = '<run>'
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
new file mode 100644
--- /dev/null
+++ b/py/_code/assertion.py
@@ -0,0 +1,94 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+_reprcompare = None # if set, will be called by assert reinterp for comparison ops
+
+def _format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "<run>".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = "<could not determine information>"
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from py._code._assertionold import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from py._code._assertionnew import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,6 +145,17 @@
         return self.frame.f_locals
     locals = property(getlocals, None, None, "locals of underlaying frame")
 
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        if self.exprinfo is None:
+            source = str(self.statement).strip()
+            x = py.code._reinterpret(source, self.frame, should_fail=True)
+            if not isinstance(x, str):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
     def getfirstlinesource(self):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
@@ -158,13 +169,12 @@
         end = self.lineno
         try:
             _, end = source.getstatementrange(end)
-        except IndexError:
+        except (IndexError, ValueError):
             end = self.lineno + 1
         # heuristic to stop displaying source on e.g.
         #   if something:  # assume this causes a NameError
         #      # _this_ lines and the one
                #        below we don't want from entry.getsource()
-        end = min(end, len(source))
         for i in range(self.lineno, end):
             if source[i].rstrip().endswith(':'):
                 end = i + 1
@@ -273,7 +283,11 @@
         """
         cache = {}
         for i, entry in enumerate(self):
-            key = entry.frame.code.path, entry.lineno
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
             #print "checking for recursion at", key
             l = cache.setdefault(key, [])
             if l:
@@ -308,7 +322,7 @@
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
         self.type, self.value, tb = self._excinfo
-        self.typename = getattr(self.type, "__name__", "???")
+        self.typename = self.type.__name__
         self.traceback = py.code.Traceback(tb)
 
     def __repr__(self):
@@ -347,14 +361,16 @@
             showlocals: show locals per traceback entry
             style: long|short|no|native traceback style
             tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
         """
         if style == 'native':
-            import traceback
-            return ''.join(traceback.format_exception(
-                self.type,
-                self.value,
-                self.traceback[0]._rawentry,
-                ))
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
 
         fmt = FormattedExcinfo(showlocals=showlocals, style=style,
             abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
@@ -452,7 +468,7 @@
     def repr_locals(self, locals):
         if self.showlocals:
             lines = []
-            keys = list(locals)
+            keys = [loc for loc in locals if loc[0] != "@"]
             keys.sort()
             for name in keys:
                 value = locals[name]
@@ -506,7 +522,10 @@
 
     def _makepath(self, path):
         if not self.abspath:
-            np = py.path.local().bestrelpath(path)
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
             if len(np) < len(str(path)):
                 path = np
         return path
@@ -595,6 +614,19 @@
         if self.extraline:
             tw.line(self.extraline)
 
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
 class ReprEntry(TerminalRepr):
     localssep = "_ "
 
@@ -680,19 +712,26 @@
 
 oldbuiltins = {}
 
-def patch_builtins(compile=True):
-    """ put compile builtins to Python's builtins. """
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from py._code import assertion
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = assertion.AssertionError
     if compile:
         l = oldbuiltins.setdefault('compile', [])
         l.append(py.builtin.builtins.compile)
         py.builtin.builtins.compile = py.code.compile
 
-def unpatch_builtins(compile=True):
+def unpatch_builtins(assertion=True, compile=True):
     """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
     if compile:
         py.builtin.builtins.compile = oldbuiltins['compile'].pop()
 
-def getrawcode(obj):
+def getrawcode(obj, trycall=True):
     """ return code object for given function. """
     try:
         return obj.__code__
@@ -701,5 +740,10 @@
         obj = getattr(obj, 'func_code', obj)
         obj = getattr(obj, 'f_code', obj)
         obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
         return obj
 
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,6 +108,7 @@
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
+            raise an IndexError if no such statementrange can be found.
         """
         # XXX there must be a better than these heuristic ways ...
         # XXX there may even be better heuristics :-)
@@ -116,6 +117,7 @@
 
         # 1. find the start of the statement
         from codeop import compile_command
+        end = None
         for start in range(lineno, -1, -1):
             if assertion:
                 line = self.lines[start]
@@ -139,7 +141,9 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, len(self)
+        if end is None:
+            raise IndexError("no valid source range around line %d " % (lineno,))
+        return start, end
 
     def getblockend(self, lineno):
         # XXX
@@ -257,23 +261,29 @@
 
 
 def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
     try:
         code = py.code.Code(obj)
     except TypeError:
-        # fallback to
-        fn = (py.std.inspect.getsourcefile(obj) or
-              py.std.inspect.getfile(obj))
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
         fspath = fn and py.path.local(fn) or None
+        lineno = -1
         if fspath:
             try:
                 _, lineno = findsource(obj)
             except IOError:
-                lineno = None
-        else:
-            lineno = None
+                pass
     else:
         fspath = code.path
         lineno = code.firstlineno
+    assert isinstance(lineno, int)
     return fspath, lineno
 
 #
@@ -286,7 +296,7 @@
     except py.builtin._sysex:
         raise
     except:
-        return None, None
+        return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
diff --git a/py/_error.py b/py/_error.py
--- a/py/_error.py
+++ b/py/_error.py
@@ -23,6 +23,7 @@
     2: errno.ENOENT,
     3: errno.ENOENT,
     17: errno.EEXIST,
+    13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -103,6 +103,7 @@
     def _parseline(self, line, lineno):
         # comments
         line = line.split('#')[0].rstrip()
+        line = line.split(';')[0].rstrip()
         # blank lines
         if not line:
             return None, None
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -12,7 +12,7 @@
     class TextIO(StringIO):
         def write(self, data):
             if not isinstance(data, unicode):
-                data = unicode(data, getattr(self, '_encoding', 'UTF-8'))
+                data = unicode(data, getattr(self, '_encoding', 'UTF-8'), 'replace')
             StringIO.write(self, data)
 else:
     TextIO = StringIO
@@ -258,6 +258,9 @@
                 f = getattr(self, name).tmpfile
                 f.seek(0)
                 res = f.read()
+                enc = getattr(f, 'encoding', None)
+                if enc:
+                    res = py.builtin._totext(res, enc, 'replace')
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -105,6 +105,8 @@
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
+    _newline = None   # the last line printed
+
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -112,11 +114,9 @@
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-                if hasattr(file, 'encoding'):
-                    encoding = file.encoding
         elif hasattr(file, '__call__'):
             file = WriteFile(file, encoding=encoding)
-        self.encoding = encoding
+        self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
@@ -182,8 +182,31 @@
         return s
 
     def line(self, s='', **kw):
+        if self._newline == False:
+            self.write("\n")
         self.write(s, **kw)
         self.write('\n')
+        self._newline = True
+
+    def reline(self, line, **opts):
+        if not self.hasmarkup:
+            raise ValueError("cannot use rewrite-line without terminal")
+        if not self._newline:
+            self.write("\r")
+        self.write(line, **opts)
+        # see if we need to fill up some spaces at the end
+        # xxx have a more exact lastlinelen working from self.write?
+        lenline = len(line)
+        try:
+            lastlen = self._lastlinelen
+        except AttributeError:
+            pass
+        else:
+            if lenline < lastlen:
+                self.write(" " * (lastlen - lenline + 1))
+        self._lastlinelen = lenline
+        self._newline = False
+
 
 class Win32ConsoleWriter(TerminalWriter):
     def write(self, s, **kw):
@@ -280,10 +303,10 @@
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
     SetConsoleTextAttribute.argtypes = [wintypes.HANDLE, wintypes.WORD]
     SetConsoleTextAttribute.restype = wintypes.BOOL
-        
+
     _GetConsoleScreenBufferInfo = \
         ctypes.windll.kernel32.GetConsoleScreenBufferInfo
-    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, 
+    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE,
                                 ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
     _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
     def GetConsoleInfo(handle):
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -64,7 +64,10 @@
                 else:
                     if bool(value) ^ bool(meth()) ^ invert:
                         return False
-            except (py.error.ENOENT, py.error.ENOTDIR):
+            except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
+                # EBUSY feels not entirely correct,
+                # but its kind of necessary since ENOMEDIUM
+                # is not accessible in python     
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
@@ -368,6 +371,5 @@
         else:
             name = str(path) # path.strpath # XXX svn?
             pattern = '*' + path.sep + pattern
-        from fnmatch import fnmatch
-        return fnmatch(name, pattern)
+        return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -157,14 +157,16 @@
         return str(self) < str(other)
 
     def samefile(self, other):
-        """ return True if 'other' references the same file as 'self'. """
-        if not iswin32:
-            return py.error.checked_call(
-                    os.path.samefile, str(self), str(other))
+        """ return True if 'other' references the same file as 'self'.
+        """
+        if not isinstance(other, py.path.local):
+            other = os.path.abspath(str(other))
         if self == other:
             return True
-        other = os.path.abspath(str(other))
-        return self == other
+        if iswin32:
+            return False # ther is no samefile
+        return py.error.checked_call(
+                os.path.samefile, str(self), str(other))
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -539,7 +541,11 @@
                 if self.basename != "__init__.py":
                     modfile = modfile[:-12]
 
-            if not self.samefile(modfile):
+            try:
+                issame = self.samefile(modfile)
+            except py.error.ENOENT:
+                issame = False
+            if not issame:
                 raise self.ImportMismatchError(modname, modfile, self)
             return mod
         else:
diff --git a/py/_path/svnurl.py b/py/_path/svnurl.py
--- a/py/_path/svnurl.py
+++ b/py/_path/svnurl.py
@@ -233,6 +233,8 @@
                 e = sys.exc_info()[1]
                 if e.err.find('non-existent in that revision') != -1:
                     raise py.error.ENOENT(self, e.err)
+                elif e.err.find("E200009:") != -1:
+                    raise py.error.ENOENT(self, e.err)
                 elif e.err.find('File not found') != -1:
                     raise py.error.ENOENT(self, e.err)
                 elif e.err.find('not part of a repository')!=-1:
diff --git a/py/_path/svnwc.py b/py/_path/svnwc.py
--- a/py/_path/svnwc.py
+++ b/py/_path/svnwc.py
@@ -482,10 +482,13 @@
         except py.process.cmdexec.Error:
             e = sys.exc_info()[1]
             strerr = e.err.lower()
-            if strerr.find('file not found') != -1:
+            if strerr.find('not found') != -1:
+                raise py.error.ENOENT(self)
+            elif strerr.find("E200009:") != -1:
                 raise py.error.ENOENT(self)
             if (strerr.find('file exists') != -1 or
                 strerr.find('file already exists') != -1 or
+                strerr.find('w150002:') != -1 or
                 strerr.find("can't create directory") != -1):
                 raise py.error.EEXIST(self)
             raise
@@ -593,7 +596,7 @@
         out = self._authsvn('lock').strip()
         if not out:
             # warning or error, raise exception
-            raise Exception(out[4:])
+            raise ValueError("unknown error in svn lock command")
 
     def unlock(self):
         """ unset a previously set lock """
@@ -1066,6 +1069,8 @@
                 modrev = '?'
                 author = '?'
                 date = ''
+            elif itemstatus == "replaced":
+                pass
             else:
                 #print entryel.toxml()
                 commitel = entryel.getElementsByTagName('commit')[0]
@@ -1148,7 +1153,11 @@
             raise  ValueError("Not a versioned resource")
             #raise ValueError, "Not a versioned resource %r" % path
         self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind']
-        self.rev = int(d['revision'])
+        try:
+            self.rev = int(d['revision'])
+        except KeyError:
+            self.rev = None
+
         self.path = py.path.local(d['path'])
         self.size = self.path.size()
         if 'lastchangedrev' in d:
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -52,7 +52,7 @@
     def unicode(self, indent=2):
         l = []
         SimpleUnicodeVisitor(l.append, indent).visit(self)
-        return "".join(l)
+        return u("").join(l)
 
     def __repr__(self):
         name = self.__class__.__name__
@@ -122,11 +122,13 @@
                 if visitmethod is not None:
                     break
             else:
-                visitmethod = self.object
+                visitmethod = self.__object
             self.cache[cls] = visitmethod
         visitmethod(node)
 
-    def object(self, obj):
+    # the default fallback handler is marked private
+    # to avoid clashes with the tag name object
+    def __object(self, obj):
         #self.write(obj)
         self.write(escape(unicode(obj)))
 
@@ -136,7 +138,8 @@
     def list(self, obj):
         assert id(obj) not in self.visited
         self.visited[id(obj)] = 1
-        map(self.visit, obj)
+        for elem in obj:
+            self.visit(elem)
 
     def Tag(self, tag):
         assert id(tag) not in self.visited
@@ -181,7 +184,11 @@
             value = getattr(attrs, name)
             if name.endswith('_'):
                 name = name[:-1]
-            return ' %s="%s"' % (name, escape(unicode(value)))
+            if isinstance(value, raw):
+                insert = value.uniobj
+            else:
+                insert = escape(unicode(value))
+            return ' %s="%s"' % (name, insert)
 
     def getstyle(self, tag):
         """ return attribute list suitable for styling. """
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -483,7 +483,7 @@
                 return a_str.strip(' ')
             elif n == 1:
                 return a_str.rstrip(' ')
-            else: 
+            else:
                 return a_str.lstrip(' ')
         s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
         assert s.no_nul
@@ -3737,6 +3737,25 @@
         s = a.build_types(f, [int])
         assert s.listdef.listitem.range_step == 0
 
+    def test_specialize_arg_memo(self):
+        @objectmodel.specialize.memo()
+        def g(n):
+            return n
+        @objectmodel.specialize.arg(0)
+        def f(i):
+            return g(i)
+        def main(i):
+            if i == 2:
+                return f(2)
+            elif i == 3:
+                return f(3)
+            else:
+                raise NotImplementedError
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(main, [int])
+        assert isinstance(s, annmodel.SomeInteger)
+
 
 def g(n):
     return [0,1,2,n]
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -158,6 +158,12 @@
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
+Note that this difference might show up indirectly in some cases.  For
+example, a generator left pending in the middle is --- again ---
+garbage-collected later in PyPy than in CPython.  You can see the
+difference if the ``yield`` keyword it is suspended at is itself
+enclosed in a ``try:`` or a ``with:`` block.
+
 Using the default GC called ``minimark``, the built-in function ``id()``
 works like it does in CPython.  With other GCs it returns numbers that
 are not real addresses (because an object can move around several times)
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
new file mode 100644
--- /dev/null
+++ b/pypy/doc/tool/makecontributor.py
@@ -0,0 +1,133 @@
+import py
+import sys
+from collections import defaultdict
+import operator
+import re
+import mercurial.localrepo
+import mercurial.ui
+
+ROOT = py.path.local(__file__).join('..', '..', '..', '..')
+author_re = re.compile('(.*) <.*>')
+pair_programming_re = re.compile(r'^\((.*?)\)')
+excluded = set(["pypy", "convert-repo"])
+
+alias = {
+    'Anders Chrigstrom': ['arre'],
+    'Antonio Cuni': ['antocuni', 'anto'],
+    'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
+    'Maciej Fijalkowski': ['fijal'],
+    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Michael Hudson': ['mwh'],
+    'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
+    "Amaury Forgeot d'Arc": ['afa'],
+    'Alex Gaynor': ['alex', 'agaynor'],
+    'David Schneider': ['bivab', 'david'],
+    'Christian Tismer': ['chris', 'christian', 'tismer',
+                         'tismer at christia-wjtqxl.localdomain'],
+    'Benjamin Peterson': ['benjamin'],
+    'Hakan Ardo': ['hakan', 'hakanardo'],
+    'Niklaus Haldimann': ['nik'],
+    'Alexander Schremmer': ['xoraxax'],
+    'Anders Hammarquist': ['iko'],
+    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'Niko Matsakis': ['niko'],
+    'Jakub Gustak': ['jlg'],
+    'Guido Wesdorp': ['guido'],
+    'Michael Foord': ['mfoord'],
+    'Mark Pearse': ['mwp'],
+    'Toon Verwaest': ['tverwaes'],
+    'Eric van Riet Paap': ['ericvrp'],
+    'Jacob Hallen': ['jacob', 'jakob'],
+    'Anders Lehmann': ['ale', 'anders'],
+    'Bert Freudenberg': ['bert'],
+    'Boris Feigin': ['boris', 'boria'],
+    'Valentino Volonghi': ['valentino', 'dialtone'],
+    'Aurelien Campeas': ['aurelien', 'aureliene'],
+    'Adrien Di Mascio': ['adim'],
+    'Jacek Generowicz': ['Jacek', 'jacek'],
+    'Jim Hunziker': ['landtuna at gmail.com'],
+    'Kristjan Valur Jonsson': ['kristjan at kristjan-lp.ccp.ad.local'],
+    'Laura Creighton': ['lac'],
+    'Aaron Iles': ['aliles'],
+    'Ludovic Aubry': ['ludal', 'ludovic'],
+    'Lukas Diekmann': ['l.diekmann', 'ldiekmann'],
+    'Matti Picus': ['Matti Picus matti.picus at gmail.com',
+                    'matthp', 'mattip', 'mattip>'],
+    'Michael Cheng': ['mikefc'],
+    'Richard Emslie': ['rxe'],
+    'Roberto De Ioris': ['roberto at goyle'],
+    'Roberto De Ioris': ['roberto at mrspurr'],
+    'Sven Hager': ['hager'],
+    'Tomo Cocoa': ['cocoatomo'],
+    }
+
+alias_map = {}
+for name, nicks in alias.iteritems():
+    for nick in nicks:
+        alias_map[nick] = name
+
+def get_canonical_author(name):
+    match = author_re.match(name)
+    if match:
+        name = match.group(1)
+    return alias_map.get(name, name)
+
+ignored_nicknames = defaultdict(int)
+
+def get_more_authors(log):
+    match = pair_programming_re.match(log)
+    if not match:
+        return set()
+    ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
+                    'in-progress', 'bits of', 'even a little', 'floating',]
+    sep_words = ['and', ';', '+', '/', 'with special  by']
+    nicknames = match.group(1)
+    for word in ignore_words:
+        nicknames = nicknames.replace(word, '')
+    for word in sep_words:
+        nicknames = nicknames.replace(word, ',')
+    nicknames = [nick.strip().lower() for nick in nicknames.split(',')]
+    authors = set()
+    for nickname in nicknames:
+        author = alias_map.get(nickname)
+        if not author:
+            ignored_nicknames[nickname] += 1
+        else:
+            authors.add(author)
+    return authors
+
+def main(show_numbers):
+    ui = mercurial.ui.ui()
+    repo = mercurial.localrepo.localrepository(ui, str(ROOT))
+    authors_count = defaultdict(int)
+    for i in repo:
+        ctx = repo[i]
+        authors = set()
+        authors.add(get_canonical_author(ctx.user()))
+        authors.update(get_more_authors(ctx.description()))
+        for author in authors:
+            if author not in excluded:
+                authors_count[author] += 1
+
+    # uncomment the next lines to get the list of nicknamed which could not be
+    # parsed from commit logs
+    ## items = ignored_nicknames.items()
+    ## items.sort(key=operator.itemgetter(1), reverse=True)
+    ## for name, n in items:
+    ##     if show_numbers:
+    ##         print '%5d %s' % (n, name)
+    ##     else:
+    ##         print name
+                
+    items = authors_count.items()
+    items.sort(key=operator.itemgetter(1), reverse=True)
+    for name, n in items:
+        if show_numbers:
+            print '%5d %s' % (n, name)
+        else:
+            print name
+
+if __name__ == '__main__':
+    show_numbers = '-n' in sys.argv
+    main(show_numbers)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -169,9 +173,17 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +198,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +236,20 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +408,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2253,24 +2253,6 @@
     """Concat two strings giving a new Unicode string."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
-def PyUnicode_Split(space, s, sep, maxsplit):
-    """Split a string giving a list of Unicode strings.  If sep is NULL, splitting
-    will be done at all whitespace substrings.  Otherwise, splits occur at the given
-    separator.  At most maxsplit splits will be done.  If negative, no limit is
-    set.  Separators are not included in the resulting list.
-
-    This function used an int type for maxsplit. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyUnicode_Splitlines(space, s, keepend):
-    """Split a Unicode string at line breaks, returning a list of Unicode strings.
-    CRLF is considered to be one line break.  If keepend is 0, the Line break
-    characters are not included in the resulting strings."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.CCHARP], PyObject)
 def PyUnicode_Translate(space, str, table, errors):
     """Translate a string by applying a character mapping table to it and return the
@@ -2287,29 +2269,6 @@
     use the default error handling."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t, error=-2)
-def PyUnicode_Find(space, str, substr, start, end, direction):
-    """Return the first position of substr in str*[*start:end] using the given
-    direction (direction == 1 means to do a forward search, direction == -1 a
-    backward search).  The return value is the index of the first match; a value of
-    -1 indicates that no match was found, and -2 indicates that an error
-    occurred and an exception has been set.
-
-    This function used an int type for start and end. This
-    might require changes in your code for properly supporting 64-bit
-    systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
-def PyUnicode_Count(space, str, substr, start, end):
-    """Return the number of non-overlapping occurrences of substr in
-    str[start:end].  Return -1 if an error occurred.
-
-    This function returned an int type and used an int
-    type for start and end. This might require changes in your code for
-    properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
 def PyUnicode_RichCompare(space, left, right, op):
     """Rich compare two unicode strings and return one of the following:
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -457,3 +457,31 @@
         assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+
+    def test_count(self, space, api):
+        w_str = space.wrap(u"abcabdab")
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, -1) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, 2) == 1
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), -5, 30) == 2
+
+    def test_find(self, space, api):
+        w_str = space.wrap(u"abcabcd")
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, 1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, 1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
+
+    def test_split(self, space, api):
+        w_str = space.wrap(u"a\nb\nc\nd")
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
+        assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 0)))
+        assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 1)))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -598,3 +598,46 @@
     else:
         return stringtype.stringendswith(str, substr, start, end)
 
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
+def PyUnicode_Count(space, w_str, w_substr, start, end):
+    """Return the number of non-overlapping occurrences of substr in
+    str[start:end].  Return -1 if an error occurred."""
+    w_count = space.call_method(w_str, "count", w_substr,
+                                space.wrap(start), space.wrap(end))
+    return space.int_w(w_count)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
+             Py_ssize_t, error=-2)
+def PyUnicode_Find(space, w_str, w_substr, start, end, direction):
+    """Return the first position of substr in str*[*start:end] using
+    the given direction (direction == 1 means to do a forward search,
+    direction == -1 a backward search).  The return value is the index
+    of the first match; a value of -1 indicates that no match was
+    found, and -2 indicates that an error occurred and an exception
+    has been set."""
+    if rffi.cast(lltype.Signed, direction) > 0:
+        w_pos = space.call_method(w_str, "find", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    else:
+        w_pos = space.call_method(w_str, "rfind", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    return space.int_w(w_pos)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
+def PyUnicode_Split(space, w_str, w_sep, maxsplit):
+    """Split a string giving a list of Unicode strings.  If sep is
+    NULL, splitting will be done at all whitespace substrings.
+    Otherwise, splits occur at the given separator.  At most maxsplit
+    splits will be done.  If negative, no limit is set.  Separators
+    are not included in the resulting list."""
+    if w_sep is None:
+        w_sep = space.w_None
+    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyUnicode_Splitlines(space, w_str, keepend):
+    """Split a Unicode string at line breaks, returning a list of
+    Unicode strings.  CRLF is considered to be one line break.  If
+    keepend is 0, the Line break characters are not included in the
+    resulting strings."""
+    return space.call_method(w_str, "splitlines", space.wrap(keepend))
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -513,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1435,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1410,6 +1410,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -592,9 +592,8 @@
 
     def test_shift(self):
         from _numpypy import left_shift, right_shift
-        import sys
 
-        assert (left_shift([5, 1], [2, 31]) == [20, 2**31]).all()
+        assert (left_shift([5, 1], [2, 13]) == [20, 2**13]).all()
         assert (right_shift(10, range(5)) == [10, 5, 2, 1, 0]).all()
 
     def test_comparisons(self):
@@ -763,6 +762,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -773,7 +774,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -786,6 +788,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -797,7 +801,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True
diff --git a/pypy/module/pyexpat/test/__init__.py b/pypy/module/pyexpat/test/__init__.py
new file mode 100644
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -434,6 +434,13 @@
         self.lastblock = block
         self.pushvalue(w_result)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # This opcode was added with pypy-1.8.  Here is a simpler
+        # version, enough for annotation.
+        last_val = self.popvalue()
+        self.pushvalue(self.space.newlist([]))
+        self.pushvalue(last_val)
+
     # XXX Unimplemented 2.7 opcodes ----------------
 
     # Set literals, set comprehensions
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old -rf
+addopts = --assert=plain -rf
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -204,6 +204,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.call_location()
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,9 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +18,17 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +40,7 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/test_all.py b/pypy/test_all.py
old mode 100755
new mode 100644
--- a/pypy/test_all.py
+++ b/pypy/test_all.py
@@ -11,11 +11,12 @@
 """
 import sys, os
 
-if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
-    print >> sys.stderr, __doc__
-    sys.exit(2)
 
 if __name__ == '__main__':
+    if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.':
+        print >> sys.stderr, __doc__
+        sys.exit(2)
+
     import tool.autopath
     import pytest
     import pytest_cov
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/jitlogparser/test/__init__.py b/pypy/tool/jitlogparser/test/__init__.py
new file mode 100644
diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py
--- a/pypy/tool/pytest/test/test_pytestsupport.py
+++ b/pypy/tool/pytest/test/test_pytestsupport.py
@@ -165,7 +165,10 @@
     def test_one(self): exec 'blow'
     """)
 
-    ev, = sorter.getreports("pytest_runtest_logreport")
+    reports = sorter.getreports("pytest_runtest_logreport")
+    setup, ev, teardown = reports
     assert ev.failed
+    assert setup.passed
+    assert teardown.passed
     assert 'NameError' in ev.longrepr.reprcrash.message
     assert 'blow' in ev.longrepr.reprcrash.message
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:
diff --git a/testrunner/runner.py b/testrunner/runner.py
--- a/testrunner/runner.py
+++ b/testrunner/runner.py
@@ -110,7 +110,10 @@
                  do_dry_run=False, timeout=None,
                  _win32=(sys.platform=='win32')):
     args = interp + test_driver
-    args += ['-p', 'resultlog', '--resultlog=%s' % logfname, test]
+    args += ['-p', 'resultlog',
+             '--resultlog=%s' % logfname,
+             '--junitxml=%s.junit' % logfname,
+             test]
 
     args = map(str, args)
     interp0 = args[0]
diff --git a/testrunner/scratchbox_runner.py b/testrunner/scratchbox_runner.py
--- a/testrunner/scratchbox_runner.py
+++ b/testrunner/scratchbox_runner.py
@@ -14,14 +14,14 @@
 def dry_run_scratchbox(args, cwd, out, timeout=None):
     return dry_run(args_for_scratchbox(cwd, args), cwd, out, timeout)
 
-import runner
-# XXX hack hack hack
-dry_run = runner.dry_run
-run = runner.run
+if __name__ == '__main__':
+    import runner
+    # XXX hack hack hack
+    dry_run = runner.dry_run
+    run = runner.run
 
-runner.dry_run = dry_run_scratchbox
-runner.run = run_scratchbox
+    runner.dry_run = dry_run_scratchbox
+    runner.run = run_scratchbox
 
-if __name__ == '__main__':
     import sys
     runner.main(sys.argv)
diff --git a/testrunner/test/conftest.py b/testrunner/test/conftest.py
new file mode 100644
--- /dev/null
+++ b/testrunner/test/conftest.py
@@ -0,0 +1,6 @@
+
+def pytest_runtest_makereport(__multicall__, item):
+    report = __multicall__.execute()
+    if 'out' in item.funcargs:
+        report.sections.append(('out', item.funcargs['out'].read()))
+    return report
diff --git a/testrunner/test/test_runner.py b/testrunner/test/test_runner.py
--- a/testrunner/test/test_runner.py
+++ b/testrunner/test/test_runner.py
@@ -53,49 +53,44 @@
     assert not should_report_failure("F Def\n. Ghi\n. Jkl\n")
 
 
+
 class TestRunHelper(object):
+    def pytest_funcarg__out(self, request):
+        tmpdir = request.getfuncargvalue('tmpdir')
+        return tmpdir.ensure('out')
 
-    def setup_method(self, meth):
-        h, self.fn = tempfile.mkstemp()
-        os.close(h)
+    def test_run(self, out):
+        res = runner.run([sys.executable, "-c", "print 42"], '.', out)
+        assert res == 0
+        assert out.read() == "42\n"
 
-    def teardown_method(self, meth):
-        os.unlink(self.fn)
-
-    def test_run(self):
-        res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn))
-        assert res == 0
-        out = py.path.local(self.fn).read('r')
-        assert out == "42\n"
-
-    def test_error(self):
-        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', py.path.local(self.fn))
+    def test_error(self, out):
+        res = runner.run([sys.executable, "-c", "import sys; sys.exit(3)"], '.', out)
         assert res == 3
 
-    def test_signal(self):
+    def test_signal(self, out):
         if sys.platform == 'win32':
             py.test.skip("no death by signal on windows")
-        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', py.path.local(self.fn))
+        res = runner.run([sys.executable, "-c", "import os; os.kill(os.getpid(), 9)"], '.', out)
         assert res == -9
 
-    def test_timeout(self):
-        res = runner.run([sys.executable, "-c", "while True: pass"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout(self, out):
+        res = runner.run([sys.executable, "-c", "while True: pass"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_lock(self):
-        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_lock(self, out):
+        res = runner.run([sys.executable, "-c", "import threading; l=threading.Lock(); l.acquire(); l.acquire()"], '.', out, timeout=3)
         assert res == -999
 
-    def test_timeout_syscall(self):
-        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', py.path.local(self.fn), timeout=3)
+    def test_timeout_syscall(self, out):
+        res = runner.run([sys.executable, "-c", "import socket; s=s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.bind(('', 0)); s.recv(1000)"], '.', out, timeout=3)
         assert res == -999        
 
-    def test_timeout_success(self):
+    def test_timeout_success(self, out):
         res = runner.run([sys.executable, "-c", "print 42"], '.',
-                         py.path.local(self.fn), timeout=2)
+                         out, timeout=2)
         assert res == 0
-        out = py.path.local(self.fn).read('r')
+        out = out.read()
         assert out == "42\n"        
 
 
@@ -122,7 +117,10 @@
 
         expected = ['INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
+
                     'test_one']
 
         assert self.called == (expected, '/wd', 'out', 'secs')        
@@ -138,9 +136,11 @@
 
         expected = ['/wd' + os.sep + './INTERP', 'IARG',
                     'driver', 'darg',
+                    '-p', 'resultlog',
                     '--resultlog=LOGFILE',
+                    '--junitxml=LOGFILE.junit',
                     'test_one']
-
+        assert self.called[0] == expected
         assert self.called == (expected, '/wd', 'out', 'secs')        
         assert res == 0
 
@@ -251,7 +251,7 @@
         assert '\n' in log        
         log_lines = log.splitlines()
 
-        assert log_lines[0] == ". test_normal/test_example.py:test_one"
+        assert ". test_normal/test_example.py::test_one" in log_lines
         nfailures = 0
         noutcomes = 0
         for line in log_lines:

From noreply at buildbot.pypy.org  Tue Apr 17 10:31:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 10:31:41 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Complete the comment.
Message-ID: <20120417083141.436CA82F4F@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54450:df717693c947
Date: 2012-04-17 10:30 +0200
http://bitbucket.org/pypy/pypy/changeset/df717693c947/

Log:	Complete the comment.

diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -74,6 +74,10 @@
 def remove_epoll(ep):
     """Explicitly unregister the epoll object.  Note that raising an
     exception in a transaction to abort run() also unregisters all epolls.
+    However, an epoll that becomes empty (doesn't wait on any fd) is not
+    automatically removed; if there is only an empty epoll left and no
+    further transactions, and no-one raised an exception, then it will
+    basically deadlock.
     """
     for key, (f, args, kwds) in _pending.items():
         if getattr(f, '_reads_from_epoll_', None) is ep:

From noreply at buildbot.pypy.org  Tue Apr 17 10:31:39 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 10:31:39 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fixes.
Message-ID: <20120417083139.D0D91820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54449:a33aa4134c22
Date: 2012-04-16 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/a33aa4134c22/

Log:	Fixes.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -128,6 +128,8 @@
         # transaction is aborted, the latter might never be called.
         # Be ready here to clean up any state.
         self._cleanup_state()
+        if self is not self.gc.main_thread_tls:
+            self.gc.root_walker.clear_current_stack_roots()
         if self.nursery_free:
             clear_size = self.nursery_free - self.nursery_start
         else:
@@ -284,7 +286,7 @@
 
     def collect_roots_from_stack(self):
         self.gc.root_walker.walk_current_stack_roots(
-            StmGCTLS._trace_drag_out1, self)
+            StmGCTLS._trace_drag_out_if_not_global1, self)
 
     def trace_and_drag_out_of_nursery(self, obj):
         # This is called to fix the references inside 'obj', to ensure that
@@ -294,8 +296,8 @@
         # objects.
         self.gc.trace(obj, self._trace_drag_out, None)
 
-    def _trace_drag_out1(self, root):
-        self._trace_drag_out(root, None)
+    def _trace_drag_out_if_not_global1(self, root):
+        self._trace_drag_out_if_not_global(root, None)
 
     def _trace_drag_out_if_not_global(self, root, ignored):
         # like _trace_drag_out(), but ignores references to GLOBAL objects.
@@ -313,6 +315,7 @@
         """
         obj = root.address[0]
         hdr = self.gc.header(obj)
+        ll_assert(hdr.tid & GCFLAG_GLOBAL == 0, "unexpected GLOBAL obj")
         #
         # If 'obj' is not in the nursery, we set GCFLAG_VISITED
         if not self.is_in_nursery(obj):
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -135,6 +135,8 @@
 class FakeRootWalker:
     def walk_current_stack_roots(self, *args):
         pass     # no stack roots in this test file
+    def clear_current_stack_roots(self):
+        pass
 
 
 class StmGCTests:
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -169,3 +169,9 @@
         self.rootstackhook(collect_stack_root, arg,
                            stackgcdata.root_stack_base,
                            stackgcdata.root_stack_top)
+
+    def clear_current_stack_roots(self):
+        """Used when we start a transaction: there might be garbage left
+        behind by the previous aborted transaction."""
+        stackgcdata = self.stackgcdata
+        stackgcdata.root_stack_top = stackgcdata.root_stack_base

From noreply at buildbot.pypy.org  Tue Apr 17 11:11:59 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 11:11:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix issue 1131: coerce the return value
 of __contains__ to bool
Message-ID: <20120417091159.20D38820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54451:3efbde36c6ed
Date: 2012-04-17 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/3efbde36c6ed/

Log:	fix issue 1131: coerce the return value of __contains__ to bool

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.wrap(space.bool_w(w_result))
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,24 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                return 42
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        #
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise TypeError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(TypeError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}

From noreply at buildbot.pypy.org  Tue Apr 17 11:12:01 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 11:12:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120417091201.2A4A7820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54452:afff86e02d5b
Date: 2012-04-17 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/afff86e02d5b/

Log:	merge heads

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -171,7 +175,13 @@
         space = self.space
         keywords, values_w = space.view_as_kwargs(w_starstararg)
         if keywords is not None: # this path also taken for empty dicts
-            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
             return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
             keys_w = space.unpackiterable(w_starstararg)
@@ -229,22 +239,16 @@
     @jit.look_inside_iff(lambda self, keywords, keywords_w:
             jit.isconstant(len(keywords) and
             jit.isconstant(self.keywords)))
-    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
-        if self.keywords is None:
-            self.keywords = keywords[:] # copy to make non-resizable
-            self.keywords_w = keywords_w[:]
-        else:
-            # looks quadratic, but the JIT should remove all of it nicely.
-            # Also, all the lists should be small
-            for key in keywords:
-                for otherkey in self.keywords:
-                    if otherkey == key:
-                        raise operationerrfmt(self.space.w_TypeError,
-                                              "got multiple values "
-                                              "for keyword argument "
-                                              "'%s'", key)
-            self.keywords = self.keywords + keywords
-            self.keywords_w = self.keywords_w + keywords_w
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
 
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -762,6 +762,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -772,7 +774,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -785,6 +788,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -796,7 +801,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True
diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,9 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +18,17 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +40,7 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False

From noreply at buildbot.pypy.org  Tue Apr 17 11:20:43 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 11:20:43 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Test and fix.
Message-ID: <20120417092043.2C49F820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54453:140fbb1aac69
Date: 2012-04-17 10:49 +0200
http://bitbucket.org/pypy/pypy/changeset/140fbb1aac69/

Log:	Test and fix.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -286,7 +286,7 @@
 
     def collect_roots_from_stack(self):
         self.gc.root_walker.walk_current_stack_roots(
-            StmGCTLS._trace_drag_out_if_not_global1, self)
+            StmGCTLS._trace_drag_out1, self)
 
     def trace_and_drag_out_of_nursery(self, obj):
         # This is called to fix the references inside 'obj', to ensure that
@@ -296,16 +296,8 @@
         # objects.
         self.gc.trace(obj, self._trace_drag_out, None)
 
-    def _trace_drag_out_if_not_global1(self, root):
-        self._trace_drag_out_if_not_global(root, None)
-
-    def _trace_drag_out_if_not_global(self, root, ignored):
-        # like _trace_drag_out(), but ignores references to GLOBAL objects.
-        # used only for the LOCAL copy of a GLOBAL object, which may still
-        # have further GLOBAL pointers.
-        obj = root.address[0]
-        if self.gc.header(obj).tid & GCFLAG_GLOBAL == 0:
-            self._trace_drag_out(root, ignored)
+    def _trace_drag_out1(self, root):
+        self._trace_drag_out(root, None)
 
     def _trace_drag_out(self, root, ignored):
         """Trace callback: 'root' is the address of some pointer.  If that
@@ -315,11 +307,12 @@
         """
         obj = root.address[0]
         hdr = self.gc.header(obj)
-        ll_assert(hdr.tid & GCFLAG_GLOBAL == 0, "unexpected GLOBAL obj")
         #
         # If 'obj' is not in the nursery, we set GCFLAG_VISITED
         if not self.is_in_nursery(obj):
-            if hdr.tid & GCFLAG_VISITED == 0:
+            # we ignore both GLOBAL objects and objects which have already
+            # been VISITED
+            if hdr.tid & (GCFLAG_GLOBAL|GCFLAG_VISITED) == 0:
                 ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0,
                           "local GCFLAG_WAS_COPIED without GCFLAG_VISITED")
                 hdr.tid |= GCFLAG_VISITED
@@ -423,7 +416,7 @@
                                    self.gc.get_type_id(globalobj))
         ll_assert(TL == TG, "in a root: type(LOCAL) != type(GLOBAL)")
         #
-        self.gc.trace(localobj, self._trace_drag_out_if_not_global, None)
+        self.trace_and_drag_out_of_nursery(localobj)
 
     def collect_flush_pending(self):
         # Follow the objects in the 'pending' stack and move the
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -359,6 +359,19 @@
         assert s_adr != t_adr
         self.gc.commit_transaction()
 
+    def test_commit_local_obj_with_global_references(self):
+        t, t_adr = self.malloc(S)
+        tr, tr_adr = self.malloc(SR)
+        tr.s1 = t
+        self.select_thread(1)
+        sr_adr = self.gc.stm_writebarrier(tr_adr)
+        assert sr_adr != tr_adr
+        sr = llmemory.cast_adr_to_ptr(sr_adr, lltype.Ptr(SR))
+        sr2, sr2_adr = self.malloc(SR)
+        sr.sr2 = sr2
+        sr2.s1 = t
+        self.gc.commit_transaction()
+
     def test_commit_transaction_no_references(self):
         py.test.skip("rewrite me")
         s, s_adr = self.malloc(S)

From noreply at buildbot.pypy.org  Tue Apr 17 11:20:56 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 11:20:56 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120417092056.89004820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54454:d121be30ed67
Date: 2012-04-17 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/d121be30ed67/

Log:	hg merge default

diff too long, truncating to 10000 out of 17619 lines

diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.1.0.dev4'
+__version__ = '2.2.4.dev2'
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
--- a/_pytest/assertion/__init__.py
+++ b/_pytest/assertion/__init__.py
@@ -2,35 +2,25 @@
 support for presenting detailed information in failing assertions.
 """
 import py
-import imp
-import marshal
-import struct
 import sys
 import pytest
 from _pytest.monkeypatch import monkeypatch
-from _pytest.assertion import reinterpret, util
-
-try:
-    from _pytest.assertion.rewrite import rewrite_asserts
-except ImportError:
-    rewrite_asserts = None
-else:
-    import ast
+from _pytest.assertion import util
 
 def pytest_addoption(parser):
     group = parser.getgroup("debugconfig")
-    group.addoption('--assertmode', action="store", dest="assertmode",
-                    choices=("on", "old", "off", "default"), default="default",
-                    metavar="on|old|off",
+    group.addoption('--assert', action="store", dest="assertmode",
+                    choices=("rewrite", "reinterp", "plain",),
+                    default="rewrite", metavar="MODE",
                     help="""control assertion debugging tools.
-'off' performs no assertion debugging.
-'old' reinterprets the expressions in asserts to glean information.
-'on' (the default) rewrites the assert statements in test modules to provide
-sub-expression results.""")
+'plain' performs no assertion debugging.
+'reinterp' reinterprets assert statements after they failed to provide assertion expression information.
+'rewrite' (the default) rewrites assert statements in test modules on import
+to provide assert expression information. """)
     group.addoption('--no-assert', action="store_true", default=False,
-        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+        dest="noassert", help="DEPRECATED equivalent to --assert=plain")
     group.addoption('--nomagic', action="store_true", default=False,
-        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+        dest="nomagic", help="DEPRECATED equivalent to --assert=plain")
 
 class AssertionState:
     """State for the assertion plugin."""
@@ -40,89 +30,90 @@
         self.trace = config.trace.root.get("assertion")
 
 def pytest_configure(config):
-    warn_about_missing_assertion()
     mode = config.getvalue("assertmode")
     if config.getvalue("noassert") or config.getvalue("nomagic"):
-        if mode not in ("off", "default"):
-            raise pytest.UsageError("assertion options conflict")
-        mode = "off"
-    elif mode == "default":
-        mode = "on"
-    if mode != "off":
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
+        mode = "plain"
+    if mode == "rewrite":
+        try:
+            import ast
+        except ImportError:
+            mode = "reinterp"
+        else:
+            if sys.platform.startswith('java'):
+                mode = "reinterp"
+    if mode != "plain":
+        _load_modules(mode)
         m = monkeypatch()
         config._cleanup.append(m.undo)
         m.setattr(py.builtin.builtins, 'AssertionError',
                   reinterpret.AssertionError)
-        m.setattr(util, '_reprcompare', callbinrepr)
-    if mode == "on" and rewrite_asserts is None:
-        mode = "old"
+    hook = None
+    if mode == "rewrite":
+        hook = rewrite.AssertionRewritingHook()
+        sys.meta_path.append(hook)
+    warn_about_missing_assertion(mode)
     config._assertstate = AssertionState(config, mode)
+    config._assertstate.hook = hook
     config._assertstate.trace("configured with mode set to %r" % (mode,))
 
-def _write_pyc(co, source_path):
-    if hasattr(imp, "cache_from_source"):
-        # Handle PEP 3147 pycs.
-        pyc = py.path.local(imp.cache_from_source(str(source_path)))
-        pyc.ensure()
-    else:
-        pyc = source_path + "c"
-    mtime = int(source_path.mtime())
-    fp = pyc.open("wb")
-    try:
-        fp.write(imp.get_magic())
-        fp.write(struct.pack("<l", mtime))
-        marshal.dump(co, fp)
-    finally:
-        fp.close()
-    return pyc
+def pytest_unconfigure(config):
+    hook = config._assertstate.hook
+    if hook is not None:
+        sys.meta_path.remove(hook)
 
-def before_module_import(mod):
-    if mod.config._assertstate.mode != "on":
-        return
-    # Some deep magic: load the source, rewrite the asserts, and write a
-    # fake pyc, so that it'll be loaded when the module is imported.
-    source = mod.fspath.read()
-    try:
-        tree = ast.parse(source)
-    except SyntaxError:
-        # Let this pop up again in the real import.
-        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
-        return
-    rewrite_asserts(tree)
-    try:
-        co = compile(tree, str(mod.fspath), "exec")
-    except SyntaxError:
-        # It's possible that this error is from some bug in the assertion
-        # rewriting, but I don't know of a fast way to tell.
-        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
-        return
-    mod._pyc = _write_pyc(co, mod.fspath)
-    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+def pytest_collection(session):
+    # this hook is only called when test modules are collected
+    # so for example not in the master process of pytest-xdist
+    # (which does not collect test modules)
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.set_session(session)
 
-def after_module_import(mod):
-    if not hasattr(mod, "_pyc"):
-        return
-    state = mod.config._assertstate
-    try:
-        mod._pyc.remove()
-    except py.error.ENOENT:
-        state.trace("couldn't find pyc: %r" % (mod._pyc,))
-    else:
-        state.trace("removed pyc: %r" % (mod._pyc,))
+def pytest_runtest_setup(item):
+    def callbinrepr(op, left, right):
+        hook_result = item.ihook.pytest_assertrepr_compare(
+            config=item.config, op=op, left=left, right=right)
+        for new_expl in hook_result:
+            if new_expl:
+                res = '\n~'.join(new_expl)
+                if item.config.getvalue("assertmode") == "rewrite":
+                    # The result will be fed back a python % formatting
+                    # operation, which will fail if there are extraneous
+                    # '%'s in the string. Escape them here.
+                    res = res.replace("%", "%%")
+                return res
+    util._reprcompare = callbinrepr
 
-def warn_about_missing_assertion():
+def pytest_runtest_teardown(item):
+    util._reprcompare = None
+
+def pytest_sessionfinish(session):
+    hook = session.config._assertstate.hook
+    if hook is not None:
+        hook.session = None
+
+def _load_modules(mode):
+    """Lazily import assertion related code."""
+    global rewrite, reinterpret
+    from _pytest.assertion import reinterpret
+    if mode == "rewrite":
+        from _pytest.assertion import rewrite
+
+def warn_about_missing_assertion(mode):
     try:
         assert False
     except AssertionError:
         pass
     else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
+        if mode == "rewrite":
+            specifically = ("assertions which are not in test modules "
+                            "will be ignored")
+        else:
+            specifically = "failing tests may report as passing"
+
+        sys.stderr.write("WARNING: " + specifically +
+                        " because assert statements are not executed "
+                        "by the underlying Python interpreter "
+                        "(are you using python -O?)\n")
 
 pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
--- a/_pytest/assertion/newinterpret.py
+++ b/_pytest/assertion/newinterpret.py
@@ -53,7 +53,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
+                "compute assert expression before the assert or use --assert=plain)")
 
 def run(offending_line, frame=None):
     if frame is None:
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
@@ -482,7 +481,7 @@
     if should_fail:
         return ("(assertion failed, but when it was re-run for "
                 "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --nomagic)")
+                "compute assert expression before the assert or use --assert=plain)")
     else:
         return None
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -1,14 +1,258 @@
 """Rewrite assertion AST to produce nice error messages"""
 
 import ast
-import collections
+import errno
 import itertools
+import imp
+import marshal
+import os
+import struct
 import sys
+import types
 
 import py
 from _pytest.assertion import util
 
 
+# Windows gives ENOENT in places *nix gives ENOTDIR.
+if sys.platform.startswith("win"):
+    PATH_COMPONENT_NOT_DIR = errno.ENOENT
+else:
+    PATH_COMPONENT_NOT_DIR = errno.ENOTDIR
+
+# py.test caches rewritten pycs in __pycache__.
+if hasattr(imp, "get_tag"):
+    PYTEST_TAG = imp.get_tag() + "-PYTEST"
+else:
+    if hasattr(sys, "pypy_version_info"):
+        impl = "pypy"
+    elif sys.platform == "java":
+        impl = "jython"
+    else:
+        impl = "cpython"
+    ver = sys.version_info
+    PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1])
+    del ver, impl
+
+PYC_EXT = ".py" + "c" if __debug__ else "o"
+PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
+
+REWRITE_NEWLINES = sys.version_info[:2] != (2, 7) and sys.version_info < (3, 2)
+
+class AssertionRewritingHook(object):
+    """Import hook which rewrites asserts."""
+
+    def __init__(self):
+        self.session = None
+        self.modules = {}
+
+    def set_session(self, session):
+        self.fnpats = session.config.getini("python_files")
+        self.session = session
+
+    def find_module(self, name, path=None):
+        if self.session is None:
+            return None
+        sess = self.session
+        state = sess.config._assertstate
+        state.trace("find_module called for: %s" % name)
+        names = name.rsplit(".", 1)
+        lastname = names[-1]
+        pth = None
+        if path is not None and len(path) == 1:
+            pth = path[0]
+        if pth is None:
+            try:
+                fd, fn, desc = imp.find_module(lastname, path)
+            except ImportError:
+                return None
+            if fd is not None:
+                fd.close()
+            tp = desc[2]
+            if tp == imp.PY_COMPILED:
+                if hasattr(imp, "source_from_cache"):
+                    fn = imp.source_from_cache(fn)
+                else:
+                    fn = fn[:-1]
+            elif tp != imp.PY_SOURCE:
+                # Don't know what this is.
+                return None
+        else:
+            fn = os.path.join(pth, name.rpartition(".")[2] + ".py")
+        fn_pypath = py.path.local(fn)
+        # Is this a test file?
+        if not sess.isinitpath(fn):
+            # We have to be very careful here because imports in this code can
+            # trigger a cycle.
+            self.session = None
+            try:
+                for pat in self.fnpats:
+                    if fn_pypath.fnmatch(pat):
+                        state.trace("matched test file %r" % (fn,))
+                        break
+                else:
+                    return None
+            finally:
+                self.session = sess
+        else:
+            state.trace("matched test file (was specified on cmdline): %r" % (fn,))
+        # The requested module looks like a test file, so rewrite it. This is
+        # the most magical part of the process: load the source, rewrite the
+        # asserts, and load the rewritten source. We also cache the rewritten
+        # module code in a special pyc. We must be aware of the possibility of
+        # concurrent py.test processes rewriting and loading pycs. To avoid
+        # tricky race conditions, we maintain the following invariant: The
+        # cached pyc is always a complete, valid pyc. Operations on it must be
+        # atomic. POSIX's atomic rename comes in handy.
+        write = not sys.dont_write_bytecode
+        cache_dir = os.path.join(fn_pypath.dirname, "__pycache__")
+        if write:
+            try:
+                os.mkdir(cache_dir)
+            except OSError:
+                e = sys.exc_info()[1].errno
+                if e == errno.EEXIST:
+                    # Either the __pycache__ directory already exists (the
+                    # common case) or it's blocked by a non-dir node. In the
+                    # latter case, we'll ignore it in _write_pyc.
+                    pass
+                elif e == PATH_COMPONENT_NOT_DIR:
+                    # One of the path components was not a directory, likely
+                    # because we're in a zip file.
+                    write = False
+                elif e == errno.EACCES:
+                    state.trace("read only directory: %r" % (fn_pypath.dirname,))
+                    write = False
+                else:
+                    raise
+        cache_name = fn_pypath.basename[:-3] + PYC_TAIL
+        pyc = os.path.join(cache_dir, cache_name)
+        # Notice that even if we're in a read-only directory, I'm going to check
+        # for a cached pyc. This may not be optimal...
+        co = _read_pyc(fn_pypath, pyc)
+        if co is None:
+            state.trace("rewriting %r" % (fn,))
+            co = _rewrite_test(state, fn_pypath)
+            if co is None:
+                # Probably a SyntaxError in the test.
+                return None
+            if write:
+                _make_rewritten_pyc(state, fn_pypath, pyc, co)
+        else:
+            state.trace("found cached rewritten pyc for %r" % (fn,))
+        self.modules[name] = co, pyc
+        return self
+
+    def load_module(self, name):
+        co, pyc = self.modules.pop(name)
+        # I wish I could just call imp.load_compiled here, but __file__ has to
+        # be set properly. In Python 3.2+, this all would be handled correctly
+        # by load_compiled.
+        mod = sys.modules[name] = imp.new_module(name)
+        try:
+            mod.__file__ = co.co_filename
+            # Normally, this attribute is 3.2+.
+            mod.__cached__ = pyc
+            py.builtin.exec_(co, mod.__dict__)
+        except:
+            del sys.modules[name]
+            raise
+        return sys.modules[name]
+
+def _write_pyc(co, source_path, pyc):
+    # Technically, we don't have to have the same pyc format as (C)Python, since
+    # these "pycs" should never be seen by builtin import. However, there's
+    # little reason deviate, and I hope sometime to be able to use
+    # imp.load_compiled to load them. (See the comment in load_module above.)
+    mtime = int(source_path.mtime())
+    try:
+        fp = open(pyc, "wb")
+    except IOError:
+        err = sys.exc_info()[1].errno
+        if err == PATH_COMPONENT_NOT_DIR:
+            # This happens when we get a EEXIST in find_module creating the
+            # __pycache__ directory and __pycache__ is by some non-dir node.
+            return False
+        raise
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return True
+
+RN = "\r\n".encode("utf-8")
+N = "\n".encode("utf-8")
+
+def _rewrite_test(state, fn):
+    """Try to read and rewrite *fn* and return the code object."""
+    try:
+        source = fn.read("rb")
+    except EnvironmentError:
+        return None
+    # On Python versions which are not 2.7 and less than or equal to 3.1, the
+    # parser expects *nix newlines.
+    if REWRITE_NEWLINES:
+        source = source.replace(RN, N) + N
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        state.trace("failed to parse: %r" % (fn,))
+        return None
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, fn.strpath, "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the
+        # assertion rewriting, but I don't know of a fast way to tell.
+        state.trace("failed to compile: %r" % (fn,))
+        return None
+    return co
+
+def _make_rewritten_pyc(state, fn, pyc, co):
+    """Try to dump rewritten code to *pyc*."""
+    if sys.platform.startswith("win"):
+        # Windows grants exclusive access to open files and doesn't have atomic
+        # rename, so just write into the final file.
+        _write_pyc(co, fn, pyc)
+    else:
+        # When not on windows, assume rename is atomic. Dump the code object
+        # into a file specific to this process and atomically replace it.
+        proc_pyc = pyc + "." + str(os.getpid())
+        if _write_pyc(co, fn, proc_pyc):
+            os.rename(proc_pyc, pyc)
+
+def _read_pyc(source, pyc):
+    """Possibly read a py.test pyc containing rewritten code.
+
+    Return rewritten code if successful or None if not.
+    """
+    try:
+        fp = open(pyc, "rb")
+    except IOError:
+        return None
+    try:
+        try:
+            mtime = int(source.mtime())
+            data = fp.read(8)
+        except EnvironmentError:
+            return None
+        # Check for invalid or out of date pyc file.
+        if (len(data) != 8 or
+            data[:4] != imp.get_magic() or
+            struct.unpack("<l", data[4:])[0] != mtime):
+            return None
+        co = marshal.load(fp)
+        if not isinstance(co, types.CodeType):
+            # That's interesting....
+            return None
+        return co
+    finally:
+        fp.close()
+
+
 def rewrite_asserts(mod):
     """Rewrite the assert statements in mod."""
     AssertionRewriter().run(mod)
@@ -17,13 +261,8 @@
 _saferepr = py.io.saferepr
 from _pytest.assertion.util import format_explanation as _format_explanation
 
-def _format_boolop(operands, explanations, is_or):
-    show_explanations = []
-    for operand, expl in zip(operands, explanations):
-        show_explanations.append(expl)
-        if operand == is_or:
-            break
-    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+def _format_boolop(explanations, is_or):
+    return "(" + (is_or and " or " or " and ").join(explanations) + ")"
 
 def _call_reprcompare(ops, results, expls, each_obj):
     for i, res, expl in zip(range(len(ops)), results, expls):
@@ -109,8 +348,8 @@
                     return
                 lineno += len(doc) - 1
                 expect_docstring = False
-            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
-                  item.identifier != "__future__"):
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or
+                  item.module != "__future__"):
                 lineno = item.lineno
                 break
             pos += 1
@@ -118,9 +357,9 @@
                    for alias in aliases]
         mod.body[pos:pos] = imports
         # Collect asserts.
-        nodes = collections.deque([mod])
+        nodes = [mod]
         while nodes:
-            node = nodes.popleft()
+            node = nodes.pop()
             for name, field in ast.iter_fields(node):
                 if isinstance(field, list):
                     new = []
@@ -143,7 +382,7 @@
         """Get a new variable."""
         # Use a character invalid in python identifiers to avoid clashing.
         name = "@py_assert" + str(next(self.variable_counter))
-        self.variables.add(name)
+        self.variables.append(name)
         return name
 
     def assign(self, expr):
@@ -198,7 +437,8 @@
             # There's already a message. Don't mess with it.
             return [assert_]
         self.statements = []
-        self.variables = set()
+        self.cond_chain = ()
+        self.variables = []
         self.variable_counter = itertools.count()
         self.stack = []
         self.on_failure = []
@@ -220,11 +460,11 @@
         else:
             raise_ = ast.Raise(exc, None, None)
         body.append(raise_)
-        # Delete temporary variables.
-        names = [ast.Name(name, ast.Del()) for name in self.variables]
-        if names:
-            delete = ast.Delete(names)
-            self.statements.append(delete)
+        # Clear temporary variables by setting them to None.
+        if self.variables:
+            variables = [ast.Name(name, ast.Store()) for name in self.variables]
+            clear = ast.Assign(variables, ast.Name("None", ast.Load()))
+            self.statements.append(clear)
         # Fix line numbers.
         for stmt in self.statements:
             set_location(stmt, assert_.lineno, assert_.col_offset)
@@ -240,21 +480,38 @@
         return name, self.explanation_param(expr)
 
     def visit_BoolOp(self, boolop):
-        operands = []
-        explanations = []
+        res_var = self.variable()
+        expl_list = self.assign(ast.List([], ast.Load()))
+        app = ast.Attribute(expl_list, "append", ast.Load())
+        is_or = int(isinstance(boolop.op, ast.Or))
+        body = save = self.statements
+        fail_save = self.on_failure
+        levels = len(boolop.values) - 1
         self.push_format_context()
-        for operand in boolop.values:
-            res, explanation = self.visit(operand)
-            operands.append(res)
-            explanations.append(explanation)
-        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
-        is_or = ast.Num(isinstance(boolop.op, ast.Or))
-        expl_template = self.helper("format_boolop",
-                                    ast.Tuple(operands, ast.Load()), expls,
-                                    is_or)
+        # Process each operand, short-circuting if needed.
+        for i, v in enumerate(boolop.values):
+            if i:
+                fail_inner = []
+                self.on_failure.append(ast.If(cond, fail_inner, []))
+                self.on_failure = fail_inner
+            self.push_format_context()
+            res, expl = self.visit(v)
+            body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
+            expl_format = self.pop_format_context(ast.Str(expl))
+            call = ast.Call(app, [expl_format], [], None, None)
+            self.on_failure.append(ast.Expr(call))
+            if i < levels:
+                cond = res
+                if is_or:
+                    cond = ast.UnaryOp(ast.Not(), cond)
+                inner = []
+                self.statements.append(ast.If(cond, inner, []))
+                self.statements = body = inner
+        self.statements = save
+        self.on_failure = fail_save
+        expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or))
         expl = self.pop_format_context(expl_template)
-        res = self.assign(ast.BoolOp(boolop.op, operands))
-        return res, self.explanation_param(expl)
+        return ast.Name(res_var, ast.Load()), self.explanation_param(expl)
 
     def visit_UnaryOp(self, unary):
         pattern = unary_map[unary.op.__class__]
@@ -288,7 +545,7 @@
             new_star, expl = self.visit(call.starargs)
             arg_expls.append("*" + expl)
         if call.kwargs:
-            new_kwarg, expl = self.visit(call.kwarg)
+            new_kwarg, expl = self.visit(call.kwargs)
             arg_expls.append("**" + expl)
         expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
         new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/_pytest/capture.py b/_pytest/capture.py
--- a/_pytest/capture.py
+++ b/_pytest/capture.py
@@ -11,22 +11,22 @@
     group._addoption('-s', action="store_const", const="no", dest="capture",
         help="shortcut for --capture=no.")
 
+ at pytest.mark.tryfirst
+def pytest_cmdline_parse(pluginmanager, args):
+    # we want to perform capturing already for plugin/conftest loading
+    if '-s' in args or "--capture=no" in args:
+        method = "no"
+    elif hasattr(os, 'dup') and '--capture=sys' not in args:
+        method = "fd"
+    else:
+        method = "sys"
+    capman = CaptureManager(method)
+    pluginmanager.register(capman, "capturemanager")
+
 def addouterr(rep, outerr):
-    repr = getattr(rep, 'longrepr', None)
-    if not hasattr(repr, 'addsection'):
-        return
     for secname, content in zip(["out", "err"], outerr):
         if content:
-            repr.addsection("Captured std%s" % secname, content.rstrip())
-
-def pytest_unconfigure(config):
-    # registered in config.py during early conftest.py loading
-    capman = config.pluginmanager.getplugin('capturemanager')
-    while capman._method2capture:
-        name, cap = capman._method2capture.popitem()
-        # XXX logging module may wants to close it itself on process exit
-        # otherwise we could do finalization here and call "reset()".
-        cap.suspend()
+            rep.sections.append(("Captured std%s" % secname, content))
 
 class NoCapture:
     def startall(self):
@@ -39,8 +39,9 @@
         return "", ""
 
 class CaptureManager:
-    def __init__(self):
+    def __init__(self, defaultmethod=None):
         self._method2capture = {}
+        self._defaultmethod = defaultmethod
 
     def _maketempfile(self):
         f = py.std.tempfile.TemporaryFile()
@@ -65,14 +66,6 @@
         else:
             raise ValueError("unknown capturing method: %r" % method)
 
-    def _getmethod_preoptionparse(self, args):
-        if '-s' in args or "--capture=no" in args:
-            return "no"
-        elif hasattr(os, 'dup') and '--capture=sys' not in args:
-            return "fd"
-        else:
-            return "sys"
-
     def _getmethod(self, config, fspath):
         if config.option.capture:
             method = config.option.capture
@@ -85,16 +78,22 @@
             method = "sys"
         return method
 
+    def reset_capturings(self):
+        for name, cap in self._method2capture.items():
+            cap.reset()
+
     def resumecapture_item(self, item):
         method = self._getmethod(item.config, item.fspath)
         if not hasattr(item, 'outerr'):
             item.outerr = ('', '') # we accumulate outerr on the item
         return self.resumecapture(method)
 
-    def resumecapture(self, method):
+    def resumecapture(self, method=None):
         if hasattr(self, '_capturing'):
             raise ValueError("cannot resume, already capturing with %r" %
                 (self._capturing,))
+        if method is None:
+            method = self._defaultmethod
         cap = self._method2capture.get(method)
         self._capturing = method
         if cap is None:
@@ -164,17 +163,6 @@
     def pytest_runtest_teardown(self, item):
         self.resumecapture_item(item)
 
-    def pytest__teardown_final(self, __multicall__, session):
-        method = self._getmethod(session.config, None)
-        self.resumecapture(method)
-        try:
-            rep = __multicall__.execute()
-        finally:
-            outerr = self.suspendcapture()
-        if rep:
-            addouterr(rep, outerr)
-        return rep
-
     def pytest_keyboard_interrupt(self, excinfo):
         if hasattr(self, '_capturing'):
             self.suspendcapture()
diff --git a/_pytest/config.py b/_pytest/config.py
--- a/_pytest/config.py
+++ b/_pytest/config.py
@@ -8,13 +8,15 @@
 def pytest_cmdline_parse(pluginmanager, args):
     config = Config(pluginmanager)
     config.parse(args)
-    if config.option.debug:
-        config.trace.root.setwriter(sys.stderr.write)
     return config
 
 def pytest_unconfigure(config):
-    for func in config._cleanup:
-        func()
+    while 1:
+        try:
+            fin = config._cleanup.pop()
+        except IndexError:
+            break
+        fin()
 
 class Parser:
     """ Parser for command line arguments. """
@@ -81,6 +83,7 @@
         self._inidict[name] = (help, type, default)
         self._ininames.append(name)
 
+
 class OptionGroup:
     def __init__(self, name, description="", parser=None):
         self.name = name
@@ -256,11 +259,14 @@
         self.hook = self.pluginmanager.hook
         self._inicache = {}
         self._cleanup = []
-    
+
     @classmethod
     def fromdictargs(cls, option_dict, args):
         """ constructor useable for subprocesses. """
         config = cls()
+        # XXX slightly crude way to initialize capturing
+        import _pytest.capture
+        _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args)
         config._preparse(args, addopts=False)
         config.option.__dict__.update(option_dict)
         for x in config.option.plugins:
@@ -285,11 +291,10 @@
 
     def _setinitialconftest(self, args):
         # capture output during conftest init (#issue93)
-        from _pytest.capture import CaptureManager
-        capman = CaptureManager()
-        self.pluginmanager.register(capman, 'capturemanager')
-        # will be unregistered in capture.py's unconfigure()
-        capman.resumecapture(capman._getmethod_preoptionparse(args))
+        # XXX introduce load_conftest hook to avoid needing to know
+        # about capturing plugin here
+        capman = self.pluginmanager.getplugin("capturemanager")
+        capman.resumecapture()
         try:
             try:
                 self._conftest.setinitial(args)
@@ -334,6 +339,7 @@
         # Note that this can only be called once per testing process.
         assert not hasattr(self, 'args'), (
                 "can only parse cmdline args at most once per Config object")
+        self._origargs = args
         self._preparse(args)
         self._parser.hints.extend(self.pluginmanager._hints)
         args = self._parser.parse_setoption(args, self.option)
@@ -341,6 +347,14 @@
             args.append(py.std.os.getcwd())
         self.args = args
 
+    def addinivalue_line(self, name, line):
+        """ add a line to an ini-file option. The option must have been
+        declared but might not yet be set in which case the line becomes the
+        the first line in its value. """
+        x = self.getini(name)
+        assert isinstance(x, list)
+        x.append(line) # modifies the cached list inline
+
     def getini(self, name):
         """ return configuration value from an ini file. If the
         specified name hasn't been registered through a prior ``parse.addini``
@@ -422,7 +436,7 @@
 
 
 def getcfg(args, inibasenames):
-    args = [x for x in args if str(x)[0] != "-"]
+    args = [x for x in args if not str(x).startswith("-")]
     if not args:
         args = [py.path.local()]
     for arg in args:
diff --git a/_pytest/core.py b/_pytest/core.py
--- a/_pytest/core.py
+++ b/_pytest/core.py
@@ -16,11 +16,10 @@
  "junitxml resultlog doctest").split()
 
 class TagTracer:
-    def __init__(self, prefix="[pytest] "):
+    def __init__(self):
         self._tag2proc = {}
         self.writer = None
         self.indent = 0
-        self.prefix = prefix
 
     def get(self, name):
         return TagTracerSub(self, (name,))
@@ -30,7 +29,7 @@
             if args:
                 indent = "  " * self.indent
                 content = " ".join(map(str, args))
-                self.writer("%s%s%s\n" %(self.prefix, indent, content))
+                self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags)))
         try:
             self._tag2proc[tags](tags, args)
         except KeyError:
@@ -212,6 +211,14 @@
             self.register(mod, modname)
             self.consider_module(mod)
 
+    def pytest_configure(self, config):
+        config.addinivalue_line("markers",
+            "tryfirst: mark a hook implementation function such that the "
+            "plugin machinery will try to call it first/as early as possible.")
+        config.addinivalue_line("markers",
+            "trylast: mark a hook implementation function such that the "
+            "plugin machinery will try to call it last/as late as possible.")
+
     def pytest_plugin_registered(self, plugin):
         import pytest
         dic = self.call_plugin(plugin, "pytest_namespace", {}) or {}
@@ -432,10 +439,7 @@
 def _preloadplugins():
     _preinit.append(PluginManager(load=True))
 
-def main(args=None, plugins=None):
-    """ returned exit code integer, after an in-process testing run
-    with the given command line arguments, preloading an optional list
-    of passed in plugin objects. """
+def _prepareconfig(args=None, plugins=None):
     if args is None:
         args = sys.argv[1:]
     elif isinstance(args, py.path.local):
@@ -449,13 +453,19 @@
     else: # subsequent calls to main will create a fresh instance
         _pluginmanager = PluginManager(load=True)
     hook = _pluginmanager.hook
+    if plugins:
+        for plugin in plugins:
+            _pluginmanager.register(plugin)
+    return hook.pytest_cmdline_parse(
+            pluginmanager=_pluginmanager, args=args)
+
+def main(args=None, plugins=None):
+    """ returned exit code integer, after an in-process testing run
+    with the given command line arguments, preloading an optional list
+    of passed in plugin objects. """
     try:
-        if plugins:
-            for plugin in plugins:
-                _pluginmanager.register(plugin)
-        config = hook.pytest_cmdline_parse(
-                pluginmanager=_pluginmanager, args=args)
-        exitstatus = hook.pytest_cmdline_main(config=config)
+        config = _prepareconfig(args, plugins)
+        exitstatus = config.hook.pytest_cmdline_main(config=config)
     except UsageError:
         e = sys.exc_info()[1]
         sys.stderr.write("ERROR: %s\n" %(e.args[0],))
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -1,7 +1,7 @@
 """ version info, help messages, tracing configuration.  """
 import py
 import pytest
-import inspect, sys
+import os, inspect, sys
 from _pytest.core import varnames
 
 def pytest_addoption(parser):
@@ -18,7 +18,29 @@
                help="trace considerations of conftest.py files."),
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
-               help="generate and show internal debugging information.")
+               help="store internal tracing debug information in 'pytestdebug.log'.")
+
+
+def pytest_cmdline_parse(__multicall__):
+    config = __multicall__.execute()
+    if config.option.debug:
+        path = os.path.abspath("pytestdebug.log")
+        f = open(path, 'w')
+        config._debugfile = f
+        f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %(
+            pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)),
+            os.getcwd(), config._origargs))
+        config.trace.root.setwriter(f.write)
+        sys.stderr.write("writing pytestdebug information to %s\n" % path)
+    return config
+
+ at pytest.mark.trylast
+def pytest_unconfigure(config):
+    if hasattr(config, '_debugfile'):
+        config._debugfile.close()
+        sys.stderr.write("wrote pytestdebug information to %s\n" %
+            config._debugfile.name)
+        config.trace.root.setwriter(None)
 
 
 def pytest_cmdline_main(config):
@@ -34,6 +56,7 @@
     elif config.option.help:
         config.pluginmanager.do_configure(config)
         showhelp(config)
+        config.pluginmanager.do_unconfigure(config)
         return 0
 
 def showhelp(config):
@@ -91,7 +114,7 @@
         verinfo = getpluginversioninfo(config)
         if verinfo:
             lines.extend(verinfo)
-            
+
     if config.option.traceconfig:
         lines.append("active plugins:")
         plugins = []
diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py
--- a/_pytest/hookspec.py
+++ b/_pytest/hookspec.py
@@ -121,16 +121,23 @@
 def pytest_itemstart(item, node=None):
     """ (deprecated, use pytest_runtest_logstart). """
 
-def pytest_runtest_protocol(item):
-    """ implements the standard runtest_setup/call/teardown protocol including
-    capturing exceptions and calling reporting hooks on the results accordingly.
+def pytest_runtest_protocol(item, nextitem):
+    """ implements the runtest_setup/call/teardown protocol for
+    the given test item, including capturing exceptions and calling
+    reporting hooks.
+
+    :arg item: test item for which the runtest protocol is performed.
+
+    :arg nexitem: the scheduled-to-be-next test item (or None if this
+                  is the end my friend).  This argument is passed on to
+                  :py:func:`pytest_runtest_teardown`.
 
     :return boolean: True if no further hook implementations should be invoked.
     """
 pytest_runtest_protocol.firstresult = True
 
 def pytest_runtest_logstart(nodeid, location):
-    """ signal the start of a test run. """
+    """ signal the start of running a single test item. """
 
 def pytest_runtest_setup(item):
     """ called before ``pytest_runtest_call(item)``. """
@@ -138,8 +145,14 @@
 def pytest_runtest_call(item):
     """ called to execute the test ``item``. """
 
-def pytest_runtest_teardown(item):
-    """ called after ``pytest_runtest_call``. """
+def pytest_runtest_teardown(item, nextitem):
+    """ called after ``pytest_runtest_call``.
+
+    :arg nexitem: the scheduled-to-be-next test item (None if no further
+                  test item is scheduled).  This argument can be used to
+                  perform exact teardowns, i.e. calling just enough finalizers
+                  so that nextitem only needs to call setup-functions.
+    """
 
 def pytest_runtest_makereport(item, call):
     """ return a :py:class:`_pytest.runner.TestReport` object
@@ -149,15 +162,8 @@
 pytest_runtest_makereport.firstresult = True
 
 def pytest_runtest_logreport(report):
-    """ process item test report. """
-
-# special handling for final teardown - somewhat internal for now
-def pytest__teardown_final(session):
-    """ called before test session finishes. """
-pytest__teardown_final.firstresult = True
-
-def pytest__teardown_final_logerror(report, session):
-    """ called if runtest_teardown_final failed. """
+    """ process a test setup/call/teardown report relating to
+    the respective phase of executing a test. """
 
 # -------------------------------------------------------------------------
 # test session related hooks
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -25,21 +25,39 @@
     long = int
 
 
+class Junit(py.xml.Namespace):
+    pass
+
+
 # We need to get the subset of the invalid unicode ranges according to
 # XML 1.0 which are valid in this python build.  Hence we calculate
 # this dynamically instead of hardcoding it.  The spec range of valid
 # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
 #                    | [#x10000-#x10FFFF]
-_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19),
-                   (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)]
-_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high))
-                  for (low, high) in _illegal_unichrs
+_legal_chars = (0x09, 0x0A, 0x0d)
+_legal_ranges = (
+    (0x20, 0xD7FF),
+    (0xE000, 0xFFFD),
+    (0x10000, 0x10FFFF),
+)
+_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high))
+                  for (low, high) in _legal_ranges
                   if low < sys.maxunicode]
-illegal_xml_re = re.compile(unicode('[%s]') %
-                            unicode('').join(_illegal_ranges))
-del _illegal_unichrs
-del _illegal_ranges
+_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
+illegal_xml_re = re.compile(unicode('[^%s]') %
+                            unicode('').join(_legal_xml_re))
+del _legal_chars
+del _legal_ranges
+del _legal_xml_re
 
+def bin_xml_escape(arg):
+    def repl(matchobj):
+        i = ord(matchobj.group())
+        if i <= 0xFF:
+            return unicode('#x%02X') % i
+        else:
+            return unicode('#x%04X') % i
+    return illegal_xml_re.sub(repl, py.xml.escape(arg))
 
 def pytest_addoption(parser):
     group = parser.getgroup("terminal reporting")
@@ -68,117 +86,97 @@
         logfile = os.path.expanduser(os.path.expandvars(logfile))
         self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
-        self.test_logs = []
+        self.tests = []
         self.passed = self.skipped = 0
         self.failed = self.errors = 0
-        self._durations = {}
 
     def _opentestcase(self, report):
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
-        names = tuple(names)
-        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
             classnames.insert(0, self.prefix)
-        d['classname'] = ".".join(classnames)
-        d['name'] = py.xml.escape(names[-1])
-        attrs = ['%s="%s"' % item for item in sorted(d.items())]
-        self.test_logs.append("\n<testcase %s>" % " ".join(attrs))
+        self.tests.append(Junit.testcase(
+            classname=".".join(classnames),
+            name=names[-1],
+            time=getattr(report, 'duration', 0)
+        ))
 
-    def _closetestcase(self):
-        self.test_logs.append("</testcase>")
-
-    def appendlog(self, fmt, *args):
-        def repl(matchobj):
-            i = ord(matchobj.group())
-            if i <= 0xFF:
-                return unicode('#x%02X') % i
-            else:
-                return unicode('#x%04X') % i
-        args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg))
-                      for arg in args])
-        self.test_logs.append(fmt % args)
+    def append(self, obj):
+        self.tests[-1].append(obj)
 
     def append_pass(self, report):
         self.passed += 1
-        self._opentestcase(report)
-        self._closetestcase()
 
     def append_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="xfail-marked test passes unexpectedly"/>')
+            self.append(
+                Junit.skipped(message="xfail-marked test passes unexpectedly"))
             self.skipped += 1
         else:
-            self.appendlog('<failure message="test failure">%s</failure>',
-                report.longrepr)
+            sec = dict(report.sections)
+            fail = Junit.failure(message="test failure")
+            fail.append(str(report.longrepr))
+            self.append(fail)
+            for name in ('out', 'err'):
+                content = sec.get("Captured std%s" % name)
+                if content:
+                    tag = getattr(Junit, 'system-'+name)
+                    self.append(tag(bin_xml_escape(content)))
             self.failed += 1
-        self._closetestcase()
 
     def append_collect_failure(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<failure message="collection failure">%s</failure>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.failure(str(report.longrepr),
+                                  message="collection failure"))
         self.errors += 1
 
     def append_collect_skipped(self, report):
-        self._opentestcase(report)
         #msg = str(report.longrepr.reprtraceback.extraline)
-        self.appendlog('<skipped message="collection skipped">%s</skipped>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.skipped(str(report.longrepr),
+                                  message="collection skipped"))
         self.skipped += 1
 
     def append_error(self, report):
-        self._opentestcase(report)
-        self.appendlog('<error message="test setup failure">%s</error>',
-            report.longrepr)
-        self._closetestcase()
+        self.append(Junit.error(str(report.longrepr),
+                                message="test setup failure"))
         self.errors += 1
 
     def append_skipped(self, report):
-        self._opentestcase(report)
         if "xfail" in report.keywords:
-            self.appendlog(
-                '<skipped message="expected test failure">%s</skipped>',
-                report.keywords['xfail'])
+            self.append(Junit.skipped(str(report.keywords['xfail']),
+                                      message="expected test failure"))
         else:
             filename, lineno, skipreason = report.longrepr
             if skipreason.startswith("Skipped: "):
                 skipreason = skipreason[9:]
-            self.appendlog('<skipped type="pytest.skip" '
-                           'message="%s">%s</skipped>',
-                skipreason, "%s:%s: %s" % report.longrepr,
-                )
-        self._closetestcase()
+            self.append(
+                Junit.skipped("%s:%s: %s" % report.longrepr,
+                              type="pytest.skip",
+                              message=skipreason
+                ))
         self.skipped += 1
 
     def pytest_runtest_logreport(self, report):
         if report.passed:
-            self.append_pass(report)
+            if report.when == "call": # ignore setup/teardown
+                self._opentestcase(report)
+                self.append_pass(report)
         elif report.failed:
+            self._opentestcase(report)
             if report.when != "call":
                 self.append_error(report)
             else:
                 self.append_failure(report)
         elif report.skipped:
+            self._opentestcase(report)
             self.append_skipped(report)
 
-    def pytest_runtest_call(self, item, __multicall__):
-        start = time.time()
-        try:
-            return __multicall__.execute()
-        finally:
-            self._durations[item.nodeid] = time.time() - start
-
     def pytest_collectreport(self, report):
         if not report.passed:
+            self._opentestcase(report)
             if report.failed:
                 self.append_collect_failure(report)
             else:
@@ -187,10 +185,11 @@
     def pytest_internalerror(self, excrepr):
         self.errors += 1
         data = py.xml.escape(excrepr)
-        self.test_logs.append(
-            '\n<testcase classname="pytest" name="internal">'
-            '    <error message="internal error">'
-            '%s</error></testcase>' % data)
+        self.tests.append(
+            Junit.testcase(
+                    Junit.error(data, message="internal error"),
+                    classname="pytest",
+                    name="internal"))
 
     def pytest_sessionstart(self, session):
         self.suite_start_time = time.time()
@@ -204,17 +203,17 @@
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
         numtests = self.passed + self.failed
+
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
-        logfile.write('<testsuite ')
-        logfile.write('name="" ')
-        logfile.write('errors="%i" ' % self.errors)
-        logfile.write('failures="%i" ' % self.failed)
-        logfile.write('skips="%i" ' % self.skipped)
-        logfile.write('tests="%i" ' % numtests)
-        logfile.write('time="%.3f"' % suite_time_delta)
-        logfile.write(' >')
-        logfile.writelines(self.test_logs)
-        logfile.write('</testsuite>')
+        logfile.write(Junit.testsuite(
+            self.tests,
+            name="",
+            errors=self.errors,
+            failures=self.failed,
+            skips=self.skipped,
+            tests=numtests,
+            time="%.3f" % suite_time_delta,
+        ).unicode(indent=0))
         logfile.close()
 
     def pytest_terminal_summary(self, terminalreporter):
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -2,7 +2,7 @@
 
 import py
 import pytest, _pytest
-import os, sys
+import os, sys, imp
 tracebackcutdir = py.path.local(_pytest.__file__).dirpath()
 
 # exitcodes for the command line
@@ -11,6 +11,8 @@
 EXIT_INTERRUPTED = 2
 EXIT_INTERNALERROR = 3
 
+name_re = py.std.re.compile("^[a-zA-Z_]\w*$")
+
 def pytest_addoption(parser):
     parser.addini("norecursedirs", "directory patterns to avoid for recursion",
         type="args", default=('.*', 'CVS', '_darcs', '{arch}'))
@@ -27,6 +29,9 @@
                action="store", type="int", dest="maxfail", default=0,
                help="exit after first num failures or errors.")
 
+    group._addoption('--strict', action="store_true",
+               help="run pytest in strict mode, warnings become errors.")
+
     group = parser.getgroup("collect", "collection")
     group.addoption('--collectonly',
         action="store_true", dest="collectonly",
@@ -48,7 +53,7 @@
 def pytest_namespace():
     collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
     return dict(collect=collect)
-        
+
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
@@ -77,11 +82,11 @@
         session.exitstatus = EXIT_INTERNALERROR
         if excinfo.errisinstance(SystemExit):
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus or (session._testsfailed and 1))
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    if initstate >= 2:
-        config.hook.pytest_sessionfinish(session=session,
-            exitstatus=session.exitstatus)
     if initstate >= 1:
         config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
@@ -101,8 +106,12 @@
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
         return True
-    for item in session.session.items:
-        item.config.hook.pytest_runtest_protocol(item=item)
+    for i, item in enumerate(session.items):
+        try:
+            nextitem = session.items[i+1]
+        except IndexError:
+            nextitem = None
+        item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         if session.shouldstop:
             raise session.Interrupted(session.shouldstop)
     return True
@@ -132,7 +141,7 @@
         return getattr(pytest, name)
     return property(fget, None, None,
         "deprecated attribute %r, use pytest.%s" % (name,name))
-    
+
 class Node(object):
     """ base class for all Nodes in the collection tree.
     Collector subclasses have children, Items are terminal nodes."""
@@ -143,13 +152,13 @@
 
         #: the parent collector node.
         self.parent = parent
-        
+
         #: the test config object
         self.config = config or parent.config
 
         #: the collection this node is part of
         self.session = session or parent.session
-        
+
         #: filesystem path where this node was collected from
         self.fspath = getattr(parent, 'fspath', None)
         self.ihook = self.session.gethookproxy(self.fspath)
@@ -224,13 +233,13 @@
     def listchain(self):
         """ return list of all parent collectors up to self,
             starting from root of collection tree. """
-        l = [self]
-        while 1:
-            x = l[0]
-            if x.parent is not None: # and x.parent.parent is not None:
-                l.insert(0, x.parent)
-            else:
-                return l
+        chain = []
+        item = self
+        while item is not None:
+            chain.append(item)
+            item = item.parent
+        chain.reverse()
+        return chain
 
     def listnames(self):
         return [x.name for x in self.listchain()]
@@ -325,6 +334,8 @@
     """ a basic test invocation item. Note that for a single function
     there might be multiple test invocation items.
     """
+    nextitem = None
+
     def reportinfo(self):
         return self.fspath, None, ""
 
@@ -399,6 +410,7 @@
         self._notfound = []
         self._initialpaths = set()
         self._initialparts = []
+        self.items = items = []
         for arg in args:
             parts = self._parsearg(arg)
             self._initialparts.append(parts)
@@ -414,7 +426,6 @@
         if not genitems:
             return rep.result
         else:
-            self.items = items = []
             if rep.passed:
                 for node in rep.result:
                     self.items.extend(self.genitems(node))
@@ -469,16 +480,29 @@
         return True
 
     def _tryconvertpyarg(self, x):
-        try:
-            mod = __import__(x, None, None, ['__doc__'])
-        except (ValueError, ImportError):
-            return x
-        p = py.path.local(mod.__file__)
-        if p.purebasename == "__init__":
-            p = p.dirpath()
-        else:
-            p = p.new(basename=p.purebasename+".py")
-        return str(p)
+        mod = None
+        path = [os.path.abspath('.')] + sys.path
+        for name in x.split('.'):
+            # ignore anything that's not a proper name here
+            # else something like --pyargs will mess up '.'
+            # since imp.find_module will actually sometimes work for it
+            # but it's supposed to be considered a filesystem path
+            # not a package
+            if name_re.match(name) is None:
+                return x
+            try:
+                fd, mod, type_ = imp.find_module(name, path)
+            except ImportError:
+                return x
+            else:
+                if fd is not None:
+                    fd.close()
+
+            if type_[2] != imp.PKG_DIRECTORY:
+                path = [os.path.dirname(mod)]
+            else:
+                path = [mod]
+        return mod
 
     def _parsearg(self, arg):
         """ return (fspath, names) tuple after checking the file exists. """
@@ -496,7 +520,7 @@
             raise pytest.UsageError(msg + arg)
         parts[0] = path
         return parts
-   
+
     def matchnodes(self, matching, names):
         self.trace("matchnodes", matching, names)
         self.trace.root.indent += 1
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -14,12 +14,37 @@
              "Terminate expression with ':' to make the first match match "
              "all subsequent tests (usually file-order). ")
 
+    group._addoption("-m",
+        action="store", dest="markexpr", default="", metavar="MARKEXPR",
+        help="only run tests matching given mark expression.  "
+             "example: -m 'mark1 and not mark2'."
+             )
+
+    group.addoption("--markers", action="store_true", help=
+        "show markers (builtin, plugin and per-project ones).")
+
+    parser.addini("markers", "markers for test functions", 'linelist')
+
+def pytest_cmdline_main(config):
+    if config.option.markers:
+        config.pluginmanager.do_configure(config)
+        tw = py.io.TerminalWriter()
+        for line in config.getini("markers"):
+            name, rest = line.split(":", 1)
+            tw.write("@pytest.mark.%s:" %  name, bold=True)
+            tw.line(rest)
+            tw.line()
+        config.pluginmanager.do_unconfigure(config)
+        return 0
+pytest_cmdline_main.tryfirst = True
+
 def pytest_collection_modifyitems(items, config):
     keywordexpr = config.option.keyword
-    if not keywordexpr:
+    matchexpr = config.option.markexpr
+    if not keywordexpr and not matchexpr:
         return
     selectuntil = False
-    if keywordexpr[-1] == ":":
+    if keywordexpr[-1:] == ":":
         selectuntil = True
         keywordexpr = keywordexpr[:-1]
 
@@ -29,21 +54,38 @@
         if keywordexpr and skipbykeyword(colitem, keywordexpr):
             deselected.append(colitem)
         else:
-            remaining.append(colitem)
             if selectuntil:
                 keywordexpr = None
+            if matchexpr:
+                if not matchmark(colitem, matchexpr):
+                    deselected.append(colitem)
+                    continue
+            remaining.append(colitem)
 
     if deselected:
         config.hook.pytest_deselected(items=deselected)
         items[:] = remaining
 
+class BoolDict:
+    def __init__(self, mydict):
+        self._mydict = mydict
+    def __getitem__(self, name):
+        return name in self._mydict
+
+def matchmark(colitem, matchexpr):
+    return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__))
+
+def pytest_configure(config):
+    if config.option.strict:
+        pytest.mark._config = config
+
 def skipbykeyword(colitem, keywordexpr):
     """ return True if they given keyword expression means to
         skip this collector/item.
     """
     if not keywordexpr:
         return
-    
+
     itemkeywords = getkeywords(colitem)
     for key in filter(None, keywordexpr.split()):
         eor = key[:1] == '-'
@@ -77,15 +119,31 @@
          @py.test.mark.slowtest
          def test_function():
             pass
-  
+
     will set a 'slowtest' :class:`MarkInfo` object
     on the ``test_function`` object. """
 
     def __getattr__(self, name):
         if name[0] == "_":
             raise AttributeError(name)
+        if hasattr(self, '_config'):
+            self._check(name)
         return MarkDecorator(name)
 
+    def _check(self, name):
+        try:
+            if name in self._markers:
+                return
+        except AttributeError:
+            pass
+        self._markers = l = set()
+        for line in self._config.getini("markers"):
+            beginning = line.split(":", 1)
+            x = beginning[0].split("(", 1)[0]
+            l.add(x)
+        if name not in self._markers:
+            raise AttributeError("%r not a registered marker" % (name,))
+
 class MarkDecorator:
     """ A decorator for test functions and test classes.  When applied
     it will create :class:`MarkInfo` objects which may be
@@ -133,8 +191,7 @@
                         holder = MarkInfo(self.markname, self.args, self.kwargs)
                         setattr(func, self.markname, holder)
                     else:
-                        holder.kwargs.update(self.kwargs)
-                        holder.args += self.args
+                        holder.add(self.args, self.kwargs)
                 return func
         kw = self.kwargs.copy()
         kw.update(kwargs)
@@ -150,27 +207,20 @@
         self.args = args
         #: keyword argument dictionary, empty if nothing specified
         self.kwargs = kwargs
+        self._arglist = [(args, kwargs.copy())]
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
                 self.name, self.args, self.kwargs)
 
-def pytest_itemcollected(item):
-    if not isinstance(item, pytest.Function):
-        return
-    try:
-        func = item.obj.__func__
-    except AttributeError:
-        func = getattr(item.obj, 'im_func', item.obj)
-    pyclasses = (pytest.Class, pytest.Module)
-    for node in item.listchain():
-        if isinstance(node, pyclasses):
-            marker = getattr(node.obj, 'pytestmark', None)
-            if marker is not None:
-                if isinstance(marker, list):
-                    for mark in marker:
-                        mark(func)
-                else:
-                    marker(func)
-        node = node.parent
-    item.keywords.update(py.builtin._getfuncdict(func))
+    def add(self, args, kwargs):
+        """ add a MarkInfo with the given args and kwargs. """
+        self._arglist.append((args, kwargs))
+        self.args += args
+        self.kwargs.update(kwargs)
+
+    def __iter__(self):
+        """ yield MarkInfo objects each relating to a marking-call. """
+        for args, kwargs in self._arglist:
+            yield MarkInfo(self.name, args, kwargs)
+
diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py
--- a/_pytest/monkeypatch.py
+++ b/_pytest/monkeypatch.py
@@ -13,6 +13,7 @@
         monkeypatch.setenv(name, value, prepend=False)
         monkeypatch.delenv(name, value, raising=True)
         monkeypatch.syspath_prepend(path)
+        monkeypatch.chdir(path)
 
     All modifications will be undone after the requesting
     test function has finished. The ``raising``
@@ -30,6 +31,7 @@
     def __init__(self):
         self._setattr = []
         self._setitem = []
+        self._cwd = None
 
     def setattr(self, obj, name, value, raising=True):
         """ set attribute ``name`` on ``obj`` to ``value``, by default
@@ -83,6 +85,17 @@
             self._savesyspath = sys.path[:]
         sys.path.insert(0, str(path))
 
+    def chdir(self, path):
+        """ change the current working directory to the specified path
+        path can be a string or a py.path.local object
+        """
+        if self._cwd is None:
+            self._cwd = os.getcwd()
+        if hasattr(path, "chdir"):
+            path.chdir()
+        else:
+            os.chdir(path)
+
     def undo(self):
         """ undo previous changes.  This call consumes the
         undo stack.  Calling it a second time has no effect unless
@@ -95,9 +108,17 @@
         self._setattr[:] = []
         for dictionary, name, value in self._setitem:
             if value is notset:
-                del dictionary[name]
+                try:
+                    del dictionary[name]
+                except KeyError:
+                    pass # was already deleted, so we have the desired state
             else:
                 dictionary[name] = value
         self._setitem[:] = []
         if hasattr(self, '_savesyspath'):
             sys.path[:] = self._savesyspath
+            del self._savesyspath
+
+        if self._cwd is not None:
+            os.chdir(self._cwd)
+            self._cwd = None
diff --git a/_pytest/nose.py b/_pytest/nose.py
--- a/_pytest/nose.py
+++ b/_pytest/nose.py
@@ -13,6 +13,7 @@
             call.excinfo = call2.excinfo
 
 
+ at pytest.mark.trylast
 def pytest_runtest_setup(item):
     if isinstance(item, (pytest.Function)):
         if isinstance(item.parent, pytest.Generator):
diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py
--- a/_pytest/pastebin.py
+++ b/_pytest/pastebin.py
@@ -38,7 +38,11 @@
         del tr._tw.__dict__['write']
 
 def getproxy():
-    return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes
+    if sys.version_info < (3, 0):
+        from xmlrpclib import ServerProxy
+    else:
+        from xmlrpc.client import ServerProxy
+    return ServerProxy(url.xmlrpc).pastes
 
 def pytest_terminal_summary(terminalreporter):
     if terminalreporter.config.option.pastebin != "failed":
diff --git a/_pytest/pdb.py b/_pytest/pdb.py
--- a/_pytest/pdb.py
+++ b/_pytest/pdb.py
@@ -19,11 +19,13 @@
 class pytestPDB:
     """ Pseudo PDB that defers to the real pdb. """
     item = None
+    collector = None
 
     def set_trace(self):
         """ invoke PDB set_trace debugging, dropping any IO capturing. """
         frame = sys._getframe().f_back
-        item = getattr(self, 'item', None)
+        item = self.item or self.collector
+
         if item is not None:
             capman = item.config.pluginmanager.getplugin("capturemanager")
             out, err = capman.suspendcapture()
@@ -38,6 +40,14 @@
     pytestPDB.item = item
 pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem
 
+ at pytest.mark.tryfirst
+def pytest_make_collect_report(__multicall__, collector):
+    try:
+        pytestPDB.collector = collector
+        return __multicall__.execute()
+    finally:
+        pytestPDB.collector = None
+
 def pytest_runtest_makereport():
     pytestPDB.item = None
     
@@ -60,7 +70,13 @@
         tw.sep(">", "traceback")
         rep.toterminal(tw)
         tw.sep(">", "entering PDB")
-        post_mortem(call.excinfo._excinfo[2])
+        # A doctest.UnexpectedException is not useful for post_mortem.
+        # Use the underlying exception instead:
+        if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException):
+            tb = call.excinfo.value.exc_info[2]
+        else:
+            tb = call.excinfo._excinfo[2]
+        post_mortem(tb)
         rep._pdbshown = True
         return rep
 
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -25,6 +25,7 @@
         _pytest_fullpath
     except NameError:
         _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc"))
+        _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py")
 
 def pytest_funcarg___pytest(request):
     return PytestArg(request)
@@ -313,16 +314,6 @@
             result.extend(session.genitems(colitem))
         return result
 
-    def inline_genitems(self, *args):
-        #config = self.parseconfig(*args)
-        config = self.parseconfigure(*args)
-        rec = self.getreportrecorder(config)
-        session = Session(config)
-        config.hook.pytest_sessionstart(session=session)
-        session.perform_collect()
-        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
-        return session.items, rec
-
     def runitem(self, source):
         # used from runner functional tests
         item = self.getitem(source)
@@ -343,64 +334,57 @@
         l = list(args) + [p]
         reprec = self.inline_run(*l)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 1, reports
-        return reports[0]
+        assert len(reports) == 3, reports # setup/call/teardown
+        return reports[1]
+
+    def inline_genitems(self, *args):
+        return self.inprocess_run(list(args) + ['--collectonly'])
 
     def inline_run(self, *args):
-        args = ("-s", ) + args # otherwise FD leakage
-        config = self.parseconfig(*args)
-        reprec = self.getreportrecorder(config)
-        #config.pluginmanager.do_configure(config)
-        config.hook.pytest_cmdline_main(config=config)
-        #config.pluginmanager.do_unconfigure(config)
-        return reprec
+        items, rec = self.inprocess_run(args)
+        return rec
 
-    def config_preparse(self):
-        config = self.Config()
-        for plugin in self.plugins:
-            if isinstance(plugin, str):
-                config.pluginmanager.import_plugin(plugin)
-            else:
-                if isinstance(plugin, dict):
-                    plugin = PseudoPlugin(plugin)
-                if not config.pluginmanager.isregistered(plugin):
-                    config.pluginmanager.register(plugin)
-        return config
+    def inprocess_run(self, args, plugins=None):
+        rec = []
+        items = []
+        class Collect:
+            def pytest_configure(x, config):
+                rec.append(self.getreportrecorder(config))
+            def pytest_itemcollected(self, item):
+                items.append(item)
+        if not plugins:
+            plugins = []
+        plugins.append(Collect())
+        ret = self.pytestmain(list(args), plugins=[Collect()])
+        reprec = rec[0]
+        reprec.ret = ret
+        assert len(rec) == 1
+        return items, reprec
 
     def parseconfig(self, *args):
-        if not args:
-            args = (self.tmpdir,)
-        config = self.config_preparse()
-        args = list(args)
+        args = [str(x) for x in args]
         for x in args:
             if str(x).startswith('--basetemp'):
                 break
         else:
             args.append("--basetemp=%s" % self.tmpdir.dirpath('basetemp'))
-        config.parse(args)
+        import _pytest.core
+        config = _pytest.core._prepareconfig(args, self.plugins)
+        # the in-process pytest invocation needs to avoid leaking FDs
+        # so we register a "reset_capturings" callmon the capturing manager
+        # and make sure it gets called
+        config._cleanup.append(
+            config.pluginmanager.getplugin("capturemanager").reset_capturings)
+        import _pytest.config
+        self.request.addfinalizer(
+            lambda: _pytest.config.pytest_unconfigure(config))
         return config
 
-    def reparseconfig(self, args=None):
-        """ this is used from tests that want to re-invoke parse(). """
-        if not args:
-            args = [self.tmpdir]
-        oldconfig = getattr(py.test, 'config', None)
-        try:
-            c = py.test.config = self.Config()
-            c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
-                keep=0, rootdir=self.tmpdir, lock_timeout=None)
-            c.parse(args)
-            c.pluginmanager.do_configure(c)
-            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
-            return c
-        finally:
-            py.test.config = oldconfig
-
     def parseconfigure(self, *args):
         config = self.parseconfig(*args)
         config.pluginmanager.do_configure(config)
         self.request.addfinalizer(lambda:
-            config.pluginmanager.do_unconfigure(config))
+        config.pluginmanager.do_unconfigure(config))
         return config
 
     def getitem(self,  source, funcname="test_func"):
@@ -420,7 +404,6 @@
             self.makepyfile(__init__ = "#")
         self.config = config = self.parseconfigure(path, *configargs)
         node = self.getnode(config, path)
-        #config.pluginmanager.do_unconfigure(config)
         return node
 
     def collect_by_name(self, modcol, name):
@@ -437,9 +420,16 @@
         return py.std.subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw)
 
     def pytestmain(self, *args, **kwargs):
-        ret = pytest.main(*args, **kwargs)
-        if ret == 2:
-            raise KeyboardInterrupt()
+        class ResetCapturing:
+            @pytest.mark.trylast
+            def pytest_unconfigure(self, config):
+                capman = config.pluginmanager.getplugin("capturemanager")
+                capman.reset_capturings()
+        plugins = kwargs.setdefault("plugins", [])
+        rc = ResetCapturing()
+        plugins.append(rc)
+        return pytest.main(*args, **kwargs)
+
     def run(self, *cmdargs):
         return self._run(*cmdargs)
 
@@ -528,6 +518,8 @@
         pexpect = py.test.importorskip("pexpect", "2.4")
         if hasattr(sys, 'pypy_version_info') and '64' in py.std.platform.machine():
             pytest.skip("pypy-64 bit not supported")
+        if sys.platform == "darwin":
+            pytest.xfail("pexpect does not work reliably on darwin?!")
         logfile = self.tmpdir.join("spawn.out")
         child = pexpect.spawn(cmd, logfile=logfile.open("w"))
         child.timeout = expect_timeout
@@ -540,10 +532,6 @@
             return "INTERNAL not-utf8-decodeable, truncated string:\n%s" % (
                     py.io.saferepr(out),)
 
-class PseudoPlugin:
-    def __init__(self, vars):
-        self.__dict__.update(vars)
-
 class ReportRecorder(object):
     def __init__(self, hook):
         self.hook = hook
@@ -565,10 +553,17 @@
     def getreports(self, names="pytest_runtest_logreport pytest_collectreport"):
         return [x.report for x in self.getcalls(names)]
 
-    def matchreport(self, inamepart="", names="pytest_runtest_logreport pytest_collectreport", when=None):
+    def matchreport(self, inamepart="",
+        names="pytest_runtest_logreport pytest_collectreport", when=None):
         """ return a testreport whose dotted import path matches """
         l = []
         for rep in self.getreports(names=names):
+            try:
+                if not when and rep.when != "call" and rep.passed:
+                    # setup/teardown passing reports - let's ignore those
+                    continue
+            except AttributeError:
+                pass
             if when and getattr(rep, 'when', None) != when:
                 continue
             if not inamepart or inamepart in rep.nodeid.split("::"):
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -4,6 +4,7 @@
 import sys
 import pytest
 from py._code.code import TerminalRepr
+from _pytest.monkeypatch import monkeypatch
 
 import _pytest
 cutdir = py.path.local(_pytest.__file__).dirpath()
@@ -26,6 +27,24 @@
         showfuncargs(config)
         return 0
 
+
+def pytest_generate_tests(metafunc):
+    try:
+        param = metafunc.function.parametrize
+    except AttributeError:
+        return
+    for p in param:
+        metafunc.parametrize(*p.args, **p.kwargs)
+
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "parametrize(argnames, argvalues): call a test function multiple "
+        "times passing in multiple different argument value sets. Example: "
+        "@parametrize('arg1', [1,2]) would lead to two calls of the decorated "
+        "test function, one with arg1=1 and another with arg1=2."
+    )
+
+
 @pytest.mark.trylast
 def pytest_namespace():
     raises.Exception = pytest.fail.Exception
@@ -138,6 +157,7 @@
             obj = obj.place_as
 
         self._fslineno = py.code.getfslineno(obj)
+        assert isinstance(self._fslineno[1], int), obj
         return self._fslineno
 
     def reportinfo(self):
@@ -155,6 +175,7 @@
         else:
             fspath, lineno = self._getfslineno()
             modpath = self.getmodpath()
+        assert isinstance(lineno, int)
         return fspath, lineno, modpath
 
 class PyCollectorMixin(PyobjMixin, pytest.Collector):
@@ -200,6 +221,7 @@
         module = self.getparent(Module).obj
         clscol = self.getparent(Class)
         cls = clscol and clscol.obj or None
+        transfer_markers(funcobj, cls, module)
         metafunc = Metafunc(funcobj, config=self.config,
             cls=cls, module=module)
         gentesthook = self.config.hook.pytest_generate_tests
@@ -219,6 +241,19 @@
             l.append(function)
         return l
 
+def transfer_markers(funcobj, cls, mod):
+    # XXX this should rather be code in the mark plugin or the mark
+    # plugin should merge with the python plugin.
+    for holder in (cls, mod):
+        try:
+            pytestmark = holder.pytestmark
+        except AttributeError:
+            continue
+        if isinstance(pytestmark, list):
+            for mark in pytestmark:
+                mark(funcobj)
+        else:
+            pytestmark(funcobj)
 
 class Module(pytest.File, PyCollectorMixin):
     def _getobj(self):
@@ -226,13 +261,8 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
-        from _pytest import assertion
-        assertion.before_module_import(self)
         try:
-            try:
-                mod = self.fspath.pyimport(ensuresyspath=True)
-            finally:
-                assertion.after_module_import(self)
+            mod = self.fspath.pyimport(ensuresyspath=True)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -244,7 +274,8 @@
                 "  %s\n"
                 "which is not the same as the test file we want to collect:\n"
                 "  %s\n"
-                "HINT: use a unique basename for your test file modules"
+                "HINT: remove __pycache__ / .pyc files and/or use a "
+                "unique basename for your test file modules"
                  % e.args
             )
         #print "imported test module", mod
@@ -374,6 +405,7 @@
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 
+
 class Generator(FunctionMixin, PyCollectorMixin, pytest.Collector):
     def collect(self):
         # test generators are seen as collectors but they also
@@ -430,6 +462,7 @@
                 "yielded functions (deprecated) cannot have funcargs")
         else:
             if callspec is not None:
+                self.callspec = callspec
                 self.funcargs = callspec.funcargs or {}
                 self._genid = callspec.id
                 if hasattr(callspec, "param"):
@@ -506,15 +539,59 @@
     request._fillfuncargs()
 
 _notexists = object()
-class CallSpec:
-    def __init__(self, funcargs, id, param):
-        self.funcargs = funcargs
-        self.id = id
+
+class CallSpec2(object):
+    def __init__(self, metafunc):
+        self.metafunc = metafunc
+        self.funcargs = {}
+        self._idlist = []
+        self.params = {}
+        self._globalid = _notexists
+        self._globalid_args = set()
+        self._globalparam = _notexists
+
+    def copy(self, metafunc):
+        cs = CallSpec2(self.metafunc)
+        cs.funcargs.update(self.funcargs)
+        cs.params.update(self.params)
+        cs._idlist = list(self._idlist)
+        cs._globalid = self._globalid
+        cs._globalid_args = self._globalid_args
+        cs._globalparam = self._globalparam
+        return cs
+
+    def _checkargnotcontained(self, arg):
+        if arg in self.params or arg in self.funcargs:
+            raise ValueError("duplicate %r" %(arg,))
+
+    def getparam(self, name):
+        try:
+            return self.params[name]
+        except KeyError:
+            if self._globalparam is _notexists:
+                raise ValueError(name)
+            return self._globalparam
+
+    @property
+    def id(self):
+        return "-".join(map(str, filter(None, self._idlist)))
+
+    def setmulti(self, valtype, argnames, valset, id):
+        for arg,val in zip(argnames, valset):
+            self._checkargnotcontained(arg)
+            getattr(self, valtype)[arg] = val
+        self._idlist.append(id)
+
+    def setall(self, funcargs, id, param):
+        for x in funcargs:
+            self._checkargnotcontained(x)
+        self.funcargs.update(funcargs)
+        if id is not _notexists:
+            self._idlist.append(id)
         if param is not _notexists:
-            self.param = param
-    def __repr__(self):
-        return "<CallSpec id=%r param=%r funcargs=%r>" %(
-            self.id, getattr(self, 'param', '?'), self.funcargs)
+            assert self._globalparam is _notexists
+            self._globalparam = param
+
 
 class Metafunc:
     def __init__(self, function, config=None, cls=None, module=None):
@@ -528,31 +605,71 @@
         self._calls = []
         self._ids = py.builtin.set()
 
+    def parametrize(self, argnames, argvalues, indirect=False, ids=None):
+        """ Add new invocations to the underlying test function using the list
+        of argvalues for the given argnames.  Parametrization is performed
+        during the collection phase.  If you need to setup expensive resources
+        you may pass indirect=True and implement a funcarg factory which can
+        perform the expensive setup just before a test is actually run.
+
+        :arg argnames: an argument name or a list of argument names
+
+        :arg argvalues: a list of values for the argname or a list of tuples of
+            values for the list of argument names.
+
+        :arg indirect: if True each argvalue corresponding to an argument will
+            be passed as request.param to its respective funcarg factory so
+            that it can perform more expensive setups during the setup phase of
+            a test rather than at collection time.
+
+        :arg ids: list of string ids each corresponding to the argvalues so
+            that they are part of the test id. If no ids are provided they will
+            be generated automatically from the argvalues.
+        """
+        if not isinstance(argnames, (tuple, list)):
+            argnames = (argnames,)
+            argvalues = [(val,) for val in argvalues]
+        if not indirect:
+            #XXX should we also check for the opposite case?
+            for arg in argnames:
+                if arg not in self.funcargnames:
+                    raise ValueError("%r has no argument %r" %(self.function, arg))
+        valtype = indirect and "params" or "funcargs"
+        if not ids:
+            idmaker = IDMaker()
+            ids = list(map(idmaker, argvalues))
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for i, valset in enumerate(argvalues):
+                assert len(valset) == len(argnames)
+                newcallspec = callspec.copy(self)
+                newcallspec.setmulti(valtype, argnames, valset, ids[i])
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
     def addcall(self, funcargs=None, id=_notexists, param=_notexists):
-        """ add a new call to the underlying test function during the
-        collection phase of a test run.  Note that request.addcall() is
-        called during the test collection phase prior and independently
-        to actual test execution.  Therefore you should perform setup
-        of resources in a funcarg factory which can be instrumented
-        with the ``param``.
+        """ (deprecated, use parametrize) Add a new call to the underlying
+        test function during the collection phase of a test run.  Note that
+        request.addcall() is called during the test collection phase prior and
+        independently to actual test execution.  You should only use addcall()
+        if you need to specify multiple arguments of a test function.
 
         :arg funcargs: argument keyword dictionary used when invoking
             the test function.
 
         :arg id: used for reporting and identification purposes.  If you
-            don't supply an `id` the length of the currently
-            list of calls to the test function will be used.
+            don't supply an `id` an automatic unique id will be generated.
 
-        :arg param: will be exposed to a later funcarg factory invocation
-            through the ``request.param`` attribute.  It allows to
-            defer test fixture setup activities to when an actual
-            test is run.
+        :arg param: a parameter which will be exposed to a later funcarg factory
+            invocation through the ``request.param`` attribute.
         """
         assert funcargs is None or isinstance(funcargs, dict)
         if funcargs is not None:
             for name in funcargs:
                 if name not in self.funcargnames:
                     pytest.fail("funcarg %r not used in this function." % name)
+        else:
+            funcargs = {}
         if id is None:
             raise ValueError("id=None not allowed")
         if id is _notexists:
@@ -561,11 +678,26 @@
         if id in self._ids:
             raise ValueError("duplicate id %r" % id)
         self._ids.add(id)
-        self._calls.append(CallSpec(funcargs, id, param))
+
+        cs = CallSpec2(self)
+        cs.setall(funcargs, id, param)
+        self._calls.append(cs)
+
+class IDMaker:
+    def __init__(self):
+        self.counter = 0
+    def __call__(self, valset):
+        l = []
+        for val in valset:
+            if not isinstance(val, (int, str)):
+                val = "."+str(self.counter)
+            self.counter += 1
+            l.append(str(val))
+        return "-".join(l)
 
 class FuncargRequest:
     """ A request for function arguments from a test function.
-        
+
         Note that there is an optional ``param`` attribute in case
         there was an invocation to metafunc.addcall(param=...).
         If no such call was done in a ``pytest_generate_tests``
@@ -637,7 +769,7 @@
 
 
     def applymarker(self, marker):
-        """ apply a marker to a single test function invocation.
+        """ Apply a marker to a single test function invocation.
         This method is useful if you don't want to have a keyword/marker
         on all function invocations.
 
@@ -649,7 +781,7 @@
         self._pyfuncitem.keywords[marker.markname] = marker
 
     def cached_setup(self, setup, teardown=None, scope="module", extrakey=None):
-        """ return a testing resource managed by ``setup`` &
+        """ Return a testing resource managed by ``setup`` &
         ``teardown`` calls.  ``scope`` and ``extrakey`` determine when the
         ``teardown`` function will be called so that subsequent calls to
         ``setup`` would recreate the resource.
@@ -698,11 +830,18 @@
             self._raiselookupfailed(argname)
         funcargfactory = self._name2factory[argname].pop()
         oldarg = self._currentarg
-        self._currentarg = argname
+        mp = monkeypatch()
+        mp.setattr(self, '_currentarg', argname)
+        try:
+            param = self._pyfuncitem.callspec.getparam(argname)
+        except (AttributeError, ValueError):
+            pass
+        else:
+            mp.setattr(self, 'param', param, raising=False)
         try:
             self._funcargs[argname] = res = funcargfactory(request=self)
         finally:
-            self._currentarg = oldarg
+            mp.undo()
         return res
 
     def _getscopeitem(self, scope):
@@ -817,8 +956,7 @@
         >>> raises(ZeroDivisionError, f, x=0)
         <ExceptionInfo ...>
 
-    A third possibility is to use a string which which will
-    be executed::
+    A third possibility is to use a string to be executed::
 
         >>> raises(ZeroDivisionError, "f(0)")
         <ExceptionInfo ...>
diff --git a/_pytest/resultlog.py b/_pytest/resultlog.py
--- a/_pytest/resultlog.py
+++ b/_pytest/resultlog.py
@@ -63,6 +63,8 @@
         self.write_log_entry(testpath, lettercode, longrepr)
 
     def pytest_runtest_logreport(self, report):
+        if report.when != "call" and report.passed:
+            return
         res = self.config.hook.pytest_report_teststatus(report=report)
         code = res[1]
         if code == 'x':
@@ -89,5 +91,8 @@
             self.log_outcome(report, code, longrepr)
 
     def pytest_internalerror(self, excrepr):
-        path = excrepr.reprcrash.path
+        reprcrash = getattr(excrepr, 'reprcrash', None)
+        path = getattr(reprcrash, "path", None)
+        if path is None:
+            path = "cwd:%s" % py.path.local()
         self.write_log_entry(path, '!', str(excrepr))
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -1,6 +1,6 @@
 """ basic collect and runtest protocol implementations """
 
-import py, sys
+import py, sys, time
 from py._code.code import TerminalRepr
 
 def pytest_namespace():
@@ -14,33 +14,60 @@
 #
 # pytest plugin hooks
 
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "reporting", after="general")
+    group.addoption('--durations',
+         action="store", type="int", default=None, metavar="N",
+         help="show N slowest setup/test durations (N=0 for all)."),
+
+def pytest_terminal_summary(terminalreporter):
+    durations = terminalreporter.config.option.durations
+    if durations is None:
+        return
+    tr = terminalreporter
+    dlist = []
+    for replist in tr.stats.values():
+        for rep in replist:
+            if hasattr(rep, 'duration'):
+                dlist.append(rep)
+    if not dlist:
+        return
+    dlist.sort(key=lambda x: x.duration)
+    dlist.reverse()
+    if not durations:
+        tr.write_sep("=", "slowest test durations")
+    else:
+        tr.write_sep("=", "slowest %s test durations" % durations)
+        dlist = dlist[:durations]
+
+    for rep in dlist:
+        nodeid = rep.nodeid.replace("::()::", "::")
+        tr.write_line("%02.2fs %-8s %s" %
+            (rep.duration, rep.when, nodeid))
+
 def pytest_sessionstart(session):
     session._setupstate = SetupState()
-
-def pytest_sessionfinish(session, exitstatus):
-    hook = session.config.hook
-    rep = hook.pytest__teardown_final(session=session)
-    if rep:
-        hook.pytest__teardown_final_logerror(session=session, report=rep)
-        session.exitstatus = 1
+def pytest_sessionfinish(session):
+    session._setupstate.teardown_all()
 
 class NodeInfo:
     def __init__(self, location):
         self.location = location
 
-def pytest_runtest_protocol(item):
+def pytest_runtest_protocol(item, nextitem):
     item.ihook.pytest_runtest_logstart(
         nodeid=item.nodeid, location=item.location,
     )
-    runtestprotocol(item)
+    runtestprotocol(item, nextitem=nextitem)
     return True
 
-def runtestprotocol(item, log=True):
+def runtestprotocol(item, log=True, nextitem=None):
     rep = call_and_report(item, "setup", log)
     reports = [rep]
     if rep.passed:
         reports.append(call_and_report(item, "call", log))
-    reports.append(call_and_report(item, "teardown", log))
+    reports.append(call_and_report(item, "teardown", log,
+        nextitem=nextitem))
     return reports
 
 def pytest_runtest_setup(item):
@@ -49,16 +76,8 @@
 def pytest_runtest_call(item):
     item.runtest()
 
-def pytest_runtest_teardown(item):
-    item.session._setupstate.teardown_exact(item)
-
-def pytest__teardown_final(session):
-    call = CallInfo(session._setupstate.teardown_all, when="teardown")
-    if call.excinfo:
-        ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
-        call.excinfo.traceback = ntraceback.filter()
-        longrepr = call.excinfo.getrepr(funcargs=True)
-        return TeardownErrorReport(longrepr)
+def pytest_runtest_teardown(item, nextitem):
+    item.session._setupstate.teardown_exact(item, nextitem)
 
 def pytest_report_teststatus(report):
     if report.when in ("setup", "teardown"):
@@ -74,18 +93,18 @@
 #
 # Implementation
 
-def call_and_report(item, when, log=True):
-    call = call_runtest_hook(item, when)
+def call_and_report(item, when, log=True, **kwds):
+    call = call_runtest_hook(item, when, **kwds)
     hook = item.ihook
     report = hook.pytest_runtest_makereport(item=item, call=call)
-    if log and (when == "call" or not report.passed):
+    if log:
         hook.pytest_runtest_logreport(report=report)
     return report
 
-def call_runtest_hook(item, when):
+def call_runtest_hook(item, when, **kwds):
     hookname = "pytest_runtest_" + when
     ihook = getattr(item.ihook, hookname)
-    return CallInfo(lambda: ihook(item=item), when=when)
+    return CallInfo(lambda: ihook(item=item, **kwds), when=when)
 
 class CallInfo:
     """ Result/Exception info a function invocation. """
@@ -95,12 +114,16 @@
         #: context of invocation: one of "setup", "call",
         #: "teardown", "memocollect"
         self.when = when
+        self.start = time.time()
         try:
-            self.result = func()
-        except KeyboardInterrupt:
-            raise
-        except:
-            self.excinfo = py.code.ExceptionInfo()
+            try:
+                self.result = func()
+            except KeyboardInterrupt:
+                raise
+            except:
+                self.excinfo = py.code.ExceptionInfo()
+        finally:
+            self.stop = time.time()
 
     def __repr__(self):
         if self.excinfo:
@@ -120,6 +143,10 @@
         return s
 
 class BaseReport(object):
+
+    def __init__(self, **kw):
+        self.__dict__.update(kw)
+
     def toterminal(self, out):
         longrepr = self.longrepr
         if hasattr(self, 'node'):
@@ -139,6 +166,7 @@
 
 def pytest_runtest_makereport(item, call):
     when = call.when
+    duration = call.stop-call.start
     keywords = dict([(x,1) for x in item.keywords])
     excinfo = call.excinfo
     if not call.excinfo:
@@ -160,14 +188,15 @@
             else: # exception in setup or teardown
                 longrepr = item._repr_failure_py(excinfo)
     return TestReport(item.nodeid, item.location,
-        keywords, outcome, longrepr, when)
+                      keywords, outcome, longrepr, when,
+                      duration=duration)
 
 class TestReport(BaseReport):
     """ Basic test report object (also used for setup and teardown calls if
     they fail).
     """
     def __init__(self, nodeid, location,
-            keywords, outcome, longrepr, when):
+            keywords, outcome, longrepr, when, sections=(), duration=0, **extra):
         #: normalized collection node id
         self.nodeid = nodeid
 
@@ -179,16 +208,25 @@
         #: a name -> value dictionary containing all keywords and
         #: markers associated with a test invocation.
         self.keywords = keywords
-        
+
         #: test outcome, always one of "passed", "failed", "skipped".
         self.outcome = outcome
 
         #: None or a failure representation.
         self.longrepr = longrepr
-        
+
         #: one of 'setup', 'call', 'teardown' to indicate runtest phase.
         self.when = when
 
+        #: list of (secname, data) extra information which needs to
+        #: marshallable
+        self.sections = list(sections)
+
+        #: time it took to run just the test
+        self.duration = duration
+
+        self.__dict__.update(extra)
+
     def __repr__(self):
         return "<TestReport %r when=%r outcome=%r>" % (
             self.nodeid, self.when, self.outcome)
@@ -196,8 +234,10 @@
 class TeardownErrorReport(BaseReport):
     outcome = "failed"
     when = "teardown"
-    def __init__(self, longrepr):
+    def __init__(self, longrepr, **extra):
         self.longrepr = longrepr
+        self.sections = []
+        self.__dict__.update(extra)
 
 def pytest_make_collect_report(collector):
     call = CallInfo(collector._memocollect, "memocollect")
@@ -219,11 +259,13 @@
         getattr(call, 'result', None))
 
 class CollectReport(BaseReport):
-    def __init__(self, nodeid, outcome, longrepr, result):
+    def __init__(self, nodeid, outcome, longrepr, result, sections=(), **extra):
         self.nodeid = nodeid
         self.outcome = outcome
         self.longrepr = longrepr
         self.result = result or []
+        self.sections = list(sections)
+        self.__dict__.update(extra)
 
     @property
     def location(self):
@@ -277,20 +319,22 @@
         self._teardown_with_finalization(None)
         assert not self._finalizers
 
-    def teardown_exact(self, item):
-        if self.stack and item == self.stack[-1]:
+    def teardown_exact(self, item, nextitem):
+        needed_collectors = nextitem and nextitem.listchain() or []
+        self._teardown_towards(needed_collectors)
+
+    def _teardown_towards(self, needed_collectors):
+        while self.stack:
+            if self.stack == needed_collectors[:len(self.stack)]:
+                break
             self._pop_and_teardown()
-        else:
-            self._callfinalizers(item)
 
     def prepare(self, colitem):
         """ setup objects along the collector chain to the test-method
             and teardown previously setup objects."""
         needed_collectors = colitem.listchain()
-        while self.stack:
-            if self.stack == needed_collectors[:len(self.stack)]:
-                break
-            self._pop_and_teardown()
+        self._teardown_towards(needed_collectors)
+
         # check if the last collection node has raised an error
         for col in self.stack:
             if hasattr(col, '_prepare_exc'):
diff --git a/_pytest/skipping.py b/_pytest/skipping.py
--- a/_pytest/skipping.py
+++ b/_pytest/skipping.py
@@ -9,6 +9,21 @@
            action="store_true", dest="runxfail", default=False,
            help="run tests even if they are marked xfail")
 
+def pytest_configure(config):
+    config.addinivalue_line("markers",
+        "skipif(*conditions): skip the given test function if evaluation "
+        "of all conditions has a True value.  Evaluation happens within the "
+        "module global context. Example: skipif('sys.platform == \"win32\"') "
+        "skips the test if we are on the win32 platform. "
+    )
+    config.addinivalue_line("markers",
+        "xfail(*conditions, reason=None, run=True): mark the the test function "
+        "as an expected failure. Optionally specify a reason and run=False "
+        "if you don't even want to execute the test function. Any positional "
+        "condition strings will be evaluated (like with skipif) and if one is "
+        "False the marker will not be applied."
+    )
+
 def pytest_namespace():
     return dict(xfail=xfail)
 
@@ -117,6 +132,14 @@
 def pytest_runtest_makereport(__multicall__, item, call):
     if not isinstance(item, pytest.Function):
         return
+    # unitttest special case, see setting of _unexpectedsuccess
+    if hasattr(item, '_unexpectedsuccess'):
+        rep = __multicall__.execute()
+        if rep.when == "call":
+            # we need to translate into how py.test encodes xpass
+            rep.keywords['xfail'] = "reason: " + item._unexpectedsuccess
+            rep.outcome = "failed"
+        return rep
     if not (call.excinfo and
         call.excinfo.errisinstance(py.test.xfail.Exception)):
         evalxfail = getattr(item, '_evalxfail', None)
@@ -169,21 +192,23 @@
         elif char == "X":
             show_xpassed(terminalreporter, lines)
         elif char in "fF":
-            show_failed(terminalreporter, lines)
+            show_simple(terminalreporter, lines, 'failed', "FAIL %s")
         elif char in "sS":
             show_skipped(terminalreporter, lines)
+        elif char == "E":
+            show_simple(terminalreporter, lines, 'error', "ERROR %s")
     if lines:
         tr._tw.sep("=", "short test summary info")
         for line in lines:
             tr._tw.line(line)
 
-def show_failed(terminalreporter, lines):
+def show_simple(terminalreporter, lines, stat, format):
     tw = terminalreporter._tw
-    failed = terminalreporter.stats.get("failed")
+    failed = terminalreporter.stats.get(stat)
     if failed:
         for rep in failed:
             pos = rep.nodeid
-            lines.append("FAIL %s" %(pos, ))
+            lines.append(format %(pos, ))
 
 def show_xfailed(terminalreporter, lines):
     xfailed = terminalreporter.stats.get("xfailed")
diff --git a/_pytest/terminal.py b/_pytest/terminal.py
--- a/_pytest/terminal.py
+++ b/_pytest/terminal.py
@@ -15,7 +15,7 @@
     group._addoption('-r',
          action="store", dest="reportchars", default=None, metavar="chars",
          help="show extra test summary info as specified by chars (f)ailed, "
-              "(s)skipped, (x)failed, (X)passed.")
+              "(E)error, (s)skipped, (x)failed, (X)passed.")
     group._addoption('-l', '--showlocals',
          action="store_true", dest="showlocals", default=False,
          help="show locals in tracebacks (disabled by default).")
@@ -43,7 +43,8 @@
             pass
         else:
             stdout = os.fdopen(newfd, stdout.mode, 1)
-            config._toclose = stdout
+            config._cleanup.append(lambda: stdout.close())
+
     reporter = TerminalReporter(config, stdout)
     config.pluginmanager.register(reporter, 'terminalreporter')
     if config.option.debug or config.option.traceconfig:
@@ -52,11 +53,6 @@
             reporter.write_line("[traceconfig] " + msg)
         config.trace.root.setprocessor("pytest:config", mywriter)
 
-def pytest_unconfigure(config):
-    if hasattr(config, '_toclose'):
-        #print "closing", config._toclose, config._toclose.fileno()
-        config._toclose.close()
-
 def getreportopt(config):
     reportopts = ""
     optvalue = config.option.report
@@ -165,9 +161,6 @@
     def pytest_deselected(self, items):
         self.stats.setdefault('deselected', []).extend(items)
 
-    def pytest__teardown_final_logerror(self, report):
-        self.stats.setdefault("error", []).append(report)
-
     def pytest_runtest_logstart(self, nodeid, location):
         # ensure that the path is printed before the
         # 1st test of a module starts running
@@ -259,7 +252,7 @@
         msg = "platform %s -- Python %s" % (sys.platform, verinfo)
         if hasattr(sys, 'pypy_version_info'):
             verinfo = ".".join(map(str, sys.pypy_version_info[:3]))
-            msg += "[pypy-%s]" % verinfo
+            msg += "[pypy-%s-%s]" % (verinfo, sys.pypy_version_info[3])
         msg += " -- pytest-%s" % (py.test.__version__)
         if self.verbosity > 0 or self.config.option.debug or \
            getattr(self.config.option, 'pastebin', None):
@@ -289,10 +282,18 @@
         # we take care to leave out Instances aka ()
         # because later versions are going to get rid of them anyway
         if self.config.option.verbose < 0:
-            for item in items:
-                nodeid = item.nodeid
-                nodeid = nodeid.replace("::()::", "::")
-                self._tw.line(nodeid)
+            if self.config.option.verbose < -1:
+                counts = {}
+                for item in items:
+                    name = item.nodeid.split('::', 1)[0]
+                    counts[name] = counts.get(name, 0) + 1
+                for name, count in sorted(counts.items()):
+                    self._tw.line("%s: %d" % (name, count))
+            else:
+                for item in items:
+                    nodeid = item.nodeid
+                    nodeid = nodeid.replace("::()::", "::")
+                    self._tw.line(nodeid)
             return
         stack = []
         indent = ""
@@ -318,12 +319,17 @@
             self.config.hook.pytest_terminal_summary(terminalreporter=self)
         if exitstatus == 2:
             self._report_keyboardinterrupt()
+            del self._keyboardinterrupt_memo
         self.summary_deselected()
         self.summary_stats()
 
     def pytest_keyboard_interrupt(self, excinfo):
         self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True)
 
+    def pytest_unconfigure(self):
+        if hasattr(self, '_keyboardinterrupt_memo'):
+            self._report_keyboardinterrupt()
+
     def _report_keyboardinterrupt(self):
         excrepr = self._keyboardinterrupt_memo
         msg = excrepr.reprcrash.message
@@ -388,7 +394,7 @@
                 else:
                     msg = self._getfailureheadline(rep)
                     self.write_sep("_", msg)
-                    rep.toterminal(self._tw)
+                    self._outrep_summary(rep)
 
     def summary_errors(self):
         if self.config.option.tbstyle != "no":
@@ -406,7 +412,15 @@
                 elif rep.when == "teardown":
                     msg = "ERROR at teardown of " + msg
                 self.write_sep("_", msg)
-                rep.toterminal(self._tw)
+                self._outrep_summary(rep)
+
+    def _outrep_summary(self, rep):
+        rep.toterminal(self._tw)
+        for secname, content in rep.sections:
+            self._tw.sep("-", secname)
+            if content[-1:] == "\n":
+                content = content[:-1]
+            self._tw.line(content)
 
     def summary_stats(self):
         session_duration = py.std.time.time() - self._sessionstarttime
@@ -417,9 +431,10 @@
                 keys.append(key)
         parts = []
         for key in keys:
-            val = self.stats.get(key, None)
-            if val:
-                parts.append("%d %s" %(len(val), key))
+            if key: # setup/teardown reports have an empty key, ignore them
+                val = self.stats.get(key, None)
+                if val:
+                    parts.append("%d %s" %(len(val), key))
         line = ", ".join(parts)
         # XXX coloring
         msg = "%s in %.2f seconds" %(line, session_duration)
@@ -430,8 +445,15 @@
 
     def summary_deselected(self):
         if 'deselected' in self.stats:
+            l = []
+            k = self.config.option.keyword
+            if k:
+                l.append("-k%s" % k)
+            m = self.config.option.markexpr
+            if m:
+                l.append("-m %r" % m)
             self.write_sep("=", "%d tests deselected by %r" %(
-                len(self.stats['deselected']), self.config.option.keyword), bold=True)
+                len(self.stats['deselected']), " ".join(l)), bold=True)
 
 def repr_pythonversion(v=None):
     if v is None:
diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py
--- a/_pytest/tmpdir.py
+++ b/_pytest/tmpdir.py
@@ -46,7 +46,7 @@
 
     def finish(self):
         self.trace("finish")
-        
+
 def pytest_configure(config):
     mp = monkeypatch()
     t = TempdirHandler(config)
@@ -64,5 +64,5 @@
     name = request._pyfuncitem.name
     name = py.std.re.sub("[\W]", "_", name)
     x = request.config._tmpdirhandler.mktemp(name, numbered=True)
-    return x.realpath()
+    return x
 
diff --git a/_pytest/unittest.py b/_pytest/unittest.py
--- a/_pytest/unittest.py
+++ b/_pytest/unittest.py
@@ -2,6 +2,9 @@
 import pytest, py
 import sys, pdb
 
+# for transfering markers
+from _pytest.python import transfer_markers
+
 def pytest_pycollect_makeitem(collector, name, obj):
     unittest = sys.modules.get('unittest')
     if unittest is None:
@@ -19,7 +22,14 @@
 class UnitTestCase(pytest.Class):
     def collect(self):
         loader = py.std.unittest.TestLoader()
+        module = self.getparent(pytest.Module).obj
+        cls = self.obj
         for name in loader.getTestCaseNames(self.obj):
+            x = getattr(self.obj, name)
+            funcobj = getattr(x, 'im_func', x)
+            transfer_markers(funcobj, cls, module)
+            if hasattr(funcobj, 'todo'):
+                pytest.mark.xfail(reason=str(funcobj.todo))(funcobj)
             yield TestCaseFunction(name, parent=self)
 
     def setup(self):
@@ -37,15 +47,13 @@
 class TestCaseFunction(pytest.Function):
     _excinfo = None
 
-    def __init__(self, name, parent):
-        super(TestCaseFunction, self).__init__(name, parent)
-        if hasattr(self._obj, 'todo'):
-            getattr(self._obj, 'im_func', self._obj).xfail = \
-                pytest.mark.xfail(reason=str(self._obj.todo))
-
     def setup(self):
         self._testcase = self.parent.obj(self.name)
         self._obj = getattr(self._testcase, self.name)
+        if hasattr(self._testcase, 'skip'):
+            pytest.skip(self._testcase.skip)
+        if hasattr(self._obj, 'skip'):
+            pytest.skip(self._obj.skip)
         if hasattr(self._testcase, 'setup_method'):
             self._testcase.setup_method(self._obj)
 
@@ -83,28 +91,37 @@
         self._addexcinfo(rawexcinfo)
     def addFailure(self, testcase, rawexcinfo):
         self._addexcinfo(rawexcinfo)
+
     def addSkip(self, testcase, reason):
         try:
             pytest.skip(reason)
         except pytest.skip.Exception:
             self._addexcinfo(sys.exc_info())
-    def addExpectedFailure(self, testcase, rawexcinfo, reason):
+
+    def addExpectedFailure(self, testcase, rawexcinfo, reason=""):
         try:
             pytest.xfail(str(reason))
         except pytest.xfail.Exception:
             self._addexcinfo(sys.exc_info())
-    def addUnexpectedSuccess(self, testcase, reason):
-        pass
+
+    def addUnexpectedSuccess(self, testcase, reason=""):
+        self._unexpectedsuccess = reason
+
     def addSuccess(self, testcase):
         pass
+
     def stopTest(self, testcase):
         pass
+
     def runtest(self):
         self._testcase(result=self)
 
     def _prunetraceback(self, excinfo):
         pytest.Function._prunetraceback(self, excinfo)
-        excinfo.traceback = excinfo.traceback.filter(lambda x:not x.frame.f_globals.get('__unittest'))
+        traceback = excinfo.traceback.filter(
+            lambda x:not x.frame.f_globals.get('__unittest'))
+        if traceback:
+            excinfo.traceback = traceback
 
 @pytest.mark.tryfirst
 def pytest_runtest_makereport(item, call):
@@ -120,14 +137,19 @@
             ut = sys.modules['twisted.python.failure']
             Failure__init__ = ut.Failure.__init__.im_func
             check_testcase_implements_trial_reporter()
-            def excstore(self, exc_value=None, exc_type=None, exc_tb=None):
+            def excstore(self, exc_value=None, exc_type=None, exc_tb=None,
+                captureVars=None):
                 if exc_value is None:
                     self._rawexcinfo = sys.exc_info()
                 else:
                     if exc_type is None:
                         exc_type = type(exc_value)
                     self._rawexcinfo = (exc_type, exc_value, exc_tb)
-                Failure__init__(self, exc_value, exc_type, exc_tb)
+                try:
+                    Failure__init__(self, exc_value, exc_type, exc_tb,
+                        captureVars=captureVars)
+                except TypeError:
+                    Failure__init__(self, exc_value, exc_type, exc_tb)
             ut.Failure.__init__ = excstore
             try:
                 return __multicall__.execute()
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -93,6 +93,7 @@
     return result
 
 def parse_plain(graph_id, plaincontent, links={}, fixedfont=False):
+    plaincontent = plaincontent.replace('\r\n', '\n')    # fix Windows EOL
     lines = plaincontent.splitlines(True)
     for i in range(len(lines)-2, -1, -1):
         if lines[i].endswith('\\\n'):   # line ending in '\'
diff --git a/lib-python/modified-2.7/test/test_set.py b/lib-python/modified-2.7/test/test_set.py
--- a/lib-python/modified-2.7/test/test_set.py
+++ b/lib-python/modified-2.7/test/test_set.py
@@ -1568,7 +1568,7 @@
             for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint):
                 for g in (G, I, Ig, L, R):
                     expected = meth(data)
-                    actual = meth(G(data))
+                    actual = meth(g(data))
                     if isinstance(expected, bool):
                         self.assertEqual(actual, expected)
                     else:
diff --git a/lib_pypy/_locale.py b/lib_pypy/_locale.py
deleted file mode 100644
--- a/lib_pypy/_locale.py
+++ /dev/null
@@ -1,337 +0,0 @@
-# ctypes implementation of _locale module by Victor Stinner, 2008-03-27
-
-# ------------------------------------------------------------
-#  Note that we also have our own interp-level implementation
-# ------------------------------------------------------------
-
-"""
-Support for POSIX locales.
-"""
-
-from ctypes import (Structure, POINTER, create_string_buffer,
-    c_ubyte, c_int, c_char_p, c_wchar_p, c_size_t)
-from ctypes_support import standard_c_lib as libc
-from ctypes_support import get_errno
-
-# load the platform-specific cache made by running locale.ctc.py
-from ctypes_config_cache._locale_cache import *
-
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
-
-
-# Ubuntu Gusty i386 structure
-class lconv(Structure):
-    _fields_ = (
-        # Numeric (non-monetary) information.
-        ("decimal_point", c_char_p),    # Decimal point character.
-        ("thousands_sep", c_char_p),    # Thousands separator.
-
-        # Each element is the number of digits in each group;
-        # elements with higher indices are farther left.
-        # An element with value CHAR_MAX means that no further grouping is done.
-        # An element with value 0 means that the previous element is used
-        # for all groups farther left.  */
-        ("grouping", c_char_p),
-
-        # Monetary information.
-
-        # First three chars are a currency symbol from ISO 4217.
-        # Fourth char is the separator.  Fifth char is '\0'.
-        ("int_curr_symbol", c_char_p),
-        ("currency_symbol", c_char_p),   # Local currency symbol.
-        ("mon_decimal_point", c_char_p), # Decimal point character.
-        ("mon_thousands_sep", c_char_p), # Thousands separator.
-        ("mon_grouping", c_char_p),      # Like `grouping' element (above).
-        ("positive_sign", c_char_p),     # Sign for positive values.
-        ("negative_sign", c_char_p),     # Sign for negative values.
-        ("int_frac_digits", c_ubyte),    # Int'l fractional digits.
-        ("frac_digits", c_ubyte),        # Local fractional digits.
-        # 1 if currency_symbol precedes a positive value, 0 if succeeds.
-        ("p_cs_precedes", c_ubyte),
-        # 1 iff a space separates currency_symbol from a positive value.
-        ("p_sep_by_space", c_ubyte),
-        # 1 if currency_symbol precedes a negative value, 0 if succeeds.
-        ("n_cs_precedes", c_ubyte),
-        # 1 iff a space separates currency_symbol from a negative value.
-        ("n_sep_by_space", c_ubyte),
-
-        # Positive and negative sign positions:
-        # 0 Parentheses surround the quantity and currency_symbol.
-        # 1 The sign string precedes the quantity and currency_symbol.
-        # 2 The sign string follows the quantity and currency_symbol.
-        # 3 The sign string immediately precedes the currency_symbol.
-        # 4 The sign string immediately follows the currency_symbol.
-        ("p_sign_posn", c_ubyte),
-        ("n_sign_posn", c_ubyte),
-        # 1 if int_curr_symbol precedes a positive value, 0 if succeeds.
-        ("int_p_cs_precedes", c_ubyte),
-        # 1 iff a space separates int_curr_symbol from a positive value.
-        ("int_p_sep_by_space", c_ubyte),
-        # 1 if int_curr_symbol precedes a negative value, 0 if succeeds.
-        ("int_n_cs_precedes", c_ubyte),
-        # 1 iff a space separates int_curr_symbol from a negative value.
-        ("int_n_sep_by_space", c_ubyte),
-         # Positive and negative sign positions:
-         # 0 Parentheses surround the quantity and int_curr_symbol.
-         # 1 The sign string precedes the quantity and int_curr_symbol.
-         # 2 The sign string follows the quantity and int_curr_symbol.
-         # 3 The sign string immediately precedes the int_curr_symbol.
-         # 4 The sign string immediately follows the int_curr_symbol.
-        ("int_p_sign_posn", c_ubyte),
-        ("int_n_sign_posn", c_ubyte),
-    )
-
-_setlocale = libc.setlocale
-_setlocale.argtypes = (c_int, c_char_p)
-_setlocale.restype = c_char_p
-
-_localeconv = libc.localeconv
-_localeconv.argtypes = None
-_localeconv.restype = POINTER(lconv)
-
-_strcoll = libc.strcoll
-_strcoll.argtypes = (c_char_p, c_char_p)
-_strcoll.restype = c_int
-
-_wcscoll = libc.wcscoll
-_wcscoll.argtypes = (c_wchar_p, c_wchar_p)
-_wcscoll.restype = c_int
-
-_strxfrm = libc.strxfrm
-_strxfrm.argtypes = (c_char_p, c_char_p, c_size_t)
-_strxfrm.restype = c_size_t
-
-HAS_LIBINTL = hasattr(libc, 'gettext')
-if HAS_LIBINTL:
-    _gettext = libc.gettext
-    _gettext.argtypes = (c_char_p,)
-    _gettext.restype = c_char_p
-
-    _dgettext = libc.dgettext
-    _dgettext.argtypes = (c_char_p, c_char_p)
-    _dgettext.restype = c_char_p
-
-    _dcgettext = libc.dcgettext
-    _dcgettext.argtypes = (c_char_p, c_char_p, c_int)
-    _dcgettext.restype = c_char_p
-
-    _textdomain = libc.textdomain
-    _textdomain.argtypes = (c_char_p,)
-    _textdomain.restype = c_char_p
-
-    _bindtextdomain = libc.bindtextdomain
-    _bindtextdomain.argtypes = (c_char_p, c_char_p)
-    _bindtextdomain.restype = c_char_p
-
-    HAS_BIND_TEXTDOMAIN_CODESET = hasattr(libc, 'bindtextdomain_codeset')
-    if HAS_BIND_TEXTDOMAIN_CODESET:
-        _bind_textdomain_codeset = libc.bindtextdomain_codeset
-        _bind_textdomain_codeset.argtypes = (c_char_p, c_char_p)
-        _bind_textdomain_codeset.restype = c_char_p
-
-class Error(Exception):
-    pass
-
-def fixup_ulcase():
-    import string
-    #import strop
-
-    # create uppercase map string
-    ul = []
-    for c in xrange(256):
-        c = chr(c)
-        if c.isupper():
-            ul.append(c)
-    ul = ''.join(ul)
-    string.uppercase = ul
-    #strop.uppercase = ul
-
-    # create lowercase string
-    ul = []
-    for c in xrange(256):
-        c = chr(c)
-        if c.islower():
-            ul.append(c)
-    ul = ''.join(ul)
-    string.lowercase = ul
-    #strop.lowercase = ul
-
-    # create letters string
-    ul = []
-    for c in xrange(256):
-        c = chr(c)
-        if c.isalpha():
-            ul.append(c)
-    ul = ''.join(ul)
-    string.letters = ul
-
- at builtinify
-def setlocale(category, locale=None):
-    "(integer,string=None) -> string. Activates/queries locale processing."
-    if locale:
-        # set locale
-        result = _setlocale(category, locale)
-        if not result:
-            raise Error("unsupported locale setting")
-
-        # record changes to LC_CTYPE
-        if category in (LC_CTYPE, LC_ALL):
-            fixup_ulcase()
-    else:
-        # get locale
-        result = _setlocale(category, None)
-        if not result:
-            raise Error("locale query failed")
-    return result
-
-def _copy_grouping(text):
-    groups = [ ord(group) for group in text ]
-    if groups:
-        groups.append(0)
-    return groups
-
- at builtinify
-def localeconv():
-    "() -> dict. Returns numeric and monetary locale-specific parameters."
-
-    # if LC_NUMERIC is different in the C library, use saved value
-    lp = _localeconv()
-    l = lp.contents
-
-    # hopefully, the localeconv result survives the C library calls
-    # involved herein
-
-    # Numeric information
-    result = {
-        "decimal_point": l.decimal_point,
-        "thousands_sep": l.thousands_sep,
-        "grouping": _copy_grouping(l.grouping),
-        "int_curr_symbol": l.int_curr_symbol,
-        "currency_symbol": l.currency_symbol,
-        "mon_decimal_point": l.mon_decimal_point,
-        "mon_thousands_sep": l.mon_thousands_sep,
-        "mon_grouping": _copy_grouping(l.mon_grouping),
-        "positive_sign": l.positive_sign,
-        "negative_sign": l.negative_sign,
-        "int_frac_digits": l.int_frac_digits,
-        "frac_digits": l.frac_digits,
-        "p_cs_precedes": l.p_cs_precedes,
-        "p_sep_by_space": l.p_sep_by_space,
-        "n_cs_precedes": l.n_cs_precedes,
-        "n_sep_by_space": l.n_sep_by_space,
-        "p_sign_posn": l.p_sign_posn,
-        "n_sign_posn": l.n_sign_posn,
-    }
-    return result
-
- at builtinify
-def strcoll(s1, s2):
-    "string,string -> int. Compares two strings according to the locale."
-
-    # If both arguments are byte strings, use strcoll.
-    if isinstance(s1, str) and isinstance(s2, str):
-        return _strcoll(s1, s2)
-
-    # If neither argument is unicode, it's an error.
-    if not isinstance(s1, unicode) and not isinstance(s2, unicode):
-        raise ValueError("strcoll arguments must be strings")
-
-    # Convert the non-unicode argument to unicode.
-    s1 = unicode(s1)
-    s2 = unicode(s2)
-
-    # Collate the strings.
-    return _wcscoll(s1, s2)
-
- at builtinify
-def strxfrm(s):
-    "string -> string. Returns a string that behaves for cmp locale-aware."
-
-    # assume no change in size, first
-    n1 = len(s) + 1
-    buf = create_string_buffer(n1)
-    n2 = _strxfrm(buf, s, n1) + 1
-    if n2 > n1:
-        # more space needed
-        buf = create_string_buffer(n2)
-        _strxfrm(buf, s, n2)
-    return buf.value
-
- at builtinify
-def getdefaultlocale():
-    # TODO: Port code from CPython for Windows and Mac OS
-    raise NotImplementedError()
-
-if HAS_LANGINFO:
-    _nl_langinfo = libc.nl_langinfo
-    _nl_langinfo.argtypes = (nl_item,)
-    _nl_langinfo.restype = c_char_p
-
-    def nl_langinfo(key):
-        """nl_langinfo(key) -> string
-        Return the value for the locale information associated with key."""
-        # Check whether this is a supported constant. GNU libc sometimes
-        # returns numeric values in the char* return value, which would
-        # crash PyString_FromString.
-        result = _nl_langinfo(key)
-        if result is not None:
-            return result
-        raise ValueError("unsupported langinfo constant")
-
-if HAS_LIBINTL:
-    @builtinify
-    def gettext(msg):
-        """gettext(msg) -> string
-        Return translation of msg."""
-        return _gettext(msg)
-
-    @builtinify
-    def dgettext(domain, msg):
-        """dgettext(domain, msg) -> string
-        Return translation of msg in domain."""
-        return _dgettext(domain, msg)
-
-    @builtinify
-    def dcgettext(domain, msg, category):
-        """dcgettext(domain, msg, category) -> string
-        Return translation of msg in domain and category."""
-        return _dcgettext(domain, msg, category)
-
-    @builtinify
-    def textdomain(domain):
-        """textdomain(domain) -> string
-        Set the C library's textdomain to domain, returning the new domain."""
-        return _textdomain(domain)
-
-    @builtinify
-    def bindtextdomain(domain, dir):
-        """bindtextdomain(domain, dir) -> string
-        Bind the C library's domain to dir."""
-        dirname = _bindtextdomain(domain, dir)
-        if not dirname:
-            errno = get_errno()
-            raise OSError(errno)
-        return dirname
-
-    if HAS_BIND_TEXTDOMAIN_CODESET:
-        @builtinify
-        def bind_textdomain_codeset(domain, codeset):
-            """bind_textdomain_codeset(domain, codeset) -> string
-            Bind the C library's domain to codeset."""
-            codeset = _bind_textdomain_codeset(domain, codeset)
-            if codeset:
-                return codeset
-            return None
-
-__all__ = (
-    'Error',
-    'setlocale', 'localeconv', 'strxfrm', 'strcoll',
-) + ALL_CONSTANTS
-if HAS_LIBINTL:
-    __all__ += ('gettext', 'dgettext', 'dcgettext', 'textdomain',
-                'bindtextdomain')
-    if HAS_BIND_TEXTDOMAIN_CODESET:
-        __all__ += ('bind_textdomain_codeset',)
-if HAS_LANGINFO:
-    __all__ += ('nl_langinfo',)
diff --git a/lib_pypy/array.py b/lib_pypy/array.py
deleted file mode 100644
--- a/lib_pypy/array.py
+++ /dev/null
@@ -1,531 +0,0 @@
-"""This module defines an object type which can efficiently represent
-an array of basic values: characters, integers, floating point
-numbers.  Arrays are sequence types and behave very much like lists,
-except that the type of objects stored in them is constrained.  The
-type is specified at object creation time by using a type code, which
-is a single character.  The following type codes are defined:
-
-    Type code   C Type             Minimum size in bytes 
-    'c'         character          1 
-    'b'         signed integer     1 
-    'B'         unsigned integer   1 
-    'u'         Unicode character  2 
-    'h'         signed integer     2 
-    'H'         unsigned integer   2 
-    'i'         signed integer     2 
-    'I'         unsigned integer   2 
-    'l'         signed integer     4 
-    'L'         unsigned integer   4 
-    'f'         floating point     4 
-    'd'         floating point     8 
-
-The constructor is:
-
-array(typecode [, initializer]) -- create a new array
-"""
-
-from struct import calcsize, pack, pack_into, unpack_from
-import operator
-
-# the buffer-like object to use internally: trying from
-# various places in order...
-try:
-    import _rawffi                    # a reasonable implementation based
-    _RAWARRAY = _rawffi.Array('c')    # on raw_malloc, and providing a
-    def bytebuffer(size):             # real address
-        return _RAWARRAY(size, autofree=True)
-    def getbufaddress(buf):
-        return buf.buffer
-except ImportError:
-    try:
-        from __pypy__ import bytebuffer     # a reasonable implementation
-        def getbufaddress(buf):             # compatible with oo backends,
-            return 0                        # but no address
-    except ImportError:
-        # not running on PyPy.  Fall back to ctypes...
-        import ctypes
-        bytebuffer = ctypes.create_string_buffer
-        def getbufaddress(buf):
-            voidp = ctypes.cast(ctypes.pointer(buf), ctypes.c_void_p)
-            return voidp.value
-
-# ____________________________________________________________
-
-TYPECODES = "cbBuhHiIlLfd"
-
-class array(object):
-    """array(typecode [, initializer]) -> array
-    
-    Return a new array whose items are restricted by typecode, and
-    initialized from the optional initializer value, which must be a list,
-    string. or iterable over elements of the appropriate type.
-    
-    Arrays represent basic values and behave very much like lists, except
-    the type of objects stored in them is constrained.
-    
-    Methods:
-    
-    append() -- append a new item to the end of the array
-    buffer_info() -- return information giving the current memory info
-    byteswap() -- byteswap all the items of the array
-    count() -- return number of occurences of an object
-    extend() -- extend array by appending multiple elements from an iterable
-    fromfile() -- read items from a file object
-    fromlist() -- append items from the list
-    fromstring() -- append items from the string
-    index() -- return index of first occurence of an object
-    insert() -- insert a new item into the array at a provided position
-    pop() -- remove and return item (default last)
-    read() -- DEPRECATED, use fromfile()
-    remove() -- remove first occurence of an object
-    reverse() -- reverse the order of the items in the array
-    tofile() -- write all items to a file object
-    tolist() -- return the array converted to an ordinary list
-    tostring() -- return the array converted to a string
-    write() -- DEPRECATED, use tofile()
-    
-    Attributes:
-    
-    typecode -- the typecode character used to create the array
-    itemsize -- the length in bytes of one array item
-    """
-    __slots__ = ["typecode", "itemsize", "_data", "_descriptor", "__weakref__"]
-
-    def __new__(cls, typecode, initializer=[], **extrakwds):
-        self = object.__new__(cls)
-        if cls is array and extrakwds:
-            raise TypeError("array() does not take keyword arguments")
-        if not isinstance(typecode, str) or len(typecode) != 1:
-            raise TypeError(
-                     "array() argument 1 must be char, not %s" % type(typecode))
-        if typecode not in TYPECODES:
-            raise ValueError(
-                  "bad typecode (must be one of %s)" % ', '.join(TYPECODES))
-        self._data = bytebuffer(0)
-        self.typecode = typecode
-        self.itemsize = calcsize(typecode)
-        if isinstance(initializer, list):
-            self.fromlist(initializer)
-        elif isinstance(initializer, str):
-            self.fromstring(initializer)
-        elif isinstance(initializer, unicode) and self.typecode == "u":
-            self.fromunicode(initializer)
-        else:
-            self.extend(initializer)
-        return self
-
-    def _clear(self):
-        self._data = bytebuffer(0)
-
-    ##### array-specific operations
-
-    def fromfile(self, f, n):
-        """Read n objects from the file object f and append them to the end of
-        the array. Also called as read."""
-        if not isinstance(f, file):
-            raise TypeError("arg1 must be open file")
-        size = self.itemsize * n
-        item = f.read(size)
-        if len(item) < size:
-            raise EOFError("not enough items in file")
-        self.fromstring(item)
-
-    def fromlist(self, l):
-        """Append items to array from list."""
-        if not isinstance(l, list):
-            raise TypeError("arg must be list")
-        self._fromiterable(l)
-        
-    def fromstring(self, s):
-        """Appends items from the string, interpreting it as an array of machine
-        values, as if it had been read from a file using the fromfile()
-        method."""
-        if isinstance(s, unicode):
-            s = str(s)
-        self._frombuffer(s)
-
-    def _frombuffer(self, s):
-        length = len(s)
-        if length % self.itemsize != 0:
-            raise ValueError("string length not a multiple of item size")
-        boundary = len(self._data)
-        newdata = bytebuffer(boundary + length)
-        newdata[:boundary] = self._data
-        newdata[boundary:] = s
-        self._data = newdata
-
-    def fromunicode(self, ustr):
-        """Extends this array with data from the unicode string ustr. The array
-        must be a type 'u' array; otherwise a ValueError is raised. Use
-        array.fromstring(ustr.encode(...)) to append Unicode data to an array of
-        some other type."""
-        if not self.typecode == "u":
-            raise ValueError(
-                          "fromunicode() may only be called on type 'u' arrays")
-        # XXX the following probable bug is not emulated:
-        # CPython accepts a non-unicode string or a buffer, and then
-        # behaves just like fromstring(), except that it strangely truncates
-        # string arguments at multiples of the unicode byte size.
-        # Let's only accept unicode arguments for now.
-        if not isinstance(ustr, unicode):
-            raise TypeError("fromunicode() argument should probably be "
-                            "a unicode string")
-        # _frombuffer() does the currect thing using
-        # the buffer behavior of unicode objects
-        self._frombuffer(buffer(ustr))
-
-    def tofile(self, f):
-        """Write all items (as machine values) to the file object f.  Also
-        called as write."""
-        if not isinstance(f, file):
-            raise TypeError("arg must be open file")
-        f.write(self.tostring())
-        
-    def tolist(self):
-        """Convert array to an ordinary list with the same items."""
-        count = len(self._data) // self.itemsize
-        return list(unpack_from('%d%s' % (count, self.typecode), self._data))
-
-    def tostring(self):
-        return self._data[:]
-
-    def __buffer__(self):
-        return buffer(self._data)
-
-    def tounicode(self):
-        """Convert the array to a unicode string. The array must be a type 'u'
-        array; otherwise a ValueError is raised. Use array.tostring().decode()
-        to obtain a unicode string from an array of some other type."""
-        if self.typecode != "u":
-            raise ValueError("tounicode() may only be called on type 'u' arrays")
-        # XXX performance is not too good
-        return u"".join(self.tolist())
-
-    def byteswap(self):
-        """Byteswap all items of the array.  If the items in the array are not
-        1, 2, 4, or 8 bytes in size, RuntimeError is raised."""
-        if self.itemsize not in [1, 2, 4, 8]:
-            raise RuntimeError("byteswap not supported for this array")
-        # XXX slowish
-        itemsize = self.itemsize
-        bytes = self._data
-        for start in range(0, len(bytes), itemsize):
-            stop = start + itemsize
-            bytes[start:stop] = bytes[start:stop][::-1]
-
-    def buffer_info(self):
-        """Return a tuple (address, length) giving the current memory address
-        and the length in items of the buffer used to hold array's contents. The
-        length should be multiplied by the itemsize attribute to calculate the
-        buffer length in bytes. On PyPy the address might be meaningless
-        (returned as 0), depending on the available modules."""
-        return (getbufaddress(self._data), len(self))
-    
-    read = fromfile
-
-    write = tofile
-
-    ##### general object protocol
-    
-    def __repr__(self):
-        if len(self._data) == 0:
-            return "array('%s')" % self.typecode
-        elif self.typecode == "c":
-            return "array('%s', %s)" % (self.typecode, repr(self.tostring()))
-        elif self.typecode == "u":
-            return "array('%s', %s)" % (self.typecode, repr(self.tounicode()))
-        else:
-            return "array('%s', %s)" % (self.typecode, repr(self.tolist()))
-
-    def __copy__(self):
-        a = array(self.typecode)
-        a._data = bytebuffer(len(self._data))
-        a._data[:] = self._data
-        return a
-
-    def __eq__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) == buffer(other._data)
-        else:
-            return self.tolist() == other.tolist()
-
-    def __ne__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) != buffer(other._data)
-        else:
-            return self.tolist() != other.tolist()
-
-    def __lt__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) < buffer(other._data)
-        else:
-            return self.tolist() < other.tolist()
-
-    def __gt__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) > buffer(other._data)
-        else:
-            return self.tolist() > other.tolist()
-
-    def __le__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) <= buffer(other._data)
-        else:
-            return self.tolist() <= other.tolist()
-
-    def __ge__(self, other):
-        if not isinstance(other, array):
-            return NotImplemented
-        if self.typecode == 'c':
-            return buffer(self._data) >= buffer(other._data)
-        else:
-            return self.tolist() >= other.tolist()
-
-    def __reduce__(self):
-        dict = getattr(self, '__dict__', None)
-        data = self.tostring()
-        if data:
-            initargs = (self.typecode, data)
-        else:
-            initargs = (self.typecode,)
-        return (type(self), initargs, dict)
-
-    ##### list methods
-    
-    def append(self, x):
-        """Append new value x to the end of the array."""
-        self._frombuffer(pack(self.typecode, x))
-
-    def count(self, x):
-        """Return number of occurences of x in the array."""
-        return operator.countOf(self, x)
-
-    def extend(self, iterable):
-        """Append items to the end of the array."""
-        if isinstance(iterable, array) \
-                                    and not self.typecode == iterable.typecode:
-            raise TypeError("can only extend with array of same kind")
-        self._fromiterable(iterable)
-
-    def index(self, x):
-        """Return index of first occurence of x in the array."""
-        return operator.indexOf(self, x)
-    
-    def insert(self, i, x):
-        """Insert a new item x into the array before position i."""
-        seqlength = len(self)
-        if i < 0:
-            i += seqlength
-            if i < 0:
-                i = 0
-        elif i > seqlength:
-            i = seqlength
-        boundary = i * self.itemsize
-        data = pack(self.typecode, x)
-        newdata = bytebuffer(len(self._data) + len(data))
-        newdata[:boundary] = self._data[:boundary]
-        newdata[boundary:boundary+self.itemsize] = data
-        newdata[boundary+self.itemsize:] = self._data[boundary:]
-        self._data = newdata
-        
-    def pop(self, i=-1):
-        """Return the i-th element and delete it from the array. i defaults to
-        -1."""
-        seqlength = len(self)
-        if i < 0:
-            i += seqlength
-        if not (0 <= i < seqlength):
-            raise IndexError(i)
-        boundary = i * self.itemsize
-        result = unpack_from(self.typecode, self._data, boundary)[0]
-        newdata = bytebuffer(len(self._data) - self.itemsize)
-        newdata[:boundary] = self._data[:boundary]
-        newdata[boundary:] = self._data[boundary+self.itemsize:]
-        self._data = newdata
-        return result
-        
-    def remove(self, x):
-        """Remove the first occurence of x in the array."""
-        self.pop(self.index(x))
-        
-    def reverse(self):
-        """Reverse the order of the items in the array."""
-        lst = self.tolist()
-        lst.reverse()
-        self._clear()
-        self.fromlist(lst)
-
-    ##### list protocol
-    
-    def __len__(self):
-        return len(self._data) // self.itemsize
-    
-    def __add__(self, other):
-        if not isinstance(other, array):
-            raise TypeError("can only append array to array")
-        if self.typecode != other.typecode:
-            raise TypeError("bad argument type for built-in operation")
-        return array(self.typecode, buffer(self._data) + buffer(other._data))
-
-    def __mul__(self, repeat):
-        return array(self.typecode, buffer(self._data) * repeat)
-
-    __rmul__ = __mul__
-
-    def __getitem__(self, i):
-        seqlength = len(self)
-        if isinstance(i, slice):
-            start, stop, step = i.indices(seqlength)
-            if step != 1:
-                sublist = self.tolist()[i]    # fall-back
-                return array(self.typecode, sublist)
-            if start < 0:
-                start = 0
-            if stop < start:
-                stop = start
-            assert stop <= seqlength
-            return array(self.typecode, self._data[start * self.itemsize :
-                                                   stop * self.itemsize])
-        else:
-            if i < 0:
-                i += seqlength
-            if self.typecode == 'c':  # speed trick
-                return self._data[i]
-            if not (0 <= i < seqlength):
-                raise IndexError(i)
-            boundary = i * self.itemsize
-            return unpack_from(self.typecode, self._data, boundary)[0]
-
-    def __getslice__(self, i, j):
-        return self.__getitem__(slice(i, j))
-
-    def __setitem__(self, i, x):
-        if isinstance(i, slice):
-            if (not isinstance(x, array)
-                or self.typecode != x.typecode):
-                raise TypeError("can only assign array of same kind"
-                                " to array slice")
-            seqlength = len(self)
-            start, stop, step = i.indices(seqlength)
-            if step != 1:
-                sublist = self.tolist()    # fall-back
-                sublist[i] = x.tolist()
-                self._clear()
-                self.fromlist(sublist)
-                return
-            if start < 0:
-                start = 0
-            if stop < start:
-                stop = start
-            assert stop <= seqlength
-            boundary1 = start * self.itemsize
-            boundary2 = stop * self.itemsize
-            boundary2new = boundary1 + len(x._data)
-            if boundary2 == boundary2new:
-                self._data[boundary1:boundary2] = x._data
-            else:
-                newdata = bytebuffer(len(self._data) + boundary2new-boundary2)
-                newdata[:boundary1] = self._data[:boundary1]
-                newdata[boundary1:boundary2new] = x._data
-                newdata[boundary2new:] = self._data[boundary2:]
-                self._data = newdata
-        else:
-            seqlength = len(self)
-            if i < 0:
-                i += seqlength
-            if self.typecode == 'c':  # speed trick
-                self._data[i] = x
-                return
-            if not (0 <= i < seqlength):
-                raise IndexError(i)
-            boundary = i * self.itemsize
-            pack_into(self.typecode, self._data, boundary, x)
-
-    def __setslice__(self, i, j, x):
-        self.__setitem__(slice(i, j), x)
-
-    def __delitem__(self, i):
-        if isinstance(i, slice):
-            seqlength = len(self)
-            start, stop, step = i.indices(seqlength)
-            if start < 0:
-                start = 0
-            if stop < start:
-                stop = start
-            assert stop <= seqlength
-            if step != 1:
-                sublist = self.tolist()    # fall-back
-                del sublist[i]
-                self._clear()
-                self.fromlist(sublist)
-                return
-            dellength = stop - start
-            boundary1 = start * self.itemsize
-            boundary2 = stop * self.itemsize
-            newdata = bytebuffer(len(self._data) - (boundary2-boundary1))
-            newdata[:boundary1] = self._data[:boundary1]
-            newdata[boundary1:] = self._data[boundary2:]
-            self._data = newdata
-        else:            
-            seqlength = len(self)
-            if i < 0:
-                i += seqlength
-            if not (0 <= i < seqlength):
-                raise IndexError(i)
-            boundary = i * self.itemsize
-            newdata = bytebuffer(len(self._data) - self.itemsize)
-            newdata[:boundary] = self._data[:boundary]
-            newdata[boundary:] = self._data[boundary+self.itemsize:]
-            self._data = newdata
-
-    def __delslice__(self, i, j):
-        self.__delitem__(slice(i, j))
-
-    def __contains__(self, item):
-        for x in self:
-            if x == item:
-                return True
-        return False
-
-    def __iadd__(self, other):
-        if not isinstance(other, array):
-            raise TypeError("can only extend array with array")
-        self.extend(other)
-        return self
-
-    def __imul__(self, repeat):
-        newdata = buffer(self._data) * repeat
-        self._data = bytebuffer(len(newdata))
-        self._data[:] = newdata
-        return self
-
-    def __iter__(self):
-        p = 0
-        typecode = self.typecode
-        itemsize = self.itemsize
-        while p < len(self._data):
-            yield unpack_from(typecode, self._data, p)[0]
-            p += itemsize
-
-    ##### internal methods
-
-    def _fromiterable(self, iterable):
-        iterable = tuple(iterable)
-        n = len(iterable)
-        boundary = len(self._data)
-        newdata = bytebuffer(boundary + n * self.itemsize)
-        newdata[:boundary] = self._data
-        pack_into('%d%s' % (n, self.typecode), newdata, boundary, *iterable)
-        self._data = newdata
-
-ArrayType = array
diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py
deleted file mode 100644
--- a/lib_pypy/binascii.py
+++ /dev/null
@@ -1,720 +0,0 @@
-"""A pure Python implementation of binascii.
-
-Rather slow and buggy in corner cases.
-PyPy provides an RPython version too.
-"""
-
-class Error(Exception):
-    pass
-
-class Done(Exception):
-    pass
-
-class Incomplete(Exception):
-    pass
-
-def a2b_uu(s):
-    if not s:
-        return ''
-    
-    length = (ord(s[0]) - 0x20) % 64
-
-    def quadruplets_gen(s):
-        while s:
-            try:
-                yield ord(s[0]), ord(s[1]), ord(s[2]), ord(s[3])
-            except IndexError:
-                s += '   '
-                yield ord(s[0]), ord(s[1]), ord(s[2]), ord(s[3])
-                return
-            s = s[4:]
-
-    try:
-        result = [''.join(
-            [chr((A - 0x20) << 2 | (((B - 0x20) >> 4) & 0x3)),
-            chr(((B - 0x20) & 0xf) << 4 | (((C - 0x20) >> 2) & 0xf)),
-            chr(((C - 0x20) & 0x3) << 6 | ((D - 0x20) & 0x3f))
-            ]) for A, B, C, D in quadruplets_gen(s[1:].rstrip())]
-    except ValueError:
-        raise Error('Illegal char')
-    result = ''.join(result)
-    trailingdata = result[length:]
-    if trailingdata.strip('\x00'):
-        raise Error('Trailing garbage')
-    result = result[:length]
-    if len(result) < length:
-        result += ((length - len(result)) * '\x00')
-    return result
-
-                               
-def b2a_uu(s):
-    length = len(s)
-    if length > 45:
-        raise Error('At most 45 bytes at once')
-
-    def triples_gen(s):
-        while s:
-            try:
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-            except IndexError:
-                s += '\0\0'
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-                return
-            s = s[3:]
-
-    result = [''.join(
-        [chr(0x20 + (( A >> 2                    ) & 0x3F)),
-         chr(0x20 + (((A << 4) | ((B >> 4) & 0xF)) & 0x3F)),
-         chr(0x20 + (((B << 2) | ((C >> 6) & 0x3)) & 0x3F)),
-         chr(0x20 + (( C                         ) & 0x3F))])
-              for A, B, C in triples_gen(s)]
-    return chr(ord(' ') + (length & 077)) + ''.join(result) + '\n'
-
-
-table_a2b_base64 = {
-    'A': 0,
-    'B': 1,
-    'C': 2,
-    'D': 3,
-    'E': 4,
-    'F': 5,
-    'G': 6,
-    'H': 7,
-    'I': 8,
-    'J': 9,
-    'K': 10,
-    'L': 11,
-    'M': 12,
-    'N': 13,
-    'O': 14,
-    'P': 15,
-    'Q': 16,
-    'R': 17,
-    'S': 18,
-    'T': 19,
-    'U': 20,
-    'V': 21,
-    'W': 22,
-    'X': 23,
-    'Y': 24,
-    'Z': 25,
-    'a': 26,
-    'b': 27,
-    'c': 28,
-    'd': 29,
-    'e': 30,
-    'f': 31,
-    'g': 32,
-    'h': 33,
-    'i': 34,
-    'j': 35,
-    'k': 36,
-    'l': 37,
-    'm': 38,
-    'n': 39,
-    'o': 40,
-    'p': 41,
-    'q': 42,
-    'r': 43,
-    's': 44,
-    't': 45,
-    'u': 46,
-    'v': 47,
-    'w': 48,
-    'x': 49,
-    'y': 50,
-    'z': 51,
-    '0': 52,
-    '1': 53,
-    '2': 54,
-    '3': 55,
-    '4': 56,
-    '5': 57,
-    '6': 58,
-    '7': 59,
-    '8': 60,
-    '9': 61,
-    '+': 62,
-    '/': 63,
-    '=': 0,
-}
-
-
-def a2b_base64(s):
-    if not isinstance(s, (str, unicode)):
-        raise TypeError("expected string or unicode, got %r" % (s,))
-    s = s.rstrip()
-    # clean out all invalid characters, this also strips the final '=' padding
-    # check for correct padding
-
-    def next_valid_char(s, pos):
-        for i in range(pos + 1, len(s)):
-            c = s[i]
-            if c < '\x7f':
-                try:
-                    table_a2b_base64[c]
-                    return c
-                except KeyError:
-                    pass
-        return None
-    
-    quad_pos = 0
-    leftbits = 0
-    leftchar = 0
-    res = []
-    for i, c in enumerate(s):
-        if c > '\x7f' or c == '\n' or c == '\r' or c == ' ':
-            continue
-        if c == '=':
-            if quad_pos < 2 or (quad_pos == 2 and next_valid_char(s, i) != '='):
-                continue
-            else:
-                leftbits = 0
-                break
-        try:
-            next_c = table_a2b_base64[c]
-        except KeyError:
-            continue
-        quad_pos = (quad_pos + 1) & 0x03
-        leftchar = (leftchar << 6) | next_c
-        leftbits += 6
-        if leftbits >= 8:
-            leftbits -= 8
-            res.append((leftchar >> leftbits & 0xff))
-            leftchar &= ((1 << leftbits) - 1)
-    if leftbits != 0:
-        raise Error('Incorrect padding')
-    
-    return ''.join([chr(i) for i in res])
-    
-table_b2a_base64 = \
-"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
-
-def b2a_base64(s):
-    length = len(s)
-    final_length = length % 3
-
-    def triples_gen(s):
-        while s:
-            try:
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-            except IndexError:
-                s += '\0\0'
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-                return
-            s = s[3:]
-
-    
-    a = triples_gen(s[ :length - final_length])
-
-    result = [''.join(
-        [table_b2a_base64[( A >> 2                    ) & 0x3F],
-         table_b2a_base64[((A << 4) | ((B >> 4) & 0xF)) & 0x3F],
-         table_b2a_base64[((B << 2) | ((C >> 6) & 0x3)) & 0x3F],
-         table_b2a_base64[( C                         ) & 0x3F]])
-              for A, B, C in a]
-
-    final = s[length - final_length:]
-    if final_length == 0:
-        snippet = ''
-    elif final_length == 1:
-        a = ord(final[0])
-        snippet = table_b2a_base64[(a >> 2 ) & 0x3F] + \
-                  table_b2a_base64[(a << 4 ) & 0x3F] + '=='
-    else:
-        a = ord(final[0])
-        b = ord(final[1])
-        snippet = table_b2a_base64[(a >> 2) & 0x3F] + \
-                  table_b2a_base64[((a << 4) | (b >> 4) & 0xF) & 0x3F] + \
-                  table_b2a_base64[(b << 2) & 0x3F] + '='
-    return ''.join(result) + snippet + '\n'
-
-def a2b_qp(s, header=False):
-    inp = 0
-    odata = []
-    while inp < len(s):
-        if s[inp] == '=':
-            inp += 1
-            if inp >= len(s):
-                break
-            # Soft line breaks
-            if (s[inp] == '\n') or (s[inp] == '\r'):
-                if s[inp] != '\n':
-                    while inp < len(s) and s[inp] != '\n':
-                        inp += 1
-                if inp < len(s):
-                    inp += 1
-            elif s[inp] == '=':
-                # broken case from broken python qp
-                odata.append('=')
-                inp += 1
-            elif s[inp] in hex_numbers and s[inp + 1] in hex_numbers:
-                ch = chr(int(s[inp:inp+2], 16))
-                inp += 2
-                odata.append(ch)
-            else:
-                odata.append('=')
-        elif header and s[inp] == '_':
-            odata.append(' ')
-            inp += 1
-        else:
-            odata.append(s[inp])
-            inp += 1
-    return ''.join(odata)
-
-def b2a_qp(data, quotetabs=False, istext=True, header=False):
-    """quotetabs=True means that tab and space characters are always
-       quoted.
-       istext=False means that \r and \n are treated as regular characters
-       header=True encodes space characters with '_' and requires
-       real '_' characters to be quoted.
-    """
-    MAXLINESIZE = 76
-
-    # See if this string is using CRLF line ends
-    lf = data.find('\n')
-    crlf = lf > 0 and data[lf-1] == '\r'
-
-    inp = 0
-    linelen = 0
-    odata = []
-    while inp < len(data):
-        c = data[inp]
-        if (c > '~' or
-            c == '=' or
-            (header and c == '_') or
-            (c == '.' and linelen == 0 and (inp+1 == len(data) or
-                                            data[inp+1] == '\n' or
-                                            data[inp+1] == '\r')) or
-            (not istext and (c == '\r' or c == '\n')) or
-            ((c == '\t' or c == ' ') and (inp + 1 == len(data))) or
-            (c <= ' ' and c != '\r' and c != '\n' and
-             (quotetabs or (not quotetabs and (c != '\t' and c != ' '))))):
-            linelen += 3
-            if linelen >= MAXLINESIZE:
-                odata.append('=')
-                if crlf: odata.append('\r')
-                odata.append('\n')
-                linelen = 3
-            odata.append('=' + two_hex_digits(ord(c)))
-            inp += 1
-        else:
-            if (istext and
-                (c == '\n' or (inp+1 < len(data) and c == '\r' and
-                               data[inp+1] == '\n'))):
-                linelen = 0
-                # Protect against whitespace on end of line
-                if (len(odata) > 0 and
-                    (odata[-1] == ' ' or odata[-1] == '\t')):
-                    ch = ord(odata[-1])
-                    odata[-1] = '='
-                    odata.append(two_hex_digits(ch))
-
-                if crlf: odata.append('\r')
-                odata.append('\n')
-                if c == '\r':
-                    inp += 2
-                else:
-                    inp += 1
-            else:
-                if (inp + 1 < len(data) and
-                    data[inp+1] != '\n' and
-                    (linelen + 1) >= MAXLINESIZE):
-                    odata.append('=')
-                    if crlf: odata.append('\r')
-                    odata.append('\n')
-                    linelen = 0
-
-                linelen += 1
-                if header and c == ' ':
-                    c = '_'
-                odata.append(c)
-                inp += 1
-    return ''.join(odata)
-
-hex_numbers = '0123456789ABCDEF'
-def hex(n):
-    if n == 0:
-        return '0'
-    
-    if n < 0:
-        n = -n
-        sign = '-'
-    else:
-        sign = ''
-    arr = []
-
-    def hex_gen(n):
-        """ Yield a nibble at a time. """
-        while n:
-            yield n % 0x10
-            n = n / 0x10
-
-    for nibble in hex_gen(n):
-        arr = [hex_numbers[nibble]] + arr
-    return sign + ''.join(arr)
-
-def two_hex_digits(n):
-    return hex_numbers[n / 0x10] + hex_numbers[n % 0x10]
-    
-
-def strhex_to_int(s):
-    i = 0
-    for c in s:
-        i = i * 0x10 + hex_numbers.index(c)
-    return i
-
-hqx_encoding = '!"#$%&\'()*+,-012345689 at ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr'
-
-DONE = 0x7f
-SKIP = 0x7e
-FAIL = 0x7d
-    
-table_a2b_hqx = [
-    #^@    ^A    ^B    ^C    ^D    ^E    ^F    ^G   
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #\b    \t    \n    ^K    ^L    \r    ^N    ^O   
-    FAIL, FAIL, SKIP, FAIL, FAIL, SKIP, FAIL, FAIL,
-    #^P    ^Q    ^R    ^S    ^T    ^U    ^V    ^W   
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #^X    ^Y    ^Z    ^[    ^\    ^]    ^^    ^_   
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #      !     "     #     $     %     &     '   
-    FAIL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
-    #(     )     *     +     ,     -     .     /   
-    0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, FAIL, FAIL,
-    #0     1     2     3     4     5     6     7   
-    0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, FAIL,
-    #8     9     :     ;     <     =     >     ?   
-    0x14, 0x15, DONE, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #@     A     B     C     D     E     F     G   
-    0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D,
-    #H     I     J     K     L     M     N     O   
-    0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, FAIL,
-    #P     Q     R     S     T     U     V     W   
-    0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, FAIL,
-    #X     Y     Z     [     \     ]     ^     _   
-    0x2C, 0x2D, 0x2E, 0x2F, FAIL, FAIL, FAIL, FAIL,
-    #`     a     b     c     d     e     f     g   
-    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, FAIL,
-    #h     i     j     k     l     m     n     o   
-    0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, FAIL, FAIL,
-    #p     q     r     s     t     u     v     w   
-    0x3D, 0x3E, 0x3F, FAIL, FAIL, FAIL, FAIL, FAIL,
-    #x     y     z     {     |     }     ~    ^?   
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-    FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL,
-]
-
-def a2b_hqx(s):
-    result = []
-
-    def quadruples_gen(s):
-        t = []
-        for c in s:
-            res = table_a2b_hqx[ord(c)]
-            if res == SKIP:
-                continue
-            elif res == FAIL:
-                raise Error('Illegal character')
-            elif res == DONE:
-                yield t
-                raise Done
-            else:
-                t.append(res)
-            if len(t) == 4:
-                yield t
-                t = []
-        yield t
-        
-    done = 0
-    try:
-        for snippet in quadruples_gen(s):
-            length = len(snippet)
-            if length == 4:
-                result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) 
-                result.append(chr(((snippet[1] & 0x0f) << 4) | (snippet[2] >> 2))) 
-                result.append(chr(((snippet[2] & 0x03) << 6) | (snippet[3]))) 
-            elif length == 3:
-                result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) 
-                result.append(chr(((snippet[1] & 0x0f) << 4) | (snippet[2] >> 2))) 
-            elif length == 2:
-                result.append(chr(((snippet[0] & 0x3f) << 2) | (snippet[1] >> 4))) 
-    except Done:
-        done = 1
-    except Error:
-        raise
-    return (''.join(result), done)
-
-def b2a_hqx(s):
-    result =[]
-
-    def triples_gen(s):
-        while s:
-            try:
-                yield ord(s[0]), ord(s[1]), ord(s[2])
-            except IndexError:
-                yield tuple([ord(c) for c in s])
-            s = s[3:]
-
-    for snippet in triples_gen(s):
-        length = len(snippet)
-        if length == 3:
-            result.append(
-                hqx_encoding[(snippet[0] & 0xfc) >> 2])
-            result.append(hqx_encoding[
-                ((snippet[0] & 0x03) << 4) | ((snippet[1] & 0xf0) >> 4)])
-            result.append(hqx_encoding[
-                (snippet[1] & 0x0f) << 2 | ((snippet[2] & 0xc0) >> 6)])
-            result.append(hqx_encoding[snippet[2] & 0x3f])
-        elif length == 2:
-            result.append(
-                hqx_encoding[(snippet[0] & 0xfc) >> 2])
-            result.append(hqx_encoding[
-                ((snippet[0] & 0x03) << 4) | ((snippet[1] & 0xf0) >> 4)])
-            result.append(hqx_encoding[
-                (snippet[1] & 0x0f) << 2])
-        elif length == 1:
-            result.append(
-                hqx_encoding[(snippet[0] & 0xfc) >> 2])
-            result.append(hqx_encoding[
-                ((snippet[0] & 0x03) << 4)])
-    return ''.join(result)
-
-crctab_hqx = [
-        0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7,
-        0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef,
-        0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6,
-        0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de,
-        0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485,
-        0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d,
-        0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4,
-        0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc,
-        0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
-        0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b,
-        0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
-        0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a,
-        0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41,
-        0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49,
-        0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70,
-        0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78,
-        0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f,
-        0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
-        0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e,
-        0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256,
-        0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d,
-        0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
-        0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c,
-        0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634,
-        0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab,
-        0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3,
-        0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
-        0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92,
-        0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9,
-        0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1,
-        0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8,
-        0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
-]
-
-def crc_hqx(s, crc):
-    for c in s:
-        crc = ((crc << 8) & 0xff00) ^ crctab_hqx[((crc >> 8) & 0xff) ^ ord(c)]
-
-    return crc
-
-def rlecode_hqx(s):
-    """
-    Run length encoding for binhex4.
-    The CPython implementation does not do run length encoding
-    of \x90 characters. This implementation does.
-    """
-    if not s:
-        return ''
-    result = []
-    prev = s[0]
-    count = 1
-    # Add a dummy character to get the loop to go one extra round.
-    # The dummy must be different from the last character of s.
-    # In the same step we remove the first character, which has
-    # already been stored in prev.
-    if s[-1] == '!':
-        s = s[1:] + '?'
-    else:
-        s = s[1:] + '!'
-        
-    for c in s:
-        if c == prev and count < 255:
-            count += 1
-        else:
-            if count == 1:
-                if prev != '\x90':
-                    result.append(prev)
-                else:
-                    result.extend(['\x90', '\x00'])
-            elif count < 4:
-                if prev != '\x90':
-                    result.extend([prev] * count)
-                else:
-                    result.extend(['\x90', '\x00'] * count)
-            else:
-                if prev != '\x90':
-                    result.extend([prev, '\x90', chr(count)])
-                else:
-                    result.extend(['\x90', '\x00', '\x90', chr(count)]) 
-            count = 1
-            prev = c
-        
-    return ''.join(result)
-
-def rledecode_hqx(s):
-    s = s.split('\x90')
-    result = [s[0]]
-    prev = s[0]
-    for snippet in s[1:]:
-        count = ord(snippet[0])
-        if count > 0:
-            result.append(prev[-1] * (count-1))
-            prev = snippet
-        else:
-            result. append('\x90')
-            prev = '\x90'
-        result.append(snippet[1:])
-
-    return ''.join(result)
-
-crc_32_tab = [
-    0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L,
-    0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L,
-    0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L,
-    0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL,
-    0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L,
-    0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L,
-    0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L,
-    0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL,
-    0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L,
-    0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL,
-    0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L,
-    0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L,
-    0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L,
-    0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL,
-    0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL,
-    0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L,
-    0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL,
-    0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L,
-    0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L,
-    0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L,
-    0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL,
-    0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L,
-    0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L,
-    0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL,
-    0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L,
-    0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L,
-    0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L,
-    0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L,
-    0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L,
-    0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL,
-    0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL,
-    0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L,
-    0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L,
-    0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL,
-    0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL,
-    0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L,
-    0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL,
-    0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L,
-    0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL,
-    0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L,
-    0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL,
-    0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L,
-    0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L,
-    0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL,
-    0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L,
-    0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L,
-    0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L,
-    0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L,
-    0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L,
-    0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L,
-    0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL,
-    0x2d02ef8dL
-]
-
-def crc32(s, crc=0):
-    result = 0
-    crc = ~long(crc) & 0xffffffffL
-    for c in s:
-        crc = crc_32_tab[(crc ^ long(ord(c))) & 0xffL] ^ (crc >> 8)
-        #/* Note:  (crc >> 8) MUST zero fill on left
-
-    result = crc ^ 0xffffffffL
-    
-    if result > 2**31:
-        result = ((result + 2**31) % 2**32) - 2**31
-
-    return result
-
-def b2a_hex(s):
-    result = []
-    for char in s:
-        c = (ord(char) >> 4) & 0xf
-        if c > 9:
-            c = c + ord('a') - 10
-        else:
-            c = c + ord('0')
-        result.append(chr(c))
-        c = ord(char) & 0xf
-        if c > 9:
-            c = c + ord('a') - 10
-        else:
-            c = c + ord('0')
-        result.append(chr(c))
-    return ''.join(result)
-
-hexlify = b2a_hex
-
-table_hex = [
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    0, 1, 2, 3,  4, 5, 6, 7,  8, 9,-1,-1, -1,-1,-1,-1,
-    -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1
-]
-
-
-def a2b_hex(t):
-    result = []
-
-    def pairs_gen(s):
-        while s:
-            try:
-                yield table_hex[ord(s[0])], table_hex[ord(s[1])]
-            except IndexError:
-                if len(s):
-                    raise TypeError('Odd-length string')
-                return
-            s = s[2:]
-
-    for a, b in pairs_gen(t):
-        if a < 0 or b < 0:
-            raise TypeError('Non-hexadecimal digit found')
-        result.append(chr((a << 4) + b))
-    return ''.join(result)
-    
-
-unhexlify = a2b_hex
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -968,8 +968,7 @@
             self._checkOverflow(t.year)
             result = date(t.year, t.month, t.day)
             return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
+        return NotImplemented    # note that this doesn't work on CPython 2.2
 
     __radd__ = __add__
 
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/lib_pypy/numpypy/core/numeric.py b/lib_pypy/numpypy/core/numeric.py
--- a/lib_pypy/numpypy/core/numeric.py
+++ b/lib_pypy/numpypy/core/numeric.py
@@ -306,6 +306,125 @@
     else:
         return multiarray.set_string_function(f, repr)
 
+def array_equal(a1, a2):
+    """
+    True if two arrays have the same shape and elements, False otherwise.
+
+    Parameters
+    ----------
+    a1, a2 : array_like
+        Input arrays.
+
+    Returns
+    -------
+    b : bool
+        Returns True if the arrays are equal.
+
+    See Also
+    --------
+    allclose: Returns True if two arrays are element-wise equal within a
+              tolerance.
+    array_equiv: Returns True if input arrays are shape consistent and all
+                 elements equal.
+
+    Examples
+    --------
+    >>> np.array_equal([1, 2], [1, 2])
+    True
+    >>> np.array_equal(np.array([1, 2]), np.array([1, 2]))
+    True
+    >>> np.array_equal([1, 2], [1, 2, 3])
+    False
+    >>> np.array_equal([1, 2], [1, 4])
+    False
+
+    """
+    try:
+        a1, a2 = asarray(a1), asarray(a2)
+    except:
+        return False
+    if a1.shape != a2.shape:
+        return False
+    return bool((a1 == a2).all())
+
+def asarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
+    """
+    Convert the input to an array.
+
+    Parameters
+    ----------
+    a : array_like
+        Input data, in any form that can be converted to an array.  This
+        includes lists, lists of tuples, tuples, tuples of tuples, tuples
+        of lists and ndarrays.
+    dtype : data-type, optional
+        By default, the data-type is inferred from the input data.
+    order : {'C', 'F'}, optional
+        Whether to use row-major ('C') or column-major ('F' for FORTRAN)
+        memory representation.  Defaults to 'C'.
+   maskna : bool or None, optional
+        If this is set to True, it forces the array to have an NA mask.
+        If this is set to False, it forces the array to not have an NA
+        mask.
+    ownmaskna : bool, optional
+        If this is set to True, forces the array to have a mask which
+        it owns.
+
+    Returns
+    -------
+    out : ndarray
+        Array interpretation of `a`.  No copy is performed if the input
+        is already an ndarray.  If `a` is a subclass of ndarray, a base
+        class ndarray is returned.
+
+    See Also
+    --------
+    asanyarray : Similar function which passes through subclasses.
+    ascontiguousarray : Convert input to a contiguous array.
+    asfarray : Convert input to a floating point ndarray.
+    asfortranarray : Convert input to an ndarray with column-major
+                     memory order.
+    asarray_chkfinite : Similar function which checks input for NaNs and Infs.
+    fromiter : Create an array from an iterator.
+    fromfunction : Construct an array by executing a function on grid
+                   positions.
+
+    Examples
+    --------
+    Convert a list into an array:
+
+    >>> a = [1, 2]
+    >>> np.asarray(a)
+    array([1, 2])
+
+    Existing arrays are not copied:
+
+    >>> a = np.array([1, 2])
+    >>> np.asarray(a) is a
+    True
+
+    If `dtype` is set, array is copied only if dtype does not match:
+
+    >>> a = np.array([1, 2], dtype=np.float32)
+    >>> np.asarray(a, dtype=np.float32) is a
+    True
+    >>> np.asarray(a, dtype=np.float64) is a
+    False
+
+    Contrary to `asanyarray`, ndarray subclasses are not passed through:
+
+    >>> issubclass(np.matrix, np.ndarray)
+    True
+    >>> a = np.matrix([[1, 2]])
+    >>> np.asarray(a) is a
+    False
+    >>> np.asanyarray(a) is a
+    True
+
+    """
+    return array(a, dtype, copy=False, order=order,
+                            maskna=maskna, ownmaskna=ownmaskna)
+
 set_string_function(array_str, 0)
 set_string_function(array_repr, 1)
 
diff --git a/lib_pypy/pypy_test/test_binascii.py b/lib_pypy/pypy_test/test_binascii.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_binascii.py
+++ /dev/null
@@ -1,168 +0,0 @@
-from __future__ import absolute_import
-import py
-from lib_pypy import binascii
-
-# Create binary test data
-data = "The quick brown fox jumps over the lazy dog.\r\n"
-# Be slow so we don't depend on other modules
-data += "".join(map(chr, xrange(256)))
-data += "\r\nHello world.\n"
-
-def test_exceptions():
-    # Check module exceptions
-    assert issubclass(binascii.Error, Exception)
-    assert issubclass(binascii.Incomplete, Exception)
-
-def test_functions():
-    # Check presence of all functions
-    funcs = []
-    for suffix in "base64", "hqx", "uu", "hex":
-        prefixes = ["a2b_", "b2a_"]
-        if suffix == "hqx":
-            prefixes.extend(["crc_", "rlecode_", "rledecode_"])
-        for prefix in prefixes:
-            name = prefix + suffix
-            assert callable(getattr(binascii, name))
-            py.test.raises(TypeError, getattr(binascii, name))
-    for name in ("hexlify", "unhexlify"):
-        assert callable(getattr(binascii, name))
-        py.test.raises(TypeError, getattr(binascii, name))
-
-def test_base64valid():
-    # Test base64 with valid data
-    MAX_BASE64 = 57
-    lines = []
-    for i in range(0, len(data), MAX_BASE64):
-        b = data[i:i+MAX_BASE64]
-        a = binascii.b2a_base64(b)
-        lines.append(a)
-    res = ""
-    for line in lines:
-        b = binascii.a2b_base64(line)
-        res = res + b
-    assert res == data
-
-def test_base64invalid():
-    # Test base64 with random invalid characters sprinkled throughout
-    # (This requires a new version of binascii.)
-    MAX_BASE64 = 57
-    lines = []
-    for i in range(0, len(data), MAX_BASE64):
-        b = data[i:i+MAX_BASE64]
-        a = binascii.b2a_base64(b)
-        lines.append(a)
-
-    fillers = ""
-    valid = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/"
-    for i in xrange(256):
-        c = chr(i)
-        if c not in valid:
-            fillers += c
-    def addnoise(line):
-        noise = fillers
-        ratio = len(line) // len(noise)
-        res = ""
-        while line and noise:
-            if len(line) // len(noise) > ratio:
-                c, line = line[0], line[1:]
-            else:
-                c, noise = noise[0], noise[1:]
-            res += c
-        return res + noise + line
-    res = ""
-    for line in map(addnoise, lines):
-        b = binascii.a2b_base64(line)
-        res += b
-    assert res == data
-
-    # Test base64 with just invalid characters, which should return
-    # empty strings. TBD: shouldn't it raise an exception instead ?
-    assert binascii.a2b_base64(fillers) == ''
-
-def test_uu():
-    MAX_UU = 45
-    lines = []
-    for i in range(0, len(data), MAX_UU):
-        b = data[i:i+MAX_UU]
-        a = binascii.b2a_uu(b)
-        lines.append(a)
-    res = ""
-    for line in lines:
-        b = binascii.a2b_uu(line)
-        res += b
-    assert res == data
-
-    assert binascii.a2b_uu("\x7f") == "\x00"*31
-    assert binascii.a2b_uu("\x80") == "\x00"*32
-    assert binascii.a2b_uu("\xff") == "\x00"*31
-    py.test.raises(binascii.Error, binascii.a2b_uu, "\xff\x00")
-    py.test.raises(binascii.Error, binascii.a2b_uu, "!!!!")
-
-    py.test.raises(binascii.Error, binascii.b2a_uu, 46*"!")
-
-def test_crc32():
-    crc = binascii.crc32("Test the CRC-32 of")
-    crc = binascii.crc32(" this string.", crc)
-    assert crc == 1571220330
-    
-    crc = binascii.crc32('frotz\n', 0)
-    assert crc == -372923920
-
-    py.test.raises(TypeError, binascii.crc32)
-
-def test_hex():
-    # test hexlification
-    s = '{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000'
-    t = binascii.b2a_hex(s)
-    u = binascii.a2b_hex(t)
-    assert s == u
-    py.test.raises(TypeError, binascii.a2b_hex, t[:-1])
-    py.test.raises(TypeError, binascii.a2b_hex, t[:-1] + 'q')
-
-    # Verify the treatment of Unicode strings
-    assert binascii.hexlify(unicode('a', 'ascii')) == '61'
-
-def test_qp():
-    # A test for SF bug 534347 (segfaults without the proper fix)
-    try:
-        binascii.a2b_qp("", **{1:1})
-    except TypeError:
-        pass
-    else:
-        fail("binascii.a2b_qp(**{1:1}) didn't raise TypeError")
-    assert binascii.a2b_qp("= ") == "= "
-    assert binascii.a2b_qp("==") == "="
-    assert binascii.a2b_qp("=AX") == "=AX"
-    py.test.raises(TypeError, binascii.b2a_qp, foo="bar")
-    assert binascii.a2b_qp("=00\r\n=00") == "\x00\r\n\x00"
-    assert binascii.b2a_qp("\xff\r\n\xff\n\xff") == "=FF\r\n=FF\r\n=FF"
-    target = "0"*75+"=\r\n=FF\r\n=FF\r\n=FF"
-    assert binascii.b2a_qp("0"*75+"\xff\r\n\xff\r\n\xff") == target
-
-def test_empty_string():
-    # A test for SF bug #1022953.  Make sure SystemError is not raised.
-    for n in ['b2a_qp', 'a2b_hex', 'b2a_base64', 'a2b_uu', 'a2b_qp',
-              'b2a_hex', 'unhexlify', 'hexlify', 'crc32', 'b2a_hqx',
-              'a2b_hqx', 'a2b_base64', 'rlecode_hqx', 'b2a_uu',
-              'rledecode_hqx']:
-        f = getattr(binascii, n)
-        f('')
-    binascii.crc_hqx('', 0)
-
-def test_qp_bug_case():
-    assert binascii.b2a_qp('y'*77, False, False) == 'y'*75 + '=\nyy'
-    assert binascii.b2a_qp(' '*77, False, False) == ' '*75 + '=\n =20'
-    assert binascii.b2a_qp('y'*76, False, False) == 'y'*76
-    assert binascii.b2a_qp(' '*76, False, False) == ' '*75 + '=\n=20'
-
-def test_wrong_padding():
-    s = 'CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3IQ'
-    py.test.raises(binascii.Error, binascii.a2b_base64, s)
-
-def test_crap_after_padding():
-    s = 'xxx=axxxx'
-    assert binascii.a2b_base64(s) == '\xc7\x1c'
-
-def test_wrong_args():
-    # this should grow as a way longer list
-    py.test.raises(TypeError, binascii.a2b_base64, 42)
diff --git a/lib_pypy/pypy_test/test_locale.py b/lib_pypy/pypy_test/test_locale.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_locale.py
+++ /dev/null
@@ -1,79 +0,0 @@
-from __future__ import absolute_import
-import py
-import sys
-
-from lib_pypy.ctypes_config_cache import rebuild
-rebuild.rebuild_one('locale.ctc.py')
-
-from lib_pypy import _locale
-
-
-def setup_module(mod):
-    if sys.platform == 'darwin':
-        py.test.skip("Locale support on MacOSX is minimal and cannot be tested")
-
-class TestLocale:
-    def setup_class(cls):
-        cls.oldlocale = _locale.setlocale(_locale.LC_NUMERIC)
-        if sys.platform.startswith("win"):
-            cls.tloc = "en"
-        elif sys.platform.startswith("freebsd"):
-            cls.tloc = "en_US.US-ASCII"
-        else:
-            cls.tloc = "en_US.UTF8"
-        try:
-            _locale.setlocale(_locale.LC_NUMERIC, cls.tloc)
-        except _locale.Error:
-            py.test.skip("test locale %s not supported" % cls.tloc)
-            
-    def teardown_class(cls):
-        _locale.setlocale(_locale.LC_NUMERIC, cls.oldlocale)
-
-    def test_format(self):
-        py.test.skip("XXX fix or kill me")
-
-        def testformat(formatstr, value, grouping = 0, output=None):
-            if output:
-                print "%s %% %s =? %s ..." %\
-                      (repr(formatstr), repr(value), repr(output)),
-            else:
-                print "%s %% %s works? ..." % (repr(formatstr), repr(value)),
-            result = locale.format(formatstr, value, grouping = grouping)
-            assert result == output
-
-        testformat("%f", 1024, grouping=1, output='1,024.000000')
-        testformat("%f", 102, grouping=1, output='102.000000')
-        testformat("%f", -42, grouping=1, output='-42.000000')
-        testformat("%+f", -42, grouping=1, output='-42.000000')
-        testformat("%20.f", -42, grouping=1, output='                 -42')
-        testformat("%+10.f", -4200, grouping=1, output='    -4,200')
-        testformat("%-10.f", 4200, grouping=1, output='4,200     ')
-
-    def test_getpreferredencoding(self):
-        py.test.skip("XXX fix or kill me")
-        # Invoke getpreferredencoding to make sure it does not cause exceptions
-        _locale.getpreferredencoding()
-
-    # Test BSD Rune locale's bug for isctype functions.
-    def test_bsd_bug(self):
-        def teststrop(s, method, output):
-            print "%s.%s() =? %s ..." % (repr(s), method, repr(output)),
-            result = getattr(s, method)()
-            assert result == output
-
-        oldlocale = _locale.setlocale(_locale.LC_CTYPE)
-        _locale.setlocale(_locale.LC_CTYPE, self.tloc)
-        try:
-            teststrop('\x20', 'isspace', True)
-            teststrop('\xa0', 'isspace', False)
-            teststrop('\xa1', 'isspace', False)
-            teststrop('\xc0', 'isalpha', False)
-            teststrop('\xc0', 'isalnum', False)
-            teststrop('\xc0', 'isupper', False)
-            teststrop('\xc0', 'islower', False)
-            teststrop('\xec\xa0\xbc', 'split', ['\xec\xa0\xbc'])
-            teststrop('\xed\x95\xa0', 'strip', '\xed\x95\xa0')
-            teststrop('\xcc\x85', 'lower', '\xcc\x85')
-            teststrop('\xed\x95\xa0', 'upper', '\xed\x95\xa0')
-        finally:
-            _locale.setlocale(_locale.LC_CTYPE, oldlocale)
diff --git a/lib_pypy/pypy_test/test_struct_extra.py b/lib_pypy/pypy_test/test_struct_extra.py
deleted file mode 100644
--- a/lib_pypy/pypy_test/test_struct_extra.py
+++ /dev/null
@@ -1,25 +0,0 @@
-from __future__ import absolute_import
-from lib_pypy import struct 
-
-def test_simple():
-    morezeros = '\x00' * (struct.calcsize('l')-4)
-    assert struct.pack('<l', 16) == '\x10\x00\x00\x00' + morezeros
-    assert struct.pack('4s', 'WAVE') == 'WAVE'
-    assert struct.pack('<4sl', 'WAVE', 16) == 'WAVE\x10\x00\x00\x00' + morezeros
-    s = 'ABCD01234567\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00'
-    assert struct.unpack('<4s4H2lH', s) == ('ABCD', 0x3130, 0x3332, 0x3534,
-                                            0x3736, 1, 2, 3)
-
-def test_infinity():
-    INFINITY = 1e200 * 1e200
-    assert str(struct.unpack("!d", struct.pack("!d", INFINITY))[0]) \
-           == str(INFINITY)
-    assert str(struct.unpack("!d", struct.pack("!d", -INFINITY))[0]) \
-           == str(-INFINITY)
-
-def test_nan():
-    INFINITY = 1e200 * 1e200
-    NAN = INFINITY / INFINITY
-    assert str(struct.unpack("!d", '\xff\xf8\x00\x00\x00\x00\x00\x00')[0]) \
-           == str(NAN)
-    assert str(struct.unpack("!d", struct.pack("!d", NAN))[0]) == str(NAN)
diff --git a/lib_pypy/struct.py b/lib_pypy/struct.py
deleted file mode 100644
--- a/lib_pypy/struct.py
+++ /dev/null
@@ -1,417 +0,0 @@
-#
-# This module is a pure Python version of pypy.module.struct.
-# It is only imported if the vastly faster pypy.module.struct is not
-# compiled in.  For now we keep this version for reference and
-# because pypy.module.struct is not ootype-backend-friendly yet.
-#
-
-"""Functions to convert between Python values and C structs.
-Python strings are used to hold the data representing the C struct
-and also as format strings to describe the layout of data in the C struct.
-
-The optional first format char indicates byte order, size and alignment:
- @: native order, size & alignment (default)
- =: native order, std. size & alignment
- <: little-endian, std. size & alignment
- >: big-endian, std. size & alignment
- !: same as >
-
-The remaining chars indicate types of args and must match exactly;
-these can be preceded by a decimal repeat count:
-   x: pad byte (no data);
-   c:char;
-   b:signed byte;
-   B:unsigned byte;
-   h:short;
-   H:unsigned short;
-   i:int;
-   I:unsigned int;
-   l:long;
-   L:unsigned long;
-   f:float;
-   d:double.
-Special cases (preceding decimal count indicates length):
-   s:string (array of char); p: pascal string (with count byte).
-Special case (only available in native format):
-   P:an integer type that is wide enough to hold a pointer.
-Special case (not in native mode unless 'long long' in platform C):
-   q:long long;
-   Q:unsigned long long
-Whitespace between formats is ignored.
-
-The variable struct.error is an exception raised on errors."""
-
-import math, sys
-
-# TODO: XXX Find a way to get information on native sizes and alignments
-class StructError(Exception):
-    pass
-error = StructError
-def unpack_int(data,index,size,le):
-    bytes = [ord(b) for b in data[index:index+size]]
-    if le == 'little':
-        bytes.reverse()
-    number = 0L
-    for b in bytes:
-        number = number << 8 | b
-    return int(number)
-
-def unpack_signed_int(data,index,size,le):
-    number = unpack_int(data,index,size,le)
-    max = 2**(size*8)
-    if number > 2**(size*8 - 1) - 1:
-        number = int(-1*(max - number))
-    return number
-
-INFINITY = 1e200 * 1e200
-NAN = INFINITY / INFINITY
-
-def unpack_char(data,index,size,le):
-    return data[index:index+size]
-
-def pack_int(number,size,le):
-    x=number
-    res=[]
-    for i in range(size):
-        res.append(chr(x&0xff))
-        x >>= 8
-    if le == 'big':
-        res.reverse()
-    return ''.join(res)
-
-def pack_signed_int(number,size,le):
-    if not isinstance(number, (int,long)):
-        raise StructError,"argument for i,I,l,L,q,Q,h,H must be integer"
-    if  number > 2**(8*size-1)-1 or number < -1*2**(8*size-1):
-        raise OverflowError,"Number:%i too large to convert" % number
-    return pack_int(number,size,le)
-
-def pack_unsigned_int(number,size,le):
-    if not isinstance(number, (int,long)):
-        raise StructError,"argument for i,I,l,L,q,Q,h,H must be integer"
-    if number < 0:
-        raise TypeError,"can't convert negative long to unsigned"
-    if number > 2**(8*size)-1:
-        raise OverflowError,"Number:%i too large to convert" % number
-    return pack_int(number,size,le)
-
-def pack_char(char,size,le):
-    return str(char)
-
-def isinf(x):
-    return x != 0.0 and x / 2 == x
-def isnan(v):
-    return v != v*1.0 or (v == 1.0 and v == 2.0)
-
-def pack_float(x, size, le):
-    unsigned = float_pack(x, size)
-    result = []
-    for i in range(8):
-        result.append(chr((unsigned >> (i * 8)) & 0xFF))
-    if le == "big":
-        result.reverse()
-    return ''.join(result)
-
-def unpack_float(data, index, size, le):
-    binary = [data[i] for i in range(index, index + 8)]
-    if le == "big":
-        binary.reverse()
-    unsigned = 0
-    for i in range(8):
-        unsigned |= ord(binary[i]) << (i * 8)
-    return float_unpack(unsigned, size, le)
-
-def round_to_nearest(x):
-    """Python 3 style round:  round a float x to the nearest int, but
-    unlike the builtin Python 2.x round function:
-
-      - return an int, not a float
-      - do round-half-to-even, not round-half-away-from-zero.
-
-    We assume that x is finite and nonnegative; except wrong results
-    if you use this for negative x.
-
-    """
-    int_part = int(x)
-    frac_part = x - int_part
-    if frac_part > 0.5 or frac_part == 0.5 and int_part & 1 == 1:
-        int_part += 1
-    return int_part
-
-def float_unpack(Q, size, le):
-    """Convert a 32-bit or 64-bit integer created
-    by float_pack into a Python float."""
-
-    if size == 8:
-        MIN_EXP = -1021  # = sys.float_info.min_exp
-        MAX_EXP = 1024   # = sys.float_info.max_exp
-        MANT_DIG = 53    # = sys.float_info.mant_dig
-        BITS = 64
-    elif size == 4:
-        MIN_EXP = -125   # C's FLT_MIN_EXP
-        MAX_EXP = 128    # FLT_MAX_EXP
-        MANT_DIG = 24    # FLT_MANT_DIG
-        BITS = 32
-    else:
-        raise ValueError("invalid size value")
-
-    if Q >> BITS:
-         raise ValueError("input out of range")
-
-    # extract pieces
-    sign = Q >> BITS - 1
-    exp = (Q & ((1 << BITS - 1) - (1 << MANT_DIG - 1))) >> MANT_DIG - 1
-    mant = Q & ((1 << MANT_DIG - 1) - 1)
-
-    if exp == MAX_EXP - MIN_EXP + 2:
-        # nan or infinity
-        result = float('nan') if mant else float('inf')
-    elif exp == 0:
-        # subnormal or zero
-        result = math.ldexp(float(mant), MIN_EXP - MANT_DIG)
-    else:
-        # normal
-        mant += 1 << MANT_DIG - 1
-        result = math.ldexp(float(mant), exp + MIN_EXP - MANT_DIG - 1)
-    return -result if sign else result
-
-
-def float_pack(x, size):
-    """Convert a Python float x into a 64-bit unsigned integer
-    with the same byte representation."""
-
-    if size == 8:
-        MIN_EXP = -1021  # = sys.float_info.min_exp
-        MAX_EXP = 1024   # = sys.float_info.max_exp
-        MANT_DIG = 53    # = sys.float_info.mant_dig
-        BITS = 64
-    elif size == 4:
-        MIN_EXP = -125   # C's FLT_MIN_EXP
-        MAX_EXP = 128    # FLT_MAX_EXP
-        MANT_DIG = 24    # FLT_MANT_DIG
-        BITS = 32
-    else:
-        raise ValueError("invalid size value")
-
-    sign = math.copysign(1.0, x) < 0.0
-    if math.isinf(x):
-        mant = 0
-        exp = MAX_EXP - MIN_EXP + 2
-    elif math.isnan(x):
-        mant = 1 << (MANT_DIG-2) # other values possible
-        exp = MAX_EXP - MIN_EXP + 2
-    elif x == 0.0:
-        mant = 0
-        exp = 0
-    else:
-        m, e = math.frexp(abs(x))  # abs(x) == m * 2**e
-        exp = e - (MIN_EXP - 1)
-        if exp > 0:
-            # Normal case.
-            mant = round_to_nearest(m * (1 << MANT_DIG))
-            mant -= 1 << MANT_DIG - 1
-        else:
-            # Subnormal case.
-            if exp + MANT_DIG - 1 >= 0:
-                mant = round_to_nearest(m * (1 << exp + MANT_DIG - 1))
-            else:
-                mant = 0
-            exp = 0
-
-        # Special case: rounding produced a MANT_DIG-bit mantissa.
-        assert 0 <= mant <= 1 << MANT_DIG - 1
-        if mant == 1 << MANT_DIG - 1:
-            mant = 0
-            exp += 1
-
-        # Raise on overflow (in some circumstances, may want to return
-        # infinity instead).
-        if exp >= MAX_EXP - MIN_EXP + 2:
-             raise OverflowError("float too large to pack in this format")
-
-    # check constraints
-    assert 0 <= mant < 1 << MANT_DIG - 1
-    assert 0 <= exp <= MAX_EXP - MIN_EXP + 2
-    assert 0 <= sign <= 1
-    return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant
-
-
-big_endian_format = {
-    'x':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'b':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'B':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'c':{ 'size' : 1, 'alignment' : 0, 'pack' : pack_char, 'unpack' : unpack_char},
-    's':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'p':{ 'size' : 1, 'alignment' : 0, 'pack' : None, 'unpack' : None},
-    'h':{ 'size' : 2, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'H':{ 'size' : 2, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'i':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'I':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'l':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'L':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'q':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_signed_int, 'unpack' : unpack_signed_int},
-    'Q':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_unsigned_int, 'unpack' : unpack_int},
-    'f':{ 'size' : 4, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
-    'd':{ 'size' : 8, 'alignment' : 0, 'pack' : pack_float, 'unpack' : unpack_float},
-    }
-default = big_endian_format
-formatmode={ '<' : (default, 'little'),
-             '>' : (default, 'big'),
-             '!' : (default, 'big'),
-             '=' : (default, sys.byteorder),
-             '@' : (default, sys.byteorder)
-            }
-
-def getmode(fmt):
-    try:
-        formatdef,endianness = formatmode[fmt[0]]
-        index = 1
-    except KeyError:
-        formatdef,endianness = formatmode['@']
-        index = 0
-    return formatdef,endianness,index
-def getNum(fmt,i):
-    num=None
-    cur = fmt[i]
-    while ('0'<= cur ) and ( cur <= '9'):
-        if num == None:
-            num = int(cur)
-        else:
-            num = 10*num + int(cur)
-        i += 1
-        cur = fmt[i]
-    return num,i
-
-def calcsize(fmt):
-    """calcsize(fmt) -> int
-    Return size of C struct described by format string fmt.
-    See struct.__doc__ for more on format strings."""
-
-    formatdef,endianness,i = getmode(fmt)
-    num = 0
-    result = 0
-    while i<len(fmt):
-        num,i = getNum(fmt,i)
-        cur = fmt[i]
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError,"%s is not a valid format"%cur
-        if num != None :
-            result += num*format['size']
-        else:
-            result += format['size']
-        num = 0
-        i += 1
-    return result
-
-def pack(fmt,*args):
-    """pack(fmt, v1, v2, ...) -> string
-       Return string containing values v1, v2, ... packed according to fmt.
-       See struct.__doc__ for more on format strings."""
-    formatdef,endianness,i = getmode(fmt)
-    args = list(args)
-    n_args = len(args)
-    result = []
-    while i<len(fmt):
-        num,i = getNum(fmt,i)
-        cur = fmt[i]
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError,"%s is not a valid format"%cur
-        if num == None :
-            num_s = 0
-            num = 1
-        else:
-            num_s = num
-
-        if cur == 'x':
-            result += ['\0'*num]
-        elif cur == 's':
-            if isinstance(args[0], str):
-                padding = num - len(args[0])
-                result += [args[0][:num] + '\0'*padding]
-                args.pop(0)
-            else:
-                raise StructError,"arg for string format not a string"
-        elif cur == 'p':
-            if isinstance(args[0], str):
-                padding = num - len(args[0]) - 1
-
-                if padding > 0:
-                    result += [chr(len(args[0])) + args[0][:num-1] + '\0'*padding]
-                else:
-                    if num<255:
-                        result += [chr(num-1) + args[0][:num-1]]
-                    else:
-                        result += [chr(255) + args[0][:num-1]]
-                args.pop(0)
-            else:
-                raise StructError,"arg for string format not a string"
-
-        else:
-            if len(args) < num:
-                raise StructError,"insufficient arguments to pack"
-            for var in args[:num]:
-                result += [format['pack'](var,format['size'],endianness)]
-            args=args[num:]
-        num = None
-        i += 1
-    if len(args) != 0:
-        raise StructError,"too many arguments for pack format"
-    return ''.join(result)
-
-def unpack(fmt,data):
-    """unpack(fmt, string) -> (v1, v2, ...)
-       Unpack the string, containing packed C structure data, according
-       to fmt.  Requires len(string)==calcsize(fmt).
-       See struct.__doc__ for more on format strings."""
-    formatdef,endianness,i = getmode(fmt)
-    j = 0
-    num = 0
-    result = []
-    length= calcsize(fmt)
-    if length != len (data):
-        raise StructError,"unpack str size does not match format"
-    while i<len(fmt):
-        num,i=getNum(fmt,i)
-        cur = fmt[i]
-        i += 1
-        try:
-            format = formatdef[cur]
-        except KeyError:
-            raise StructError,"%s is not a valid format"%cur
-
-        if not num :
-            num = 1
-
-        if cur == 'x':
-            j += num
-        elif cur == 's':
-            result.append(data[j:j+num])
-            j += num
-        elif cur == 'p':
-            n=ord(data[j])
-            if n >= num:
-                n = num-1
-            result.append(data[j+1:j+n+1])
-            j += num
-        else:
-            for n in range(num):
-                result += [format['unpack'](data,j,format['size'],endianness)]
-                j += format['size']
-
-    return tuple(result)
-
-def pack_into(fmt, buf, offset, *args):
-    data = pack(fmt, *args)
-    buffer(buf)[offset:offset+len(data)] = data
-
-def unpack_from(fmt, buf, offset=0):
-    size = calcsize(fmt)
-    data = buffer(buf)[offset:offset+size]
-    if len(data) != size:
-        raise error("unpack_from requires a buffer of at least %d bytes"
-                    % (size,))
-    return unpack(fmt, data)
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.4.dev1'
+__version__ = '1.4.7'
 
 from py import _apipkg
 
@@ -70,6 +70,11 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
+        '_AssertionError'   : '._code.assertion:AssertionError',
+        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
+        '_reinterpret'      : '._code.assertion:reinterpret',
+        '_reprcompare'      : '._code.assertion:_reprcompare',
+        '_format_explanation' : '._code.assertion:_format_explanation',
     },
 
     # backports and additions of builtins
diff --git a/py/_builtin.py b/py/_builtin.py
--- a/py/_builtin.py
+++ b/py/_builtin.py
@@ -113,9 +113,12 @@
 
     # some backward compatibility helpers
     _basestring = str
-    def _totext(obj, encoding=None):
+    def _totext(obj, encoding=None, errors=None):
         if isinstance(obj, bytes):
-            obj = obj.decode(encoding)
+            if errors is None:
+                obj = obj.decode(encoding)
+            else:
+                obj = obj.decode(encoding, errors)
         elif not isinstance(obj, str):
             obj = str(obj)
         return obj
@@ -142,7 +145,7 @@
             del back
         elif locs is None:
             locs = globs
-        fp = open(fn, "rb")
+        fp = open(fn, "r")
         try:
             source = fp.read()
         finally:
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionnew.py
@@ -0,0 +1,339 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace _assertionold.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from py._code.assertion import _format_explanation, BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(failure):
+    explanation = _format_explanation(failure.explanation)
+    value = failure.cause[1]
+    if str(value):
+        lines = explanation.splitlines()
+        if not lines:
+            lines.append("")
+        lines[0] += " << %s" % (value,)
+        explanation = "\n".join(lines)
+    text = "%s: %s" % (failure.cause[0].__name__, explanation)
+    if text.startswith("AssertionError: assert "):
+        text = text[16:]
+    return text
+
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = False
+        if not local:
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not result:
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        rcomp = py.code._reprcompare
+        if rcomp:
+            res = rcomp(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = True
+        if from_instance:
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        if test_explanation.startswith("False\n{False =") and \
+                test_explanation.endswith("\n"):
+            test_explanation = test_explanation[15:-2]
+        explanation = "assert %s" % (test_explanation,)
+        if not test_result:
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/py/_code/_assertionold.py b/py/_code/_assertionold.py
new file mode 100644
--- /dev/null
+++ b/py/_code/_assertionold.py
@@ -0,0 +1,555 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from py._code.assertion import BuiltinAssertionError, _format_explanation
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = '<eval>'
+            self.__obj__.filename = '<eval>'
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = '<run>'
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return _format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # simplify 'assert False where False = ...'
+        if (test.explanation.startswith('False\n{False = ') and
+            test.explanation.endswith('\n}')):
+            test.explanation = test.explanation[15:-2]
+        # print the result as  'assert <explanation>'
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = '<run>'
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/py/_code/assertion.py b/py/_code/assertion.py
new file mode 100644
--- /dev/null
+++ b/py/_code/assertion.py
@@ -0,0 +1,94 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+_reprcompare = None # if set, will be called by assert reinterp for comparison ops
+
+def _format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "<run>".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = "<could not determine information>"
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from py._code._assertionold import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from py._code._assertionnew import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/py/_code/code.py b/py/_code/code.py
--- a/py/_code/code.py
+++ b/py/_code/code.py
@@ -145,6 +145,17 @@
         return self.frame.f_locals
     locals = property(getlocals, None, None, "locals of underlaying frame")
 
+    def reinterpret(self):
+        """Reinterpret the failing statement and returns a detailed information
+           about what operations are performed."""
+        if self.exprinfo is None:
+            source = str(self.statement).strip()
+            x = py.code._reinterpret(source, self.frame, should_fail=True)
+            if not isinstance(x, str):
+                raise TypeError("interpret returned non-string %r" % (x,))
+            self.exprinfo = x
+        return self.exprinfo
+
     def getfirstlinesource(self):
         # on Jython this firstlineno can be -1 apparently
         return max(self.frame.code.firstlineno, 0)
@@ -158,13 +169,12 @@
         end = self.lineno
         try:
             _, end = source.getstatementrange(end)
-        except IndexError:
+        except (IndexError, ValueError):
             end = self.lineno + 1
         # heuristic to stop displaying source on e.g.
         #   if something:  # assume this causes a NameError
         #      # _this_ lines and the one
                #        below we don't want from entry.getsource()
-        end = min(end, len(source))
         for i in range(self.lineno, end):
             if source[i].rstrip().endswith(':'):
                 end = i + 1
@@ -273,7 +283,11 @@
         """
         cache = {}
         for i, entry in enumerate(self):
-            key = entry.frame.code.path, entry.lineno
+            # id for the code.raw is needed to work around
+            # the strange metaprogramming in the decorator lib from pypi
+            # which generates code objects that have hash/value equality
+            #XXX needs a test
+            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
             #print "checking for recursion at", key
             l = cache.setdefault(key, [])
             if l:
@@ -308,7 +322,7 @@
                     self._striptext = 'AssertionError: '
         self._excinfo = tup
         self.type, self.value, tb = self._excinfo
-        self.typename = getattr(self.type, "__name__", "???")
+        self.typename = self.type.__name__
         self.traceback = py.code.Traceback(tb)
 
     def __repr__(self):
@@ -347,14 +361,16 @@
             showlocals: show locals per traceback entry
             style: long|short|no|native traceback style
             tbfilter: hide entries (where __tracebackhide__ is true)
+
+            in case of style==native, tbfilter and showlocals is ignored.
         """
         if style == 'native':
-            import traceback
-            return ''.join(traceback.format_exception(
-                self.type,
-                self.value,
-                self.traceback[0]._rawentry,
-                ))
+            return ReprExceptionInfo(ReprTracebackNative(
+                py.std.traceback.format_exception(
+                    self.type,
+                    self.value,
+                    self.traceback[0]._rawentry,
+                )), self._getreprcrash())
 
         fmt = FormattedExcinfo(showlocals=showlocals, style=style,
             abspath=abspath, tbfilter=tbfilter, funcargs=funcargs)
@@ -452,7 +468,7 @@
     def repr_locals(self, locals):
         if self.showlocals:
             lines = []
-            keys = list(locals)
+            keys = [loc for loc in locals if loc[0] != "@"]
             keys.sort()
             for name in keys:
                 value = locals[name]
@@ -506,7 +522,10 @@
 
     def _makepath(self, path):
         if not self.abspath:
-            np = py.path.local().bestrelpath(path)
+            try:
+                np = py.path.local().bestrelpath(path)
+            except OSError:
+                return path
             if len(np) < len(str(path)):
                 path = np
         return path
@@ -595,6 +614,19 @@
         if self.extraline:
             tw.line(self.extraline)
 
+class ReprTracebackNative(ReprTraceback):
+    def __init__(self, tblines):
+        self.style = "native"
+        self.reprentries = [ReprEntryNative(tblines)]
+        self.extraline = None
+
+class ReprEntryNative(TerminalRepr):
+    def __init__(self, tblines):
+        self.lines = tblines
+
+    def toterminal(self, tw):
+        tw.write("".join(self.lines))
+
 class ReprEntry(TerminalRepr):
     localssep = "_ "
 
@@ -680,19 +712,26 @@
 
 oldbuiltins = {}
 
-def patch_builtins(compile=True):
-    """ put compile builtins to Python's builtins. """
+def patch_builtins(assertion=True, compile=True):
+    """ put compile and AssertionError builtins to Python's builtins. """
+    if assertion:
+        from py._code import assertion
+        l = oldbuiltins.setdefault('AssertionError', [])
+        l.append(py.builtin.builtins.AssertionError)
+        py.builtin.builtins.AssertionError = assertion.AssertionError
     if compile:
         l = oldbuiltins.setdefault('compile', [])
         l.append(py.builtin.builtins.compile)
         py.builtin.builtins.compile = py.code.compile
 
-def unpatch_builtins(compile=True):
+def unpatch_builtins(assertion=True, compile=True):
     """ remove compile and AssertionError builtins from Python builtins. """
+    if assertion:
+        py.builtin.builtins.AssertionError = oldbuiltins['AssertionError'].pop()
     if compile:
         py.builtin.builtins.compile = oldbuiltins['compile'].pop()
 
-def getrawcode(obj):
+def getrawcode(obj, trycall=True):
     """ return code object for given function. """
     try:
         return obj.__code__
@@ -701,5 +740,10 @@
         obj = getattr(obj, 'func_code', obj)
         obj = getattr(obj, 'f_code', obj)
         obj = getattr(obj, '__code__', obj)
+        if trycall and not hasattr(obj, 'co_firstlineno'):
+            if hasattr(obj, '__call__') and not py.std.inspect.isclass(obj):
+                x = getrawcode(obj.__call__, trycall=False)
+                if hasattr(x, 'co_firstlineno'):
+                    return x
         return obj
 
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -108,6 +108,7 @@
     def getstatementrange(self, lineno, assertion=False):
         """ return (start, end) tuple which spans the minimal
             statement region which containing the given lineno.
+            raise an IndexError if no such statementrange can be found.
         """
         # XXX there must be a better than these heuristic ways ...
         # XXX there may even be better heuristics :-)
@@ -116,6 +117,7 @@
 
         # 1. find the start of the statement
         from codeop import compile_command
+        end = None
         for start in range(lineno, -1, -1):
             if assertion:
                 line = self.lines[start]
@@ -139,7 +141,9 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, len(self)
+        if end is None:
+            raise IndexError("no valid source range around line %d " % (lineno,))
+        return start, end
 
     def getblockend(self, lineno):
         # XXX
@@ -257,23 +261,29 @@
 
 
 def getfslineno(obj):
+    """ Return source location (path, lineno) for the given object.
+    If the source cannot be determined return ("", -1)
+    """
     try:
         code = py.code.Code(obj)
     except TypeError:
-        # fallback to
-        fn = (py.std.inspect.getsourcefile(obj) or
-              py.std.inspect.getfile(obj))
+        try:
+            fn = (py.std.inspect.getsourcefile(obj) or
+                  py.std.inspect.getfile(obj))
+        except TypeError:
+            return "", -1
+
         fspath = fn and py.path.local(fn) or None
+        lineno = -1
         if fspath:
             try:
                 _, lineno = findsource(obj)
             except IOError:
-                lineno = None
-        else:
-            lineno = None
+                pass
     else:
         fspath = code.path
         lineno = code.firstlineno
+    assert isinstance(lineno, int)
     return fspath, lineno
 
 #
@@ -286,7 +296,7 @@
     except py.builtin._sysex:
         raise
     except:
-        return None, None
+        return None, -1
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
diff --git a/py/_error.py b/py/_error.py
--- a/py/_error.py
+++ b/py/_error.py
@@ -23,6 +23,7 @@
     2: errno.ENOENT,
     3: errno.ENOENT,
     17: errno.EEXIST,
+    13: errno.EBUSY, # empty cd drive, but ENOMEDIUM seems unavailiable
     22: errno.ENOTDIR,
     267: errno.ENOTDIR,
     5: errno.EACCES,  # anything better?
diff --git a/py/_iniconfig.py b/py/_iniconfig.py
--- a/py/_iniconfig.py
+++ b/py/_iniconfig.py
@@ -103,6 +103,7 @@
     def _parseline(self, line, lineno):
         # comments
         line = line.split('#')[0].rstrip()
+        line = line.split(';')[0].rstrip()
         # blank lines
         if not line:
             return None, None
diff --git a/py/_io/capture.py b/py/_io/capture.py
--- a/py/_io/capture.py
+++ b/py/_io/capture.py
@@ -12,7 +12,7 @@
     class TextIO(StringIO):
         def write(self, data):
             if not isinstance(data, unicode):
-                data = unicode(data, getattr(self, '_encoding', 'UTF-8'))
+                data = unicode(data, getattr(self, '_encoding', 'UTF-8'), 'replace')
             StringIO.write(self, data)
 else:
     TextIO = StringIO
@@ -258,6 +258,9 @@
                 f = getattr(self, name).tmpfile
                 f.seek(0)
                 res = f.read()
+                enc = getattr(f, 'encoding', None)
+                if enc:
+                    res = py.builtin._totext(res, enc, 'replace')
                 f.truncate(0)
                 f.seek(0)
             l.append(res)
diff --git a/py/_io/terminalwriter.py b/py/_io/terminalwriter.py
--- a/py/_io/terminalwriter.py
+++ b/py/_io/terminalwriter.py
@@ -105,6 +105,8 @@
                      Blue=44, Purple=45, Cyan=46, White=47,
                      bold=1, light=2, blink=5, invert=7)
 
+    _newline = None   # the last line printed
+
     # XXX deprecate stringio argument
     def __init__(self, file=None, stringio=False, encoding=None):
         if file is None:
@@ -112,11 +114,9 @@
                 self.stringio = file = py.io.TextIO()
             else:
                 file = py.std.sys.stdout
-                if hasattr(file, 'encoding'):
-                    encoding = file.encoding
         elif hasattr(file, '__call__'):
             file = WriteFile(file, encoding=encoding)
-        self.encoding = encoding
+        self.encoding = encoding or getattr(file, 'encoding', "utf-8")
         self._file = file
         self.fullwidth = get_terminal_width()
         self.hasmarkup = should_do_markup(file)
@@ -182,8 +182,31 @@
         return s
 
     def line(self, s='', **kw):
+        if self._newline == False:
+            self.write("\n")
         self.write(s, **kw)
         self.write('\n')
+        self._newline = True
+
+    def reline(self, line, **opts):
+        if not self.hasmarkup:
+            raise ValueError("cannot use rewrite-line without terminal")
+        if not self._newline:
+            self.write("\r")
+        self.write(line, **opts)
+        # see if we need to fill up some spaces at the end
+        # xxx have a more exact lastlinelen working from self.write?
+        lenline = len(line)
+        try:
+            lastlen = self._lastlinelen
+        except AttributeError:
+            pass
+        else:
+            if lenline < lastlen:
+                self.write(" " * (lastlen - lenline + 1))
+        self._lastlinelen = lenline
+        self._newline = False
+
 
 class Win32ConsoleWriter(TerminalWriter):
     def write(self, s, **kw):
@@ -280,10 +303,10 @@
     SetConsoleTextAttribute = ctypes.windll.kernel32.SetConsoleTextAttribute
     SetConsoleTextAttribute.argtypes = [wintypes.HANDLE, wintypes.WORD]
     SetConsoleTextAttribute.restype = wintypes.BOOL
-        
+
     _GetConsoleScreenBufferInfo = \
         ctypes.windll.kernel32.GetConsoleScreenBufferInfo
-    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, 
+    _GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE,
                                 ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
     _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
     def GetConsoleInfo(handle):
diff --git a/py/_path/common.py b/py/_path/common.py
--- a/py/_path/common.py
+++ b/py/_path/common.py
@@ -64,7 +64,10 @@
                 else:
                     if bool(value) ^ bool(meth()) ^ invert:
                         return False
-            except (py.error.ENOENT, py.error.ENOTDIR):
+            except (py.error.ENOENT, py.error.ENOTDIR, py.error.EBUSY):
+                # EBUSY feels not entirely correct,
+                # but its kind of necessary since ENOMEDIUM
+                # is not accessible in python     
                 for name in self._depend_on_existence:
                     if name in kw:
                         if kw.get(name):
@@ -368,6 +371,5 @@
         else:
             name = str(path) # path.strpath # XXX svn?
             pattern = '*' + path.sep + pattern
-        from fnmatch import fnmatch
-        return fnmatch(name, pattern)
+        return py.std.fnmatch.fnmatch(name, pattern)
 
diff --git a/py/_path/local.py b/py/_path/local.py
--- a/py/_path/local.py
+++ b/py/_path/local.py
@@ -157,14 +157,16 @@
         return str(self) < str(other)
 
     def samefile(self, other):
-        """ return True if 'other' references the same file as 'self'. """
-        if not iswin32:
-            return py.error.checked_call(
-                    os.path.samefile, str(self), str(other))
+        """ return True if 'other' references the same file as 'self'.
+        """
+        if not isinstance(other, py.path.local):
+            other = os.path.abspath(str(other))
         if self == other:
             return True
-        other = os.path.abspath(str(other))
-        return self == other
+        if iswin32:
+            return False # ther is no samefile
+        return py.error.checked_call(
+                os.path.samefile, str(self), str(other))
 
     def remove(self, rec=1, ignore_errors=False):
         """ remove a file or directory (or a directory tree if rec=1).
@@ -539,7 +541,11 @@
                 if self.basename != "__init__.py":
                     modfile = modfile[:-12]
 
-            if not self.samefile(modfile):
+            try:
+                issame = self.samefile(modfile)
+            except py.error.ENOENT:
+                issame = False
+            if not issame:
                 raise self.ImportMismatchError(modname, modfile, self)
             return mod
         else:
diff --git a/py/_path/svnurl.py b/py/_path/svnurl.py
--- a/py/_path/svnurl.py
+++ b/py/_path/svnurl.py
@@ -233,6 +233,8 @@
                 e = sys.exc_info()[1]
                 if e.err.find('non-existent in that revision') != -1:
                     raise py.error.ENOENT(self, e.err)
+                elif e.err.find("E200009:") != -1:
+                    raise py.error.ENOENT(self, e.err)
                 elif e.err.find('File not found') != -1:
                     raise py.error.ENOENT(self, e.err)
                 elif e.err.find('not part of a repository')!=-1:
diff --git a/py/_path/svnwc.py b/py/_path/svnwc.py
--- a/py/_path/svnwc.py
+++ b/py/_path/svnwc.py
@@ -482,10 +482,13 @@
         except py.process.cmdexec.Error:
             e = sys.exc_info()[1]
             strerr = e.err.lower()
-            if strerr.find('file not found') != -1:
+            if strerr.find('not found') != -1:
+                raise py.error.ENOENT(self)
+            elif strerr.find("E200009:") != -1:
                 raise py.error.ENOENT(self)
             if (strerr.find('file exists') != -1 or
                 strerr.find('file already exists') != -1 or
+                strerr.find('w150002:') != -1 or
                 strerr.find("can't create directory") != -1):
                 raise py.error.EEXIST(self)
             raise
@@ -593,7 +596,7 @@
         out = self._authsvn('lock').strip()
         if not out:
             # warning or error, raise exception
-            raise Exception(out[4:])
+            raise ValueError("unknown error in svn lock command")
 
     def unlock(self):
         """ unset a previously set lock """
@@ -1066,6 +1069,8 @@
                 modrev = '?'
                 author = '?'
                 date = ''
+            elif itemstatus == "replaced":
+                pass
             else:
                 #print entryel.toxml()
                 commitel = entryel.getElementsByTagName('commit')[0]
@@ -1148,7 +1153,11 @@
             raise  ValueError("Not a versioned resource")
             #raise ValueError, "Not a versioned resource %r" % path
         self.kind = d['nodekind'] == 'directory' and 'dir' or d['nodekind']
-        self.rev = int(d['revision'])
+        try:
+            self.rev = int(d['revision'])
+        except KeyError:
+            self.rev = None
+
         self.path = py.path.local(d['path'])
         self.size = self.path.size()
         if 'lastchangedrev' in d:
diff --git a/py/_xmlgen.py b/py/_xmlgen.py
--- a/py/_xmlgen.py
+++ b/py/_xmlgen.py
@@ -52,7 +52,7 @@
     def unicode(self, indent=2):
         l = []
         SimpleUnicodeVisitor(l.append, indent).visit(self)
-        return "".join(l)
+        return u("").join(l)
 
     def __repr__(self):
         name = self.__class__.__name__
@@ -122,11 +122,13 @@
                 if visitmethod is not None:
                     break
             else:
-                visitmethod = self.object
+                visitmethod = self.__object
             self.cache[cls] = visitmethod
         visitmethod(node)
 
-    def object(self, obj):
+    # the default fallback handler is marked private
+    # to avoid clashes with the tag name object
+    def __object(self, obj):
         #self.write(obj)
         self.write(escape(unicode(obj)))
 
@@ -136,7 +138,8 @@
     def list(self, obj):
         assert id(obj) not in self.visited
         self.visited[id(obj)] = 1
-        map(self.visit, obj)
+        for elem in obj:
+            self.visit(elem)
 
     def Tag(self, tag):
         assert id(tag) not in self.visited
@@ -181,7 +184,11 @@
             value = getattr(attrs, name)
             if name.endswith('_'):
                 name = name[:-1]
-            return ' %s="%s"' % (name, escape(unicode(value)))
+            if isinstance(value, raw):
+                insert = value.uniobj
+            else:
+                insert = escape(unicode(value))
+            return ' %s="%s"' % (name, insert)
 
     def getstyle(self, tag):
         """ return attribute list suitable for styling. """
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -483,7 +483,7 @@
                 return a_str.strip(' ')
             elif n == 1:
                 return a_str.rstrip(' ')
-            else: 
+            else:
                 return a_str.lstrip(' ')
         s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
         assert s.no_nul
@@ -3737,6 +3737,25 @@
         s = a.build_types(f, [int])
         assert s.listdef.listitem.range_step == 0
 
+    def test_specialize_arg_memo(self):
+        @objectmodel.specialize.memo()
+        def g(n):
+            return n
+        @objectmodel.specialize.arg(0)
+        def f(i):
+            return g(i)
+        def main(i):
+            if i == 2:
+                return f(2)
+            elif i == 3:
+                return f(3)
+            else:
+                raise NotImplementedError
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(main, [int])
+        assert isinstance(s, annmodel.SomeInteger)
+
 
 def g(n):
     return [0,1,2,n]
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -158,6 +158,12 @@
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
+Note that this difference might show up indirectly in some cases.  For
+example, a generator left pending in the middle is --- again ---
+garbage-collected later in PyPy than in CPython.  You can see the
+difference if the ``yield`` keyword it is suspended at is itself
+enclosed in a ``try:`` or a ``with:`` block.
+
 Using the default GC called ``minimark``, the built-in function ``id()``
 works like it does in CPython.  With other GCs it returns numbers that
 are not real addresses (because an object can move around several times)
diff --git a/pypy/doc/discussion/win64_todo.txt b/pypy/doc/discussion/win64_todo.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/discussion/win64_todo.txt
@@ -0,0 +1,9 @@
+2011-11-04
+ll_os.py has a problem with the file rwin32.py.
+Temporarily disabled for the win64_gborg branch. This needs to be
+investigated and re-enabled.
+Resolved, enabled.
+
+2011-11-05
+test_typed.py needs explicit tests to ensure that we
+handle word sizes right.
\ No newline at end of file
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -103,21 +103,13 @@
 
 * A concurrent garbage collector (a lot of work)
 
-Remove the GIL
---------------
+STM, a.k.a. "remove the GIL"
+----------------------------
 
-This is a major task that requires lots of thinking. However, few subprojects
-can be potentially specified, unless a better plan can be thought out:
+Removing the GIL --- or more precisely, a GIL-less thread-less solution ---
+is `now work in progress.`__  Contributions welcome.
 
-* A thread-aware garbage collector
-
-* Better RPython primitives for dealing with concurrency
-
-* JIT passes to remove locks on objects
-
-* (maybe) implement locking in Python interpreter
-
-* alternatively, look at Software Transactional Memory
+.. __: http://pypy.org/tmdonate.html
 
 Introduce new benchmarks
 ------------------------
@@ -157,6 +149,22 @@
 exported.  This would give us a one-size-fits-all generic .so file to be
 imported by any application that wants to load .so files :-)
 
+Optimising cpyext (CPython C-API compatibility layer)
+-----------------------------------------------------
+
+A lot of work has gone into PyPy's implementation of CPython's C-API over
+the last years to let it reach a practical level of compatibility, so that
+C extensions for CPython work on PyPy without major rewrites. However,
+there are still many edges and corner cases where it misbehaves, and it has
+not received any substantial optimisation so far.
+
+The objective of this project is to fix bugs in cpyext and to optimise
+several performance critical parts of it, such as the reference counting
+support and other heavily used C-API functions. The net result would be to
+have CPython extensions run much faster on PyPy than they currently do, or
+to make them work at all if they currently don't. A part of this work would
+be to get cpyext into a shape where it supports running Cython generated
+extensions.
 
 .. _`issue tracker`: http://bugs.pypy.org
 .. _`mailing list`: http://mail.python.org/mailman/listinfo/pypy-dev
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -82,7 +82,10 @@
 
 In pypy/translator/goal::
 
-   ./translate.py --sandbox targetpypystandalone.py
+   ./translate.py -O2 --sandbox targetpypystandalone.py
+
+If you don't have a regular PyPy installed, you should, because it's
+faster to translate, but you can also run ``python translate.py`` instead.
 
 
 To run it, use the tools in the pypy/translator/sandbox directory::
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -199,17 +199,11 @@
 The following features (present in some past Stackless version of PyPy)
 are for the time being not supported any more:
 
-* Tasklets and channels (currently ``stackless.py`` seems to import,
-  but you have tasklets on top of coroutines on top of greenlets on
-  top of continulets on top of stacklets, and it's probably not too
-  hard to cut two of these levels by adapting ``stackless.py`` to
-  use directly continulets)
-
 * Coroutines (could be rewritten at app-level)
 
-* Pickling and unpickling continulets (*)
-
-* Continuing execution of a continulet in a different thread (*)
+* Continuing execution of a continulet in a different thread
+  (but if it is "simple enough", you can pickle it and unpickle it
+  in the other thread).
 
 * Automatic unlimited stack (must be emulated__ so far)
 
@@ -217,15 +211,6 @@
 
 .. __: `recursion depth limit`_
 
-(*) Pickling, as well as changing threads, could be implemented by using
-a "soft" stack switching mode again.  We would get either "hard" or
-"soft" switches, similarly to Stackless Python 3rd version: you get a
-"hard" switch (like now) when the C stack contains non-trivial C frames
-to save, and a "soft" switch (like previously) when it contains only
-simple calls from Python to Python.  Soft-switched continulets would
-also consume a bit less RAM, and the switch might be a bit faster too
-(unsure about that; what is the Stackless Python experience?).
-
 
 Recursion depth limit
 +++++++++++++++++++++
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
new file mode 100644
--- /dev/null
+++ b/pypy/doc/tool/makecontributor.py
@@ -0,0 +1,133 @@
+import py
+import sys
+from collections import defaultdict
+import operator
+import re
+import mercurial.localrepo
+import mercurial.ui
+
+ROOT = py.path.local(__file__).join('..', '..', '..', '..')
+author_re = re.compile('(.*) <.*>')
+pair_programming_re = re.compile(r'^\((.*?)\)')
+excluded = set(["pypy", "convert-repo"])
+
+alias = {
+    'Anders Chrigstrom': ['arre'],
+    'Antonio Cuni': ['antocuni', 'anto'],
+    'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
+    'Maciej Fijalkowski': ['fijal'],
+    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Michael Hudson': ['mwh'],
+    'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
+    "Amaury Forgeot d'Arc": ['afa'],
+    'Alex Gaynor': ['alex', 'agaynor'],
+    'David Schneider': ['bivab', 'david'],
+    'Christian Tismer': ['chris', 'christian', 'tismer',
+                         'tismer at christia-wjtqxl.localdomain'],
+    'Benjamin Peterson': ['benjamin'],
+    'Hakan Ardo': ['hakan', 'hakanardo'],
+    'Niklaus Haldimann': ['nik'],
+    'Alexander Schremmer': ['xoraxax'],
+    'Anders Hammarquist': ['iko'],
+    'David Edelsohn': ['edelsoh', 'edelsohn'],
+    'Niko Matsakis': ['niko'],
+    'Jakub Gustak': ['jlg'],
+    'Guido Wesdorp': ['guido'],
+    'Michael Foord': ['mfoord'],
+    'Mark Pearse': ['mwp'],
+    'Toon Verwaest': ['tverwaes'],
+    'Eric van Riet Paap': ['ericvrp'],
+    'Jacob Hallen': ['jacob', 'jakob'],
+    'Anders Lehmann': ['ale', 'anders'],
+    'Bert Freudenberg': ['bert'],
+    'Boris Feigin': ['boris', 'boria'],
+    'Valentino Volonghi': ['valentino', 'dialtone'],
+    'Aurelien Campeas': ['aurelien', 'aureliene'],
+    'Adrien Di Mascio': ['adim'],
+    'Jacek Generowicz': ['Jacek', 'jacek'],
+    'Jim Hunziker': ['landtuna at gmail.com'],
+    'Kristjan Valur Jonsson': ['kristjan at kristjan-lp.ccp.ad.local'],
+    'Laura Creighton': ['lac'],
+    'Aaron Iles': ['aliles'],
+    'Ludovic Aubry': ['ludal', 'ludovic'],
+    'Lukas Diekmann': ['l.diekmann', 'ldiekmann'],
+    'Matti Picus': ['Matti Picus matti.picus at gmail.com',
+                    'matthp', 'mattip', 'mattip>'],
+    'Michael Cheng': ['mikefc'],
+    'Richard Emslie': ['rxe'],
+    'Roberto De Ioris': ['roberto at goyle'],
+    'Roberto De Ioris': ['roberto at mrspurr'],
+    'Sven Hager': ['hager'],
+    'Tomo Cocoa': ['cocoatomo'],
+    }
+
+alias_map = {}
+for name, nicks in alias.iteritems():
+    for nick in nicks:
+        alias_map[nick] = name
+
+def get_canonical_author(name):
+    match = author_re.match(name)
+    if match:
+        name = match.group(1)
+    return alias_map.get(name, name)
+
+ignored_nicknames = defaultdict(int)
+
+def get_more_authors(log):
+    match = pair_programming_re.match(log)
+    if not match:
+        return set()
+    ignore_words = ['around', 'consulting', 'yesterday', 'for a bit', 'thanks',
+                    'in-progress', 'bits of', 'even a little', 'floating',]
+    sep_words = ['and', ';', '+', '/', 'with special  by']
+    nicknames = match.group(1)
+    for word in ignore_words:
+        nicknames = nicknames.replace(word, '')
+    for word in sep_words:
+        nicknames = nicknames.replace(word, ',')
+    nicknames = [nick.strip().lower() for nick in nicknames.split(',')]
+    authors = set()
+    for nickname in nicknames:
+        author = alias_map.get(nickname)
+        if not author:
+            ignored_nicknames[nickname] += 1
+        else:
+            authors.add(author)
+    return authors
+
+def main(show_numbers):
+    ui = mercurial.ui.ui()
+    repo = mercurial.localrepo.localrepository(ui, str(ROOT))
+    authors_count = defaultdict(int)
+    for i in repo:
+        ctx = repo[i]
+        authors = set()
+        authors.add(get_canonical_author(ctx.user()))
+        authors.update(get_more_authors(ctx.description()))
+        for author in authors:
+            if author not in excluded:
+                authors_count[author] += 1
+
+    # uncomment the next lines to get the list of nicknamed which could not be
+    # parsed from commit logs
+    ## items = ignored_nicknames.items()
+    ## items.sort(key=operator.itemgetter(1), reverse=True)
+    ## for name, n in items:
+    ##     if show_numbers:
+    ##         print '%5d %s' % (n, name)
+    ##     else:
+    ##         print name
+                
+    items = authors_count.items()
+    items.sort(key=operator.itemgetter(1), reverse=True)
+    for name, n in items:
+        if show_numbers:
+            print '%5d %s' % (n, name)
+        else:
+            print name
+
+if __name__ == '__main__':
+    show_numbers = '-n' in sys.argv
+    main(show_numbers)
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -18,7 +18,8 @@
 Edition.  Other configurations may work as well.
 
 The translation scripts will set up the appropriate environment variables
-for the compiler.  They will attempt to locate the same compiler version that
+for the compiler, so you do not need to run vcvars before translation.  
+They will attempt to locate the same compiler version that
 was used to build the Python interpreter doing the
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
@@ -26,7 +27,7 @@
 using a 32 bit Python and vice versa.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
-will be aborted in this case.
+will fail in this case.
 
 The compiler is all you need to build pypy-c, but it will miss some
 modules that relies on third-party libraries.  See below how to get
@@ -57,7 +58,8 @@
 install third-party libraries.  We chose to install them in the parent
 directory of the pypy checkout.  For example, if you installed pypy in
 ``d:\pypy\trunk\`` (This directory contains a README file), the base
-directory is ``d:\pypy``.
+directory is ``d:\pypy``. You may choose different values by setting the
+INCLUDE, LIB and PATH (for DLLs)
 
 The Boehm garbage collector
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -126,18 +128,54 @@
 ------------------------
 
 You can compile pypy with the mingw compiler, using the --cc=mingw32 option;
-mingw.exe must be on the PATH.
+gcc.exe must be on the PATH. If the -cc flag does not begin with "ming", it should be
+the name of a valid gcc-derivative compiler, i.e. x86_64-w64-mingw32-gcc for the 64 bit
+compiler creating a 64 bit target.
 
-libffi for the mingw32 compiler
+libffi for the mingw compiler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To enable the _rawffi (and ctypes) module, you need to compile a mingw32
-version of libffi.  I downloaded the `libffi source files`_, and extracted
-them in the base directory.  Then run::
+To enable the _rawffi (and ctypes) module, you need to compile a mingw
+version of libffi.  Here is one way to do this, wich should allow you to try
+to build for win64 or win32:
+
+#. Download and unzip a `mingw32 build`_ or `mingw64 build`_, say into c:\mingw
+#. If you do not use cygwin, you will need msys to provide make, 
+   autoconf tools and other goodies.
+
+    #. Download and unzip a `msys for mingw`_, say into c:\msys
+    #. Edit the c:\msys\etc\fstab file to mount c:\mingw
+
+#. Download and unzip the `libffi source files`_, and extract
+   them in the base directory.  
+#. Run c:\msys\msys.bat or a cygwin shell which should make you
+   feel better since it is a shell prompt with shell tools.
+#. From inside the shell, cd to the libffi directory and do::
 
     sh ./configure
     make
     cp .libs/libffi-5.dll <somewhere on the PATH>
 
+If you can't find the dll, and the libtool issued a warning about 
+"undefined symbols not allowed", you will need to edit the libffi
+Makefile in the toplevel directory. Add the flag -no-undefined to
+the definition of libffi_la_LDFLAGS
+
+If you wish to experiment with win64, you must run configure with flags::
+
+    sh ./configure --build=x86_64-w64-mingw32 --host=x86_64-w64-mingw32
+
+or such, depending on your mingw64 download.
+
+hacking on Pypy with the mingw compiler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Since hacking on Pypy means running tests, you will need a way to specify
+the mingw compiler when hacking (as opposed to translating). As of
+March 2012, --cc is not a valid option for pytest.py. However if you set an
+environment variable CC it will allow you to choose a compiler.
+
+.. _'mingw32 build': http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Automated%20Builds
+.. _`mingw64 build`: http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Automated%20Builds
+.. _`msys for mingw`: http://sourceforge.net/projects/mingw-w64/files/External%20binary%20packages%20%28Win64%20hosted%29/MSYS%20%2832-bit%29   
 .. _`libffi source files`: http://sourceware.org/libffi/
 .. _`RPython translation toolchain`: translation.html
diff --git a/pypy/doc/you-want-to-help.rst b/pypy/doc/you-want-to-help.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/you-want-to-help.rst
@@ -0,0 +1,86 @@
+
+You want to help with PyPy, now what?
+=====================================
+
+PyPy is a very large project that has a reputation of being hard to dive into.
+Some of this fame is warranted, some of it is purely accidental. There are three
+important lessons that everyone willing to contribute should learn:
+
+* PyPy has layers. There are many pieces of architecture that are very well
+  separated from each other. More about this below, but often the manifestation
+  of this is that things are at a different layer than you would expect them
+  to be. For example if you are looking for the JIT implementation, you will
+  not find it in the implementation of the Python programming language.
+
+* Because of the above, we are very serious about Test Driven Development.
+  It's not only what we believe in, but also that PyPy's architecture is
+  working very well with TDD in mind and not so well without it. Often
+  the development means progressing in an unrelated corner, one unittest
+  at a time; and then flipping a giant switch, bringing it all together.
+  (It generally works out of the box.  If it doesn't, then we didn't
+  write enough unit tests.)  It's worth repeating - PyPy
+  approach is great if you do TDD, not so great otherwise.
+
+* PyPy uses an entirely different set of tools - most of them included
+  in the PyPy repository. There is no Makefile, nor autoconf. More below
+
+Architecture
+============
+
+PyPy has layers. The 100 miles view:
+
+* `RPython`_ is the language in which we write interpreters. Not the entire
+  PyPy project is written in RPython, only the parts that are compiled in
+  the translation process. The interesting point is that RPython has no parser,
+  it's compiled from the live python objects, which make it possible to do
+  all kinds of metaprogramming during import time. In short, Python is a meta
+  programming language for RPython.
+
+  The RPython standard library is to be found in the ``rlib`` subdirectory.
+
+.. _`RPython`: coding-guide.html#RPython
+
+* The translation toolchain - this is the part that takes care about translating
+  RPython to flow graphs and then to C. There is more in the `architecture`_
+  document written about it.
+
+  It mostly lives in ``rpython``, ``annotator`` and ``objspace/flow``.
+
+.. _`architecture`: architecture.html 
+
+* Python Interpreter
+
+  xxx
+
+* Python modules
+
+  xxx
+
+* Just-in-Time Compiler (JIT): `we have a tracing JIT`_ that traces the
+  interpreter written in RPython, rather than the user program that it
+  interprets.  As a result it applies to any interpreter, i.e. any
+  language.  But getting it to work correctly is not trivial: it
+  requires a small number of precise "hints" and possibly some small
+  refactorings of the interpreter.  The JIT itself also has several
+  almost-independent parts: the tracer itself in ``jit/metainterp``, the
+  optimizer in ``jit/metainterp/optimizer`` that optimizes a list of
+  residual operations, and the backend in ``jit/backend/<machine-name>``
+  that turns it into machine code.  Writing a new backend is a
+  traditional way to get into the project.
+
+.. _`we have a tracing JIT`: jit/index.html
+
+* Garbage Collectors (GC): as you can notice if you are used to CPython's
+  C code, there are no ``Py_INCREF/Py_DECREF`` equivalents in RPython code.
+  `Garbage collection in PyPy`_ is inserted
+  during translation.  Moreover, this is not reference counting; it is a real
+  GC written as more RPython code.  The best one we have so far is in
+  ``rpython/memory/gc/minimark.py``.
+
+.. _`Garbage collection in PyPy`: garbage_collection.html
+
+
+Toolset
+=======
+
+xxx
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -169,9 +173,17 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +198,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +236,20 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +408,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -10,16 +10,6 @@
 from pypy.interpreter.astcompiler import ast, consts
 
 
-try:
-    all
-except NameError:
-    def all(iterable):
-        for x in iterable:
-            if not x:
-                return False
-        return True
-
-
 class TestAstBuilder:
 
     def setup_class(cls):
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -296,6 +296,7 @@
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
         self.frame_trace_action = FrameTraceAction(self)
+        self._code_of_sys_exc_info = None
 
         from pypy.interpreter.pycode import cpython_magic, default_magic
         self.our_magic = default_magic
@@ -467,9 +468,9 @@
                 if name not in modules:
                     modules.append(name)
 
-        # a bit of custom logic: time2 or rctime take precedence over time
+        # a bit of custom logic: rctime take precedence over time
         # XXX this could probably be done as a "requires" in the config
-        if ('time2' in modules or 'rctime' in modules) and 'time' in modules:
+        if 'rctime' in modules and 'time' in modules:
             modules.remove('time')
 
         if not self.config.objspace.nofaking:
@@ -913,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -159,6 +159,7 @@
         #operationerr.print_detailed_traceback(self.space)
 
     def _convert_exc(self, operr):
+        # Only for the flow object space
         return operr
 
     def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
@@ -171,6 +172,11 @@
             frame = self.getnextframe_nohidden(frame)
         return None
 
+    def set_sys_exc_info(self, operror):
+        frame = self.gettopframe_nohidden()
+        if frame:     # else, the exception goes nowhere and is lost
+            frame.last_exception = operror
+
     def settrace(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -113,6 +113,12 @@
         from pypy.interpreter.pycode import PyCode
 
         code = self.getcode() # hook for the jit
+        #
+        if (jit.we_are_jitted() and code is self.space._code_of_sys_exc_info
+                                and nargs == 0):
+            from pypy.module.sys.vm import exc_info_direct
+            return exc_info_direct(self.space, frame)
+        #
         fast_natural_arity = code.fast_natural_arity
         if nargs == fast_natural_arity:
             if nargs == 0:
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -874,6 +874,12 @@
             fn.add_to_table()
         if gateway.as_classmethod:
             fn = ClassMethod(space.wrap(fn))
+        #
+        from pypy.module.sys.vm import exc_info
+        if code._bltin is exc_info:
+            assert space._code_of_sys_exc_info is None
+            space._code_of_sys_exc_info = code
+        #
         return fn
 
 
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -2,34 +2,39 @@
 from pypy.interpreter import unicodehelper
 from pypy.rlib.rstring import StringBuilder
 
-def parsestr(space, encoding, s, unicode_literals=False):
-    # compiler.transformer.Transformer.decode_literal depends on what 
-    # might seem like minor details of this function -- changes here 
-    # must be reflected there.
+def parsestr(space, encoding, s, unicode_literal=False):
+    """Parses a string or unicode literal, and return a wrapped value.
+
+    If encoding=iso8859-1, the source string is also in this encoding.
+    If encoding=None, the source string is ascii only.
+    In other cases, the source string is in utf-8 encoding.
+
+    When a bytes string is returned, it will be encoded with the
+    original encoding.
+
+    Yes, it's very inefficient.
+    Yes, CPython has very similar code.
+    """
 
     # we use ps as "pointer to s"
     # q is the virtual last char index of the string
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = unicode_literals
 
     # string decoration handling
-    o = ord(quote)
-    isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
-    if isalpha or quote == '_':
-        if quote == 'b' or quote == 'B':
-            ps += 1
-            quote = s[ps]
-            unicode = False
-        elif quote == 'u' or quote == 'U':
-            ps += 1
-            quote = s[ps]
-            unicode = True
-        if quote == 'r' or quote == 'R':
-            ps += 1
-            quote = s[ps]
-            rawmode = True
+    if quote == 'b' or quote == 'B':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = False
+    elif quote == 'u' or quote == 'U':
+        ps += 1
+        quote = s[ps]
+        unicode_literal = True
+    if quote == 'r' or quote == 'R':
+        ps += 1
+        quote = s[ps]
+        rawmode = True
     if quote != "'" and quote != '"':
         raise_app_valueerror(space,
                              'Internal error: parser passed unquoted literal')
@@ -46,21 +51,28 @@
                                         'unmatched triple quotes in literal')
         q -= 2
 
-    if unicode: # XXX Py_UnicodeFlag is ignored for now
+    if unicode_literal: # XXX Py_UnicodeFlag is ignored for now
         if encoding is None or encoding == "iso-8859-1":
+            # 'unicode_escape' expects latin-1 bytes, string is ready.
             buf = s
             bufp = ps
             bufq = q
             u = None
         else:
-            # "\XX" may become "\u005c\uHHLL" (12 bytes)
+            # String is utf8-encoded, but 'unicode_escape' expects
+            # latin-1; So multibyte sequences must be escaped.
             lis = [] # using a list to assemble the value
             end = q
+            # Worst case: "\XX" may become "\u005c\uHHLL" (12 bytes)
             while ps < end:
                 if s[ps] == '\\':
                     lis.append(s[ps])
                     ps += 1
                     if ord(s[ps]) & 0x80:
+                        # A multibyte sequence will follow, it will be
+                        # escaped like \u1234. To avoid confusion with
+                        # the backslash we just wrote, we emit "\u005c"
+                        # instead.
                         lis.append("u005c")
                 if ord(s[ps]) & 0x80: # XXX inefficient
                     w, ps = decode_utf8(space, s, ps, end, "utf-16-be")
@@ -86,13 +98,11 @@
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
-    # XXX add strchr like interface to rtyper
     assert 0 <= ps <= q
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
             w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
-            #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -312,8 +312,8 @@
             mods = space.get_builtinmodule_to_install()
             
             assert '__pypy__' in mods                # real builtin
-            assert 'array' not in mods               # in lib_pypy
-            assert 'faked+array' not in mods         # in lib_pypy
+            assert '_functools' not in mods               # in lib_pypy
+            assert 'faked+_functools' not in mods         # in lib_pypy
             assert 'this_doesnt_exist' not in mods   # not in lib_pypy
             assert 'faked+this_doesnt_exist' in mods # not in lib_pypy, but in
                                                      # ALL_BUILTIN_MODULES
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -75,6 +75,7 @@
 class AppTestInterpObjectPickling:
     pytestmark = py.test.mark.skipif("config.option.runappdirect")
     def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['struct'])
         _attach_helpers(cls.space)
 
     def teardown_class(cls):
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -321,7 +321,7 @@
             
             def user_setup(self, space, w_subtype):
                 self.w__dict__ = space.newdict(
-                    instance=True, classofinstance=w_subtype)
+                    instance=True)
                 base_user_setup(self, space, w_subtype)
 
             def setclass(self, space, w_subtype):
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1797,6 +1797,7 @@
         if specialize_as_constant:
             def specialize_call(self, hop):
                 llvalue = func(hop.args_s[0].const)
+                hop.exception_cannot_occur()
                 return hop.inputconst(lltype.typeOf(llvalue), llvalue)
         else:
             # specialize as direct_call
@@ -1813,6 +1814,7 @@
                     sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func)
                     cfunc = hop.inputconst(FUNCTYPE, sm)
                 args_v = hop.inputargs(*hop.args_r)
+                hop.exception_is_here()
                 return hop.genop('direct_call', [cfunc] + args_v, hop.r_result)
 
 
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -27,6 +27,12 @@
 def constfloat(x):
     return ConstFloat(longlong.getfloatstorage(x))
 
+def boxlonglong(ll):
+    if longlong.is_64_bit:
+        return BoxInt(ll)
+    else:
+        return BoxFloat(ll)
+
 
 class Runner(object):
 
@@ -1623,6 +1629,11 @@
                                      [boxfloat(2.5)], t).value
         assert res == longlong2float.float2longlong(2.5)
 
+        bytes = longlong2float.float2longlong(2.5)
+        res = self.execute_operation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT,
+                                     [boxlonglong(res)], 'float').value
+        assert longlong.getrealfloat(res) == 2.5
+
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
         v = heaptracker.adr2int(llmemory.cast_ptr_to_adr(x))
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -328,6 +328,15 @@
     def produce_into(self, builder, r):
         self.put(builder, [r.choice(builder.intvars)])
 
+class CastLongLongToFloatOperation(AbstractFloatOperation):
+    def produce_into(self, builder, r):
+        if longlong.is_64_bit:
+            self.put(builder, [r.choice(builder.intvars)])
+        else:
+            if not builder.floatvars:
+                raise CannotProduceOperation
+            self.put(builder, [r.choice(builder.floatvars)])
+
 class CastFloatToIntOperation(AbstractFloatOperation):
     def produce_into(self, builder, r):
         if not builder.floatvars:
@@ -450,6 +459,7 @@
 OPERATIONS.append(CastFloatToIntOperation(rop.CAST_FLOAT_TO_INT))
 OPERATIONS.append(CastIntToFloatOperation(rop.CAST_INT_TO_FLOAT))
 OPERATIONS.append(CastFloatToIntOperation(rop.CONVERT_FLOAT_BYTES_TO_LONGLONG))
+OPERATIONS.append(CastLongLongToFloatOperation(rop.CONVERT_LONGLONG_BYTES_TO_FLOAT))
 
 OperationBuilder.OPERATIONS = OPERATIONS
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1251,6 +1251,15 @@
         else:
             self.mov(loc0, resloc)
 
+    def genop_convert_longlong_bytes_to_float(self, op, arglocs, resloc):
+        loc0, = arglocs
+        if longlong.is_64_bit:
+            assert isinstance(resloc, RegLoc)
+            assert isinstance(loc0, RegLoc)
+            self.mc.MOVD(resloc, loc0)
+        else:
+            self.mov(loc0, resloc)
+
     def genop_guard_int_is_true(self, op, guard_op, guard_token, arglocs, resloc):
         guard_opnum = guard_op.getopnum()
         self.mc.CMP(arglocs[0], imm0)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -773,10 +773,24 @@
             self.Perform(op, [loc0], loc1)
             self.xrm.possibly_free_var(op.getarg(0))
         else:
-            loc0 = self.xrm.loc(op.getarg(0))
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.loc(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
+
+    def consider_convert_longlong_bytes_to_float(self, op):
+        if longlong.is_64_bit:
+            loc0 = self.rm.make_sure_var_in_reg(op.getarg(0))
             loc1 = self.xrm.force_allocate_reg(op.result)
             self.Perform(op, [loc0], loc1)
-            self.xrm.possibly_free_var(op.getarg(0))
+            self.rm.possibly_free_var(op.getarg(0))
+        else:
+            arg0 = op.getarg(0)
+            loc0 = self.xrm.make_sure_var_in_reg(arg0)
+            loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0])
+            self.Perform(op, [loc0], loc1)
+            self.xrm.possibly_free_var(arg0)
 
     def _consider_llong_binop_xx(self, op):
         # must force both arguments into xmm registers, because we don't
diff --git a/pypy/jit/backend/x86/rx86.py b/pypy/jit/backend/x86/rx86.py
--- a/pypy/jit/backend/x86/rx86.py
+++ b/pypy/jit/backend/x86/rx86.py
@@ -601,7 +601,9 @@
     CVTSS2SD_xb = xmminsn('\xF3', rex_nw, '\x0F\x5A',
                           register(1, 8), stack_bp(2))
 
-    # These work on machine sized registers.
+    # These work on machine sized registers, so MOVD is actually MOVQ
+    # when running on 64 bits.  Note a bug in the Intel documentation:
+    # http://lists.gnu.org/archive/html/bug-binutils/2007-07/msg00095.html
     MOVD_rx = xmminsn('\x66', rex_w, '\x0F\x7E', register(2, 8), register(1), '\xC0')
     MOVD_xr = xmminsn('\x66', rex_w, '\x0F\x6E', register(1, 8), register(2), '\xC0')
     MOVD_xb = xmminsn('\x66', rex_w, '\x0F\x6E', register(1, 8), stack_bp(2))
diff --git a/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
--- a/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
+++ b/pypy/jit/backend/x86/test/test_rx86_32_auto_encoding.py
@@ -182,6 +182,12 @@
         filename  = str(testdir.join(FILENAME  % methname))
         g = open(inputname, 'w')
         g.write('\x09.string "%s"\n' % BEGIN_TAG)
+        #
+        if instrname == 'MOVD' and self.WORD == 8:
+            instrname = 'MOVQ'
+            if argmodes == 'xb':
+                py.test.skip('"as" uses an undocumented alternate encoding??')
+        #
         for args in args_lists:
             suffix = ""
     ##        all = instr.as_all_suffixes
@@ -229,9 +235,6 @@
                 # movq $xxx, %rax => movl $xxx, %eax
                 suffix = 'l'
                 ops[1] = reduce_to_32bit(ops[1])
-            if instrname.lower() == 'movd':
-                ops[0] = reduce_to_32bit(ops[0])
-                ops[1] = reduce_to_32bit(ops[1])
             #
             op = '\t%s%s %s%s' % (instrname.lower(), suffix,
                                   ', '.join(ops), following)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -295,6 +295,7 @@
         return op
 
     rewrite_op_convert_float_bytes_to_longlong = _noop_rewrite
+    rewrite_op_convert_longlong_bytes_to_float = _noop_rewrite
 
     # ----------
     # Various kinds of calls
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -968,20 +968,22 @@
             int_return %i2
         """, transform=True)
 
-    def test_convert_float_bytes_to_int(self):
-        from pypy.rlib.longlong2float import float2longlong
+    def test_convert_float_bytes(self):
+        from pypy.rlib.longlong2float import float2longlong, longlong2float
         def f(x):
-            return float2longlong(x)
+            ll = float2longlong(x)
+            return longlong2float(ll)
         if longlong.is_64_bit:
-            result_var = "%i0"
-            return_op = "int_return"
+            tmp_var = "%i0"
+            result_var = "%f1"
         else:
-            result_var = "%f1"
-            return_op = "float_return"
+            tmp_var = "%f1"
+            result_var = "%f2"
         self.encoding_test(f, [25.0], """
-            convert_float_bytes_to_longlong %%f0 -> %(result_var)s
-            %(return_op)s %(result_var)s
-        """ % {"result_var": result_var, "return_op": return_op})
+            convert_float_bytes_to_longlong %%f0 -> %(tmp_var)s
+            convert_longlong_bytes_to_float %(tmp_var)s -> %(result_var)s
+            float_return %(result_var)s
+        """ % {"result_var": result_var, "tmp_var": tmp_var}, transform=True)
 
 
 def check_force_cast(FROM, TO, operations, value):
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -672,6 +672,11 @@
         a = longlong.getrealfloat(a)
         return longlong2float.float2longlong(a)
 
+    @arguments(LONGLONG_TYPECODE, returns="f")
+    def bhimpl_convert_longlong_bytes_to_float(a):
+        a = longlong2float.longlong2float(a)
+        return longlong.getfloatstorage(a)
+
     # ----------
     # control flow operations
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -224,6 +224,7 @@
                     'float_neg', 'float_abs',
                     'cast_ptr_to_int', 'cast_int_to_ptr',
                     'convert_float_bytes_to_longlong',
+                    'convert_longlong_bytes_to_float',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -1222,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -420,6 +420,7 @@
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     'CONVERT_FLOAT_BYTES_TO_LONGLONG/1',
+    'CONVERT_LONGLONG_BYTES_TO_FLOAT/1',
     #
     'INT_LT/2b',
     'INT_LE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,3 +1,4 @@
+import math
 import sys
 
 import py
@@ -15,7 +16,7 @@
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
     isconstant, isvirtual, promote_string, set_param, record_known_class)
-from pypy.rlib.longlong2float import float2longlong
+from pypy.rlib.longlong2float import float2longlong, longlong2float
 from pypy.rlib.rarithmetic import ovfcheck, is_valid_int
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3795,15 +3796,15 @@
         res = self.interp_operations(g, [1])
         assert res == 3
 
-    def test_float2longlong(self):
+    def test_float_bytes(self):
         def f(n):
-            return float2longlong(n)
+            ll = float2longlong(n)
+            return longlong2float(ll)
 
         for x in [2.5, float("nan"), -2.5, float("inf")]:
             # There are tests elsewhere to verify the correctness of this.
-            expected = float2longlong(x)
             res = self.interp_operations(f, [x])
-            assert longlong.getfloatstorage(res) == expected
+            assert res == x or math.isnan(x) and math.isnan(res)
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -16,13 +16,15 @@
     appleveldefs = {}
     interpleveldefs = {}
     if sys.platform.startswith("linux"):
+        from pypy.module.__pypy__ import interp_time
         interpleveldefs["clock_gettime"] = "interp_time.clock_gettime"
         interpleveldefs["clock_getres"] = "interp_time.clock_getres"
         for name in [
             "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW",
             "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID"
         ]:
-            interpleveldefs[name] = "space.wrap(interp_time.%s)" % name
+            if getattr(interp_time, name) is not None:
+                interpleveldefs[name] = "space.wrap(interp_time.%s)" % name
 
 
 class Module(MixedModule):
diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py
--- a/pypy/module/__pypy__/interp_time.py
+++ b/pypy/module/__pypy__/interp_time.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import sys
 
 from pypy.interpreter.error import exception_from_errno
diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -1,9 +1,10 @@
 import py
-
+from pypy.conftest import gettestobjspace
 
 class AppTestAST:
 
     def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['struct'])
         cls.w_ast = cls.space.appexec([], """():
     import _ast
     return _ast""")
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -4,7 +4,7 @@
 
 class AppTestCodecs:
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('unicodedata',))
+        space = gettestobjspace(usemodules=('unicodedata', 'struct'))
         cls.space = space
 
     def test_register_noncallable(self):
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
--- a/pypy/module/_continuation/test/test_zpickle.py
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -106,8 +106,9 @@
     version = 0
 
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=('_continuation',),
+        cls.space = gettestobjspace(usemodules=('_continuation', 'struct'),
                                     CALL_METHOD=True)
+        cls.space.config.translation.continuation = True
         cls.space.appexec([], """():
             global continulet, A, __name__
 
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -3,7 +3,7 @@
 
 class AppTestHashlib:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_hashlib'])
+        cls.space = gettestobjspace(usemodules=['_hashlib', 'array', 'struct'])
 
     def test_simple(self):
         import _hashlib
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -158,7 +158,7 @@
 
 class AppTestOpen:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_io', '_locale'])
+        cls.space = gettestobjspace(usemodules=['_io', '_locale', 'array', 'struct'])
         tmpfile = udir.join('tmpfile').ensure()
         cls.w_tmpfile = cls.space.wrap(str(tmpfile))
 
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -92,7 +92,8 @@
 
 class AppTestSocketConnection(BaseConnectionTest):
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('_multiprocessing', 'thread', 'signal'))
+        space = gettestobjspace(usemodules=('_multiprocessing', 'thread', 'signal',
+                                            'struct', 'array'))
         cls.space = space
         cls.w_connections = space.newlist([])
 
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -6,7 +6,7 @@
 from pypy.rpython.lltypesystem import lltype, rffi
 
 def setup_module(mod):
-    mod.space = gettestobjspace(usemodules=['_socket', 'array'])
+    mod.space = gettestobjspace(usemodules=['_socket', 'array', 'struct'])
     global socket
     import socket
     mod.w_socket = space.appexec([], "(): import _socket as m; return m")
@@ -372,10 +372,9 @@
     def test_socket_connect(self):
         import _socket, os
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
-        # XXX temporarily we use python.org to test, will have more robust tests
-        # in the absence of a network connection later when more parts of the
-        # socket API are implemented.  Currently skip the test if there is no
-        # connection.
+        # it would be nice to have a test which works even if there is no
+        # network connection. However, this one is "good enough" for now. Skip
+        # it if there is no connection.
         try:
             s.connect(("www.python.org", 80))
         except _socket.gaierror, ex:
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -432,7 +432,8 @@
                     raise _ssl_seterror(self.space, self, length)
                 try:
                     # this is actually an immutable bytes sequence
-                    return self.space.wrap(rffi.charp2str(buf_ptr[0]))
+                    return self.space.wrap(rffi.charpsize2str(buf_ptr[0],
+                                                              length))
                 finally:
                     libssl_OPENSSL_free(buf_ptr[0])
         else:
diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py
--- a/pypy/module/_ssl/test/test_ssl.py
+++ b/pypy/module/_ssl/test/test_ssl.py
@@ -90,7 +90,7 @@
 
 class AppTestConnectedSSL:
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('_ssl', '_socket'))
+        space = gettestobjspace(usemodules=('_ssl', '_socket', 'struct'))
         cls.space = space
 
     def setup_method(self, method):
@@ -179,7 +179,7 @@
     # to exercise the poll() calls
 
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('_ssl', '_socket'))
+        space = gettestobjspace(usemodules=('_ssl', '_socket', 'struct'))
         cls.space = space
         cls.space.appexec([], """():
             import socket; socket.setdefaulttimeout(1)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -11,7 +11,7 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.unroll import unrolling_iterable
-from pypy.rlib.objectmodel import specialize
+from pypy.rlib.objectmodel import specialize, keepalive_until_here
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
@@ -145,18 +145,24 @@
 unroll_typecodes = unrolling_iterable(types.keys())
 
 class ArrayBuffer(RWBuffer):
-    def __init__(self, data, bytes):
-        self.data = data
-        self.len = bytes
+    def __init__(self, array):
+        self.array = array
 
     def getlength(self):
-        return self.len
+        return self.array.len * self.array.itemsize
 
     def getitem(self, index):
-        return self.data[index]
+        array = self.array
+        data = array._charbuf_start()
+        char = data[index]
+        array._charbuf_stop()
+        return char
 
     def setitem(self, index, char):
-        self.data[index] = char
+        array = self.array
+        data = array._charbuf_start()
+        data[index] = char
+        array._charbuf_stop()
 
 
 def make_array(mytype):
@@ -278,9 +284,10 @@
             oldlen = self.len
             new = len(s) / mytype.bytes
             self.setlen(oldlen + new)
-            cbuf = self.charbuf()
+            cbuf = self._charbuf_start()
             for i in range(len(s)):
                 cbuf[oldlen * mytype.bytes + i] = s[i]
+            self._charbuf_stop()
 
         def fromlist(self, w_lst):
             s = self.len
@@ -310,8 +317,11 @@
             else:
                 self.fromsequence(w_iterable)
 
-        def charbuf(self):
-            return  rffi.cast(rffi.CCHARP, self.buffer)
+        def _charbuf_start(self):
+            return rffi.cast(rffi.CCHARP, self.buffer)
+
+        def _charbuf_stop(self):
+            keepalive_until_here(self)
 
         def w_getitem(self, space, idx):
             item = self.buffer[idx]
@@ -530,8 +540,10 @@
         self.fromstring(space.str_w(w_s))
 
     def array_tostring__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        cbuf = self._charbuf_start()
+        s = rffi.charpsize2str(cbuf, self.len * mytype.bytes)
+        self._charbuf_stop()
+        return self.space.wrap(s)
 
     def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
         if not isinstance(w_f, W_File):
@@ -613,8 +625,7 @@
     # Misc methods
 
     def buffer__Array(space, self):
-        b = ArrayBuffer(self.charbuf(), self.len * mytype.bytes)
-        return space.wrap(b)
+        return space.wrap(ArrayBuffer(self))
 
     def array_buffer_info__Array(space, self):
         w_ptr = space.wrap(rffi.cast(lltype.Unsigned, self.buffer))
@@ -649,7 +660,7 @@
             raise OperationError(space.w_RuntimeError, space.wrap(msg))
         if self.len == 0:
             return
-        bytes = self.charbuf()
+        bytes = self._charbuf_start()
         tmp = [bytes[0]] * mytype.bytes
         for start in range(0, self.len * mytype.bytes, mytype.bytes):
             stop = start + mytype.bytes - 1
@@ -657,6 +668,7 @@
                 tmp[i] = bytes[start + i]
             for i in range(mytype.bytes):
                 bytes[stop - i] = tmp[i]
+        self._charbuf_stop()
 
     def repr__Array(space, self):
         if self.len == 0:
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -433,7 +433,25 @@
         a = self.array('h', 'Hi')
         buf = buffer(a)
         assert buf[1] == 'i'
-        #raises(TypeError, buf.__setitem__, 1, 'o')
+
+    def test_buffer_write(self):
+        a = self.array('c', 'hello')
+        buf = buffer(a)
+        print repr(buf)
+        try:
+            buf[3] = 'L'
+        except TypeError:
+            skip("buffer(array) returns a read-only buffer on CPython")
+        assert a.tostring() == 'helLo'
+
+    def test_buffer_keepalive(self):
+        buf = buffer(self.array('c', 'text'))
+        assert buf[2] == 'x'
+        #
+        a = self.array('c', 'foobarbaz')
+        buf = buffer(a)
+        a.fromstring('some extra text')
+        assert buf[:] == 'foobarbazsome extra text'
 
     def test_list_methods(self):
         assert repr(self.array('i')) == "array('i')"
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -2,8 +2,10 @@
 from pypy.module.cpyext.api import (
     cpython_api, Py_ssize_t, cpython_struct, bootstrap_function,
     PyObjectFields, PyObject)
-from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef
+from pypy.module.cpyext.pyobject import make_typedescr, Py_DecRef, make_ref
 from pypy.interpreter.buffer import Buffer, StringBuffer, SubBuffer
+from pypy.interpreter.error import OperationError
+from pypy.module.array.interp_array import ArrayBuffer
 
 
 PyBufferObjectStruct = lltype.ForwardReference()
@@ -41,26 +43,38 @@
         py_buf.c_b_offset = w_obj.offset
         w_obj = w_obj.buffer
 
+    # If w_obj already allocated a fixed buffer, use it, and keep a
+    # reference to w_obj.
+    # Otherwise, b_base stays NULL, and we own the b_ptr.
+
     if isinstance(w_obj, StringBuffer):
-        py_buf.c_b_base = rffi.cast(PyObject, 0) # space.wrap(w_obj.value)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.as_str()))
+        py_buf.c_b_base = lltype.nullptr(PyObject.TO)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(w_obj.value))
+        py_buf.c_b_size = w_obj.getlength()
+    elif isinstance(w_obj, ArrayBuffer):
+        w_base = w_obj.array
+        py_buf.c_b_base = make_ref(space, w_base)
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, w_obj.array._charbuf_start())
         py_buf.c_b_size = w_obj.getlength()
     else:
-        raise Exception("Fail fail fail fail fail")
+        raise OperationError(space.w_NotImplementedError, space.wrap(
+            "buffer flavor not supported"))
 
 
 def buffer_realize(space, py_obj):
     """
     Creates the buffer in the PyPy interpreter from a cpyext representation.
     """
-    raise Exception("realize fail fail fail")
-
+    raise OperationError(space.w_NotImplementedError, space.wrap(
+        "Don't know how to realize a buffer"))
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
 def buffer_dealloc(space, py_obj):
     py_buf = rffi.cast(PyBufferObject, py_obj)
-    Py_DecRef(space, py_buf.c_b_base)
-    rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
+    if py_buf.c_b_base:
+        Py_DecRef(space, py_buf.c_b_base)
+    else:
+        rffi.free_charp(rffi.cast(rffi.CCHARP, py_buf.c_b_ptr))
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.2"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.8.1"
+#define PYPY_VERSION "1.9.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -2,6 +2,7 @@
 
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError
+from pypy.interpreter import pytraceback
 from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING
 from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning
 from pypy.module.cpyext.pyobject import (
@@ -315,3 +316,65 @@
     It may be called without holding the interpreter lock."""
     space.check_signal_action.set_interrupt()
 
+ at cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
+def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
+    """---Cython extension---
+
+    Retrieve the exception info, as known from ``sys.exc_info()``.  This
+    refers to an exception that was already caught, not to an exception
+    that was freshly raised.  Returns new references for the three
+    objects, any of which may be *NULL*.  Does not modify the exception
+    info state.
+
+    .. note::
+
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_SetExcInfo` to restore or clear the exception
+       state.
+    """
+    ec = space.getexecutioncontext()
+    operror = ec.sys_exc_info()
+    if operror:
+        ptype[0] = make_ref(space, operror.w_type)
+        pvalue[0] = make_ref(space, operror.get_w_value(space))
+        ptraceback[0] = make_ref(space, space.wrap(operror.get_traceback()))
+    else:
+        ptype[0] = lltype.nullptr(PyObject.TO)
+        pvalue[0] = lltype.nullptr(PyObject.TO)
+        ptraceback[0] = lltype.nullptr(PyObject.TO)
+
+ at cpython_api([PyObject, PyObject, PyObject], lltype.Void)
+def PyErr_SetExcInfo(space, w_type, w_value, w_traceback):
+    """---Cython extension---
+
+    Set the exception info, as known from ``sys.exc_info()``.  This refers
+    to an exception that was already caught, not to an exception that was
+    freshly raised.  This function steals the references of the arguments.
+    To clear the exception state, pass *NULL* for all three arguments.
+    For general rules about the three arguments, see :c:func:`PyErr_Restore`.
+ 
+    .. note::
+ 
+       This function is not normally used by code that wants to handle
+       exceptions.  Rather, it can be used when code needs to save and
+       restore the exception state temporarily.  Use
+       :c:func:`PyErr_GetExcInfo` to read the exception state.
+    """
+    if w_value is None or space.is_w(w_value, space.w_None):
+        operror = None
+    else:
+        tb = None
+        if w_traceback is not None:
+            try:
+                tb = pytraceback.check_traceback(space, w_traceback, '?')
+            except OperationError:    # catch and ignore bogus objects
+                pass
+        operror = OperationError(w_type, w_value, tb)
+    #
+    ec = space.getexecutioncontext()
+    ec.set_sys_exc_info(operror)
+    Py_DecRef(space, w_type)
+    Py_DecRef(space, w_value)
+    Py_DecRef(space, w_traceback)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -167,14 +167,16 @@
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
 
+# Warning, confusing function name (like CPython).  Used only for sq_contains.
 def wrap_objobjproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjproc, func)
     check_num_args(space, w_args, 1)
     w_value, = space.fixedview(w_args)
     res = generic_cpy_call(space, func_target, w_self, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.wrap(bool(res))
 
 def wrap_objobjargproc(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
@@ -183,7 +185,7 @@
     res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
-    return space.wrap(res)
+    return space.w_None
 
 def wrap_delitem(space, w_self, w_args, func):
     func_target = rffi.cast(objobjargproc, func)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2253,24 +2253,6 @@
     """Concat two strings giving a new Unicode string."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
-def PyUnicode_Split(space, s, sep, maxsplit):
-    """Split a string giving a list of Unicode strings.  If sep is NULL, splitting
-    will be done at all whitespace substrings.  Otherwise, splits occur at the given
-    separator.  At most maxsplit splits will be done.  If negative, no limit is
-    set.  Separators are not included in the resulting list.
-
-    This function used an int type for maxsplit. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyUnicode_Splitlines(space, s, keepend):
-    """Split a Unicode string at line breaks, returning a list of Unicode strings.
-    CRLF is considered to be one line break.  If keepend is 0, the Line break
-    characters are not included in the resulting strings."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.CCHARP], PyObject)
 def PyUnicode_Translate(space, str, table, errors):
     """Translate a string by applying a character mapping table to it and return the
@@ -2287,29 +2269,6 @@
     use the default error handling."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t, error=-2)
-def PyUnicode_Find(space, str, substr, start, end, direction):
-    """Return the first position of substr in str*[*start:end] using the given
-    direction (direction == 1 means to do a forward search, direction == -1 a
-    backward search).  The return value is the index of the first match; a value of
-    -1 indicates that no match was found, and -2 indicates that an error
-    occurred and an exception has been set.
-
-    This function used an int type for start and end. This
-    might require changes in your code for properly supporting 64-bit
-    systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
-def PyUnicode_Count(space, str, substr, start, end):
-    """Return the number of non-overlapping occurrences of substr in
-    str[start:end].  Return -1 if an error occurred.
-
-    This function returned an int type and used an int
-    type for start and end. This might require changes in your code for
-    properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
 def PyUnicode_RichCompare(space, left, right, op):
     """Rich compare two unicode strings and return one of the following:
diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -10,7 +10,7 @@
     return False
 
 def pytest_funcarg__space(request):
-    return gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+    return gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
 
 def pytest_funcarg__api(request):
     return request.cls.api
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -176,6 +176,8 @@
     {NULL}  /* Sentinel */
 };
 
+PyDoc_STRVAR(foo_doc, "foo is for testing.");
+
 static PyTypeObject footype = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "foo.foo",               /*tp_name*/
@@ -198,7 +200,7 @@
     (setattrofunc)foo_setattro, /*tp_setattro*/
     0,                       /*tp_as_buffer*/
     Py_TPFLAGS_DEFAULT,      /*tp_flags*/
-    0,                       /*tp_doc*/
+    foo_doc,                 /*tp_doc*/
     0,                       /*tp_traverse*/
     0,                       /*tp_clear*/
     0,                       /*tp_richcompare*/
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -19,7 +19,8 @@
 
 class BaseApiTest(LeakCheckingTest):
     def setup_class(cls):
-        cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+        cls.space = space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi',
+                                                        'array'])
 
         # warm up reference counts:
         # - the posix module allocates a HCRYPTPROV on Windows
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -1,3 +1,4 @@
+from pypy.conftest import gettestobjspace
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 
 import py
diff --git a/pypy/module/cpyext/test/test_bufferobject.py b/pypy/module/cpyext/test/test_bufferobject.py
--- a/pypy/module/cpyext/test/test_bufferobject.py
+++ b/pypy/module/cpyext/test/test_bufferobject.py
@@ -48,3 +48,17 @@
             ])
         b = module.buffer_new()
         raises(AttributeError, getattr, b, 'x')
+
+    def test_array_buffer(self):
+        module = self.import_extension('foo', [
+            ("roundtrip", "METH_O",
+             """
+                 PyBufferObject *buf = (PyBufferObject *)args;
+                 return PyString_FromStringAndSize(buf->b_ptr, buf->b_size);
+             """),
+            ])
+        import array
+        a = array.array('c', 'text')
+        b = buffer(a)
+        assert module.roundtrip(b) == 'text'
+        
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -35,7 +35,7 @@
 
 class AppTestApi:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
         from pypy.rlib.libffi import get_libc_name
         cls.w_libc = cls.space.wrap(get_libc_name())
 
@@ -165,8 +165,9 @@
         return leaking
 
 class AppTestCpythonExtensionBase(LeakCheckingTest):
+    
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi'])
+        cls.space = gettestobjspace(usemodules=['cpyext', 'thread', '_rawffi', 'array'])
         cls.space.getbuiltinmodule("cpyext")
         from pypy.module.imp.importing import importhook
         importhook(cls.space, "os") # warm up reference counts
diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -19,7 +19,7 @@
                                          space.wrap('__name__'))) == 'foobar'
 
     def test_getmoduledict(self, space, api):
-        testmod = "binascii"
+        testmod = "_functools"
         w_pre_dict = api.PyImport_GetModuleDict()
         assert not space.is_true(space.contains(w_pre_dict, space.wrap(testmod)))
 
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -218,3 +218,51 @@
             assert e.filename == "blyf"
             assert e.errno == errno.EBADF
             assert e.strerror == os.strerror(errno.EBADF)
+
+    def test_GetSetExcInfo(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("getset_exc_info", "METH_VARARGS",
+             r'''
+             PyObject *type, *val, *tb;
+             PyObject *new_type, *new_val, *new_tb;
+             PyObject *result;
+
+             if (!PyArg_ParseTuple(args, "OOO", &new_type, &new_val, &new_tb))
+                 return NULL;
+
+             PyErr_GetExcInfo(&type, &val, &tb);
+
+             Py_INCREF(new_type);
+             Py_INCREF(new_val);
+             Py_INCREF(new_tb);
+             PyErr_SetExcInfo(new_type, new_val, new_tb);
+
+             result = Py_BuildValue("OOO",
+                                    type ? type : Py_None,
+                                    val  ? val  : Py_None,
+                                    tb   ? tb   : Py_None);
+             Py_XDECREF(type);
+             Py_XDECREF(val);
+             Py_XDECREF(tb);
+             return result;
+             '''
+             ),
+            ])
+        try:
+            raise ValueError(5)
+        except ValueError, old_exc:
+            new_exc = TypeError("TEST")
+            orig_sys_exc_info = sys.exc_info()
+            orig_exc_info = module.getset_exc_info(new_exc.__class__,
+                                                   new_exc, None)
+            new_sys_exc_info = sys.exc_info()
+            new_exc_info = module.getset_exc_info(*orig_exc_info)
+            reset_sys_exc_info = sys.exc_info()
+
+            assert orig_exc_info[0] is old_exc.__class__
+            assert orig_exc_info[1] is old_exc
+            assert orig_exc_info == orig_sys_exc_info
+            assert orig_exc_info == reset_sys_exc_info
+            assert new_exc_info == (new_exc.__class__, new_exc, None)
+            assert new_exc_info == new_sys_exc_info
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -20,6 +20,7 @@
         assert type(obj) is module.fooType
         print "type of obj has type", type(type(obj))
         print "type of type of obj has type", type(type(type(obj)))
+        assert module.fooType.__doc__ == "foo is for testing."
 
     def test_typeobject_method_descriptor(self):
         module = self.import_module(name='foo')
@@ -414,8 +415,11 @@
             static int
             mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value)
             {
-                PyErr_SetNone(PyExc_ZeroDivisionError);
-                return -1;
+                if (PyInt_Check(key)) {
+                    PyErr_SetNone(PyExc_ZeroDivisionError);
+                    return -1;
+                }
+                return 0;
             }
             PyMappingMethods tp_as_mapping;
             static PyTypeObject Foo_Type = {
@@ -425,6 +429,36 @@
             ''')
         obj = module.new_obj()
         raises(ZeroDivisionError, obj.__setitem__, 5, None)
+        res = obj.__setitem__('foo', None)
+        assert res is None
+
+    def test_sq_contains(self):
+        module = self.import_extension('foo', [
+           ("new_obj", "METH_NOARGS",
+            '''
+                PyObject *obj;
+                Foo_Type.tp_as_sequence = &tp_as_sequence;
+                tp_as_sequence.sq_contains = sq_contains;
+                if (PyType_Ready(&Foo_Type) < 0) return NULL;
+                obj = PyObject_New(PyObject, &Foo_Type);
+                return obj;
+            '''
+            )],
+            '''
+            static int
+            sq_contains(PyObject *self, PyObject *value)
+            {
+                return 42;
+            }
+            PySequenceMethods tp_as_sequence;
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            ''')
+        obj = module.new_obj()
+        res = "foo" in obj
+        assert res is True
 
     def test_tp_iter(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -457,3 +457,31 @@
         assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+
+    def test_count(self, space, api):
+        w_str = space.wrap(u"abcabdab")
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, -1) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, 2) == 1
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), -5, 30) == 2
+
+    def test_find(self, space, api):
+        w_str = space.wrap(u"abcabcd")
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, 1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, 1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
+
+    def test_split(self, space, api):
+        w_str = space.wrap(u"a\nb\nc\nd")
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
+        assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 0)))
+        assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 1)))
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -307,6 +307,8 @@
         if not space.is_true(space.issubtype(self, space.w_type)):
             self.flag_cpytype = True
         self.flag_heaptype = False
+        if pto.c_tp_doc:
+            self.w_doc = space.wrap(rffi.charp2str(pto.c_tp_doc))
 
 @bootstrap_function
 def init_typeobject(space):
@@ -624,7 +626,6 @@
     Creates an interpreter type from a PyTypeObject structure.
     """
     # missing:
-    # setting __doc__ if not defined and tp_doc defined
     # inheriting tp_as_* slots
     # unsupported:
     # tp_mro, tp_subclasses
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -598,3 +598,46 @@
     else:
         return stringtype.stringendswith(str, substr, start, end)
 
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
+def PyUnicode_Count(space, w_str, w_substr, start, end):
+    """Return the number of non-overlapping occurrences of substr in
+    str[start:end].  Return -1 if an error occurred."""
+    w_count = space.call_method(w_str, "count", w_substr,
+                                space.wrap(start), space.wrap(end))
+    return space.int_w(w_count)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
+             Py_ssize_t, error=-2)
+def PyUnicode_Find(space, w_str, w_substr, start, end, direction):
+    """Return the first position of substr in str*[*start:end] using
+    the given direction (direction == 1 means to do a forward search,
+    direction == -1 a backward search).  The return value is the index
+    of the first match; a value of -1 indicates that no match was
+    found, and -2 indicates that an error occurred and an exception
+    has been set."""
+    if rffi.cast(lltype.Signed, direction) > 0:
+        w_pos = space.call_method(w_str, "find", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    else:
+        w_pos = space.call_method(w_str, "rfind", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    return space.int_w(w_pos)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
+def PyUnicode_Split(space, w_str, w_sep, maxsplit):
+    """Split a string giving a list of Unicode strings.  If sep is
+    NULL, splitting will be done at all whitespace substrings.
+    Otherwise, splits occur at the given separator.  At most maxsplit
+    splits will be done.  If negative, no limit is set.  Separators
+    are not included in the resulting list."""
+    if w_sep is None:
+        w_sep = space.w_None
+    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyUnicode_Splitlines(space, w_str, keepend):
+    """Split a Unicode string at line breaks, returning a list of
+    Unicode strings.  CRLF is considered to be one line break.  If
+    keepend is 0, the Line break characters are not included in the
+    resulting strings."""
+    return space.call_method(w_str, "splitlines", space.wrap(keepend))
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -13,7 +13,7 @@
 
 class AppTestFcntl:
     def setup_class(cls):
-        space = gettestobjspace(usemodules=('fcntl', 'array'))
+        space = gettestobjspace(usemodules=('fcntl', 'array', 'struct'))
         cls.space = space
         tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_'))
         cls.w_tmp = space.wrap(tmpprefix)
diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -987,6 +987,10 @@
             os.environ['LANG'] = oldlang
 
 class AppTestImportHooks(object):
+
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('struct',))
+    
     def test_meta_path(self):
         tried_imports = []
         class Importer(object):
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -891,7 +891,7 @@
 
 class AppTestItertools27:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['itertools'])
+        cls.space = gettestobjspace(usemodules=['itertools', 'struct'])
         if cls.space.is_true(cls.space.appexec([], """():
             import sys; return sys.version_info < (2, 7)
             """)):
diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py
--- a/pypy/module/math/test/test_math.py
+++ b/pypy/module/math/test/test_math.py
@@ -6,7 +6,7 @@
 
 class AppTestMath:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['math'])
+        cls.space = gettestobjspace(usemodules=['math', 'struct'])
         cls.w_cases = cls.space.wrap(test_direct.MathTests.TESTCASES)
         cls.w_consistent_host = cls.space.wrap(test_direct.consistent_host)
 
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -29,6 +29,7 @@
         'flatiter': 'interp_numarray.W_FlatIterator',
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
+        'repeat': 'interp_numarray.repeat',
 
         'set_string_function': 'appbridge.set_string_function',
 
@@ -99,9 +100,12 @@
         ("exp2", "exp2"),
         ("expm1", "expm1"),
         ("fabs", "fabs"),
+        ("fmax", "fmax"),
+        ("fmin", "fmin"),
         ("fmod", "fmod"),
         ("floor", "floor"),
         ("ceil", "ceil"),
+        ("trunc", "trunc"),
         ("greater", "greater"),
         ("greater_equal", "greater_equal"),
         ("less", "less"),
@@ -122,12 +126,16 @@
         ("sinh", "sinh"),
         ("subtract", "subtract"),
         ('sqrt', 'sqrt'),
+        ('square', 'square'),
         ("tan", "tan"),
         ("tanh", "tanh"),
         ('bitwise_and', 'bitwise_and'),
         ('bitwise_or', 'bitwise_or'),
         ('bitwise_xor', 'bitwise_xor'),
         ('bitwise_not', 'invert'),
+        ('left_shift', 'left_shift'),
+        ('right_shift', 'right_shift'),
+        ('invert', 'invert'),
         ('isnan', 'isnan'),
         ('isinf', 'isinf'),
         ('isneginf', 'isneginf'),
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -16,7 +16,7 @@
         a[i][i] = 1
     return a
 
-def sum(a,axis=None):
+def sum(a,axis=None, out=None):
     '''sum(a, axis=None)
     Sum of array elements over a given axis.
 
@@ -43,17 +43,17 @@
     # TODO: add to doc (once it's implemented): cumsum : Cumulative sum of array elements.
     if not hasattr(a, "sum"):
         a = _numpypy.array(a)
-    return a.sum(axis)
+    return a.sum(axis=axis, out=out)
 
-def min(a, axis=None):
+def min(a, axis=None, out=None):
     if not hasattr(a, "min"):
         a = _numpypy.array(a)
-    return a.min(axis)
+    return a.min(axis=axis, out=out)
 
-def max(a, axis=None):
+def max(a, axis=None, out=None):
     if not hasattr(a, "max"):
         a = _numpypy.array(a)
-    return a.max(axis)
+    return a.max(axis=axis, out=out)
 
 def arange(start, stop=None, step=1, dtype=None):
     '''arange([start], stop[, step], dtype=None)
diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py
--- a/pypy/module/micronumpy/interp_boxes.py
+++ b/pypy/module/micronumpy/interp_boxes.py
@@ -62,21 +62,24 @@
         return space.wrap(dtype.itemtype.bool(self))
 
     def _binop_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                            [self, w_other, w_out])
         return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
 
     def _binop_right_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, 
+                                                            [w_other, self, w_out])
         return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
 
     def _unaryop_impl(ufunc_name):
-        def impl(self, space):
+        def impl(self, space, w_out=None):
             from pypy.module.micronumpy import interp_ufuncs
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                                    [self, w_out])
         return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name)
 
     descr_add = _binop_impl("add")
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -359,6 +359,7 @@
             name="int64",
             char="q",
             w_box_type=space.gettypefor(interp_boxes.W_Int64Box),
+            alternate_constructors=[space.w_long],
         )
         self.w_uint64dtype = W_Dtype(
             types.UInt64(),
@@ -386,23 +387,6 @@
             alternate_constructors=[space.w_float],
             aliases=["float"],
         )
-        self.w_longlongdtype = W_Dtype(
-            types.Int64(),
-            num=9,
-            kind=SIGNEDLTR,
-            name='int64',
-            char='q',
-            w_box_type = space.gettypefor(interp_boxes.W_LongLongBox),
-            alternate_constructors=[space.w_long],
-        )
-        self.w_ulonglongdtype = W_Dtype(
-            types.UInt64(),
-            num=10,
-            kind=UNSIGNEDLTR,
-            name='uint64',
-            char='Q',
-            w_box_type = space.gettypefor(interp_boxes.W_ULongLongBox),
-        )
         self.w_stringdtype = W_Dtype(
             types.StringType(1),
             num=18,
@@ -435,17 +419,19 @@
             self.w_booldtype, self.w_int8dtype, self.w_uint8dtype,
             self.w_int16dtype, self.w_uint16dtype, self.w_int32dtype,
             self.w_uint32dtype, self.w_longdtype, self.w_ulongdtype,
-            self.w_longlongdtype, self.w_ulonglongdtype,
+            self.w_int64dtype, self.w_uint64dtype,
             self.w_float32dtype,
             self.w_float64dtype, self.w_stringdtype, self.w_unicodedtype,
             self.w_voiddtype,
         ]
-        self.dtypes_by_num_bytes = sorted(
+        self.float_dtypes_by_num_bytes = sorted(
             (dtype.itemtype.get_element_size(), dtype)
-            for dtype in self.builtin_dtypes
+            for dtype in [self.w_float32dtype, self.w_float64dtype]
         )
         self.dtypes_by_name = {}
-        for dtype in self.builtin_dtypes:
+        # we reverse, so the stuff with lower numbers override stuff with
+        # higher numbers
+        for dtype in reversed(self.builtin_dtypes):
             self.dtypes_by_name[dtype.name] = dtype
             can_name = dtype.kind + str(dtype.itemtype.get_element_size())
             self.dtypes_by_name[can_name] = dtype
@@ -473,7 +459,7 @@
             'LONG': self.w_longdtype,
             'UNICODE': self.w_unicodedtype,
             #'OBJECT',
-            'ULONGLONG': self.w_ulonglongdtype,
+            'ULONGLONG': self.w_uint64dtype,
             'STRING': self.w_stringdtype,
             #'CDOUBLE',
             #'DATETIME',
diff --git a/pypy/module/micronumpy/interp_iter.py b/pypy/module/micronumpy/interp_iter.py
--- a/pypy/module/micronumpy/interp_iter.py
+++ b/pypy/module/micronumpy/interp_iter.py
@@ -269,7 +269,7 @@
 
     def apply_transformations(self, arr, transformations):
         v = BaseIterator.apply_transformations(self, arr, transformations)
-        if len(arr.shape) == 1:
+        if len(arr.shape) == 1 and len(v.res_shape) == 1:
             return OneDimIterator(self.offset, self.strides[0],
                                   self.res_shape[0])
         return v
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -83,8 +83,9 @@
         return space.wrap(W_NDimArray(shape[:], dtype=dtype))
 
     def _unaryop_impl(ufunc_name):
-        def impl(self, space):
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
+        def impl(self, space, w_out=None):
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                                [self, w_out])
         return func_with_new_name(impl, "unaryop_%s_impl" % ufunc_name)
 
     descr_pos = _unaryop_impl("positive")
@@ -93,8 +94,9 @@
     descr_invert = _unaryop_impl("invert")
 
     def _binop_impl(ufunc_name):
-        def impl(self, space, w_other):
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+        def impl(self, space, w_other, w_out=None):
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space,
+                                                        [self, w_other, w_out])
         return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
 
     descr_add = _binop_impl("add")
@@ -124,12 +126,12 @@
         return space.newtuple([w_quotient, w_remainder])
 
     def _binop_right_impl(ufunc_name):
-        def impl(self, space, w_other):
+        def impl(self, space, w_other, w_out=None):
             w_other = scalar_w(space,
                 interp_ufuncs.find_dtype_for_scalar(space, w_other, self.find_dtype()),
                 w_other
             )
-            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self])
+            return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [w_other, self, w_out])
         return func_with_new_name(impl, "binop_right_%s_impl" % ufunc_name)
 
     descr_radd = _binop_right_impl("add")
@@ -152,13 +154,21 @@
         return space.newtuple([w_quotient, w_remainder])
 
     def _reduce_ufunc_impl(ufunc_name, promote_to_largest=False):
-        def impl(self, space, w_axis=None):
+        def impl(self, space, w_axis=None, w_out=None):
             if space.is_w(w_axis, space.w_None):
                 axis = -1
             else:
                 axis = space.int_w(w_axis)
+            if space.is_w(w_out, space.w_None) or not w_out:
+                out = None
+            elif not isinstance(w_out, BaseArray):
+                raise OperationError(space.w_TypeError, space.wrap( 
+                        'output must be an array'))
+            else:
+                out = w_out
             return getattr(interp_ufuncs.get(space), ufunc_name).reduce(space,
-                                        self, True, promote_to_largest, axis)
+                                        self, True, promote_to_largest, axis,
+                                                                   False, out)
         return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name)
 
     descr_sum = _reduce_ufunc_impl("add")
@@ -213,6 +223,7 @@
     def descr_dot(self, space, w_other):
         other = convert_to_array(space, w_other)
         if isinstance(other, Scalar):
+            #Note: w_out is not modified, this is numpy compliant.
             return self.descr_mul(space, other)
         elif len(self.shape) < 2 and len(other.shape) < 2:
             w_res = self.descr_mul(space, other)
@@ -502,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -514,14 +548,14 @@
             )
         return w_result
 
-    def descr_mean(self, space, w_axis=None):
+    def descr_mean(self, space, w_axis=None, w_out=None):
         if space.is_w(w_axis, space.w_None):
             w_axis = space.wrap(-1)
             w_denom = space.wrap(support.product(self.shape))
         else:
             dim = space.int_w(w_axis)
             w_denom = space.wrap(self.shape[dim])
-        return space.div(self.descr_sum_promote(space, w_axis), w_denom)
+        return space.div(self.descr_sum_promote(space, w_axis, w_out), w_denom)
 
     def descr_var(self, space, w_axis=None):
         return get_appbridge_cache(space).call_method(space, '_var', self,
@@ -662,6 +696,10 @@
     def compute_first_step(self, sig, frame):
         pass
 
+    @unwrap_spec(repeats=int)
+    def descr_repeat(self, space, repeats, w_axis=None):
+        return repeat(space, self, repeats, w_axis)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -714,11 +752,12 @@
     """
     Class for representing virtual arrays, such as binary ops or ufuncs
     """
-    def __init__(self, name, shape, res_dtype):
+    def __init__(self, name, shape, res_dtype, out_arg=None):
         BaseArray.__init__(self, shape)
         self.forced_result = None
         self.res_dtype = res_dtype
         self.name = name
+        self.res = out_arg
         self.size = support.product(self.shape) * res_dtype.get_size()
 
     def _del_sources(self):
@@ -727,13 +766,18 @@
         raise NotImplementedError
 
     def compute(self):
-        ra = ResultArray(self, self.shape, self.res_dtype)
+        ra = ResultArray(self, self.shape, self.res_dtype, self.res)
         loop.compute(ra)
+        if self.res:
+            broadcast_dims = len(self.res.shape) - len(self.shape)
+            chunks = [Chunk(0,0,0,0)] * broadcast_dims + \
+                     [Chunk(0, i, 1, i) for i in self.shape]
+            return Chunks(chunks).apply(self.res)
         return ra.left
 
     def force_if_needed(self):
         if self.forced_result is None:
-            self.forced_result = self.compute()
+            self.forced_result = self.compute().get_concrete()
             self._del_sources()
 
     def get_concrete(self):
@@ -773,8 +817,9 @@
 
 
 class Call1(VirtualArray):
-    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values):
-        VirtualArray.__init__(self, name, shape, res_dtype)
+    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, values,
+                                                            out_arg=None):
+        VirtualArray.__init__(self, name, shape, res_dtype, out_arg)
         self.values = values
         self.size = values.size
         self.ufunc = ufunc
@@ -786,6 +831,12 @@
     def create_sig(self):
         if self.forced_result is not None:
             return self.forced_result.create_sig()
+        if self.shape != self.values.shape:
+            #This happens if out arg is used
+            return signature.BroadcastUfunc(self.ufunc, self.name,
+                                            self.calc_dtype,
+                                            self.values.create_sig(),
+                                            self.res.create_sig())
         return signature.Call1(self.ufunc, self.name, self.calc_dtype,
                                self.values.create_sig())
 
@@ -793,8 +844,9 @@
     """
     Intermediate class for performing binary operations.
     """
-    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right):
-        VirtualArray.__init__(self, name, shape, res_dtype)
+    def __init__(self, ufunc, name, shape, calc_dtype, res_dtype, left, right,
+            out_arg=None):
+        VirtualArray.__init__(self, name, shape, res_dtype, out_arg)
         self.ufunc = ufunc
         self.left = left
         self.right = right
@@ -832,8 +884,13 @@
         Call2.__init__(self, None, 'assign', shape, dtype, dtype, res, child)
 
     def create_sig(self):
-        return signature.ResultSignature(self.res_dtype, self.left.create_sig(),
-                                         self.right.create_sig())
+        if self.left.shape != self.right.shape:
+            sig = signature.BroadcastResultSignature(self.res_dtype,
+                        self.left.create_sig(), self.right.create_sig())
+        else:
+            sig = signature.ResultSignature(self.res_dtype, 
+                        self.left.create_sig(), self.right.create_sig())
+        return sig
 
 class ToStringArray(Call1):
     def __init__(self, child):
@@ -842,9 +899,9 @@
         self.s = StringBuilder(child.size * self.item_size)
         Call1.__init__(self, None, 'tostring', child.shape, dtype, dtype,
                        child)
-        self.res = W_NDimArray([1], dtype, 'C')
-        self.res_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
-                                    self.res.storage)
+        self.res_str = W_NDimArray([1], dtype, order='C')
+        self.res_str_casted = rffi.cast(rffi.CArrayPtr(lltype.Char),
+                                    self.res_str.storage)
 
     def create_sig(self):
         return signature.ToStringSignature(self.calc_dtype,
@@ -950,7 +1007,7 @@
 
     def setitem(self, item, value):
         self.invalidated()
-        self.dtype.setitem(self, item, value)
+        self.dtype.setitem(self, item, value.convert_to(self.dtype))
 
     def calc_strides(self, shape):
         dtype = self.find_dtype()
@@ -1125,7 +1182,8 @@
 
 @unwrap_spec(subok=bool, copy=bool, ownmaskna=bool)
 def array(space, w_item_or_iterable, w_dtype=None, w_order=None,
-          subok=True, copy=True, w_maskna=None, ownmaskna=False):
+          subok=True, copy=True, w_maskna=None, ownmaskna=False,
+          w_ndmin=None):
     # find scalar
     if w_maskna is None:
         w_maskna = space.w_None
@@ -1170,8 +1228,13 @@
                 break
         if dtype is None:
             dtype = interp_dtype.get_dtype_cache(space).w_float64dtype
+    shapelen = len(shape)
+    if w_ndmin is not None and not space.is_w(w_ndmin, space.w_None):
+        ndmin = space.int_w(w_ndmin)
+        if ndmin > shapelen:
+            shape = [1] * (ndmin - shapelen) + shape
+            shapelen = ndmin
     arr = W_NDimArray(shape[:], dtype=dtype, order=order)
-    shapelen = len(shape)
     arr_iter = arr.create_iter()
     # XXX we might want to have a jitdriver here
     for i in range(len(elems_w)):
@@ -1225,6 +1288,31 @@
         return convert_to_array(space, w_obj2).descr_dot(space, w_arr)
     return w_arr.descr_dot(space, w_obj2)
 
+ at unwrap_spec(repeats=int)
+def repeat(space, w_arr, repeats, w_axis=None):
+    arr = convert_to_array(space, w_arr)
+    if space.is_w(w_axis, space.w_None):
+        arr = arr.descr_flatten(space).get_concrete()
+        orig_size = arr.shape[0]
+        shape = [arr.shape[0] * repeats]
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            Chunks([Chunk(i, shape[0] - repeats + i, repeats,
+                          orig_size)]).apply(res).setslice(space, arr)
+    else:
+        arr = arr.get_concrete()
+        axis = space.int_w(w_axis)
+        shape = arr.shape[:]
+        chunks = [Chunk(0, i, 1, i) for i in shape]
+        orig_size = shape[axis]
+        shape[axis] *= repeats
+        res = W_NDimArray(shape, arr.find_dtype())
+        for i in range(repeats):
+            chunks[axis] = Chunk(i, shape[axis] - repeats + i, repeats,
+                                 orig_size)
+            Chunks(chunks).apply(res).setslice(space, arr)
+    return res
+
 @unwrap_spec(axis=int)
 def concatenate(space, w_args, axis=0):
     args_w = space.listview(w_args)
@@ -1347,9 +1435,11 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
+    repeat = interp2app(BaseArray.descr_repeat),
 )
 
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -3,9 +3,11 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, NoneNotWrapped
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_boxes, interp_dtype, support, loop
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
+
 class W_Ufunc(Wrappable):
     _attrs_ = ["name", "promote_to_float", "promote_bools", "identity"]
     _immutable_fields_ = ["promote_to_float", "promote_bools", "name"]
@@ -28,26 +30,38 @@
         return self.identity
 
     def descr_call(self, space, __args__):
+        from interp_numarray import BaseArray
         args_w, kwds_w = __args__.unpack()
         # it occurs to me that we don't support any datatypes that
         # require casting, change it later when we do
         kwds_w.pop('casting', None)
         w_subok = kwds_w.pop('subok', None)
         w_out = kwds_w.pop('out', space.w_None)
-        if ((w_subok is not None and space.is_true(w_subok)) or
-            not space.is_w(w_out, space.w_None)):
+        # Setup a default value for out
+        if space.is_w(w_out, space.w_None):
+            out = None
+        else:
+            out = w_out
+        if (w_subok is not None and space.is_true(w_subok)):
             raise OperationError(space.w_NotImplementedError,
                                  space.wrap("parameters unsupported"))
         if kwds_w or len(args_w) < self.argcount:
             raise OperationError(space.w_ValueError,
                 space.wrap("invalid number of arguments")
             )
-        elif len(args_w) > self.argcount:
-            # The extra arguments should actually be the output array, but we
-            # don't support that yet.
+        elif (len(args_w) > self.argcount and out is not None) or \
+             (len(args_w) > self.argcount + 1):
             raise OperationError(space.w_TypeError,
                 space.wrap("invalid number of arguments")
             )
+        # Override the default out value, if it has been provided in w_wargs
+        if len(args_w) > self.argcount:
+            out = args_w[-1]
+        else:
+            args_w = args_w[:] + [out]
+        if out is not None and not isinstance(out, BaseArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                                            'output must be an array'))
         return self.call(space, args_w)
 
     @unwrap_spec(skipna=bool, keepdims=bool)
@@ -105,28 +119,33 @@
         array([[ 1,  5],
                [ 9, 13]])
         """
-        if not space.is_w(w_out, space.w_None):
-            raise OperationError(space.w_NotImplementedError, space.wrap(
-                "out not supported"))
+        from pypy.module.micronumpy.interp_numarray import BaseArray
         if w_axis is None:
             axis = 0
         elif space.is_w(w_axis, space.w_None):
             axis = -1
         else:
             axis = space.int_w(w_axis)
-        return self.reduce(space, w_obj, False, False, axis, keepdims)
+        if space.is_w(w_out, space.w_None):
+            out = None
+        elif not isinstance(w_out, BaseArray):
+            raise OperationError(space.w_TypeError, space.wrap(
+                                                'output must be an array'))
+        else:
+            out = w_out
+        return self.reduce(space, w_obj, False, False, axis, keepdims, out)
 
-    def reduce(self, space, w_obj, multidim, promote_to_largest, dim,
-               keepdims=False):
+    def reduce(self, space, w_obj, multidim, promote_to_largest, axis,
+               keepdims=False, out=None):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, \
-                                                           Scalar, ReduceArray
+                                             Scalar, ReduceArray, W_NDimArray
         if self.argcount != 2:
             raise OperationError(space.w_ValueError, space.wrap("reduce only "
                 "supported for binary functions"))
         assert isinstance(self, W_Ufunc2)
         obj = convert_to_array(space, w_obj)
-        if dim >= len(obj.shape):
-            raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % dim))
+        if axis >= len(obj.shape):
+            raise OperationError(space.w_ValueError, space.wrap("axis(=%d) out of bounds" % axis))
         if isinstance(obj, Scalar):
             raise OperationError(space.w_TypeError, space.wrap("cannot reduce "
                 "on a scalar"))
@@ -144,21 +163,55 @@
         if self.identity is None and size == 0:
             raise operationerrfmt(space.w_ValueError, "zero-size array to "
                     "%s.reduce without identity", self.name)
-        if shapelen > 1 and dim >= 0:
-            return self.do_axis_reduce(obj, dtype, dim, keepdims)
-        arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
-        return loop.compute(arr)
+        if shapelen > 1 and axis >= 0:
+            if keepdims:
+                shape = obj.shape[:axis] + [1] + obj.shape[axis + 1:]
+            else:
+                shape = obj.shape[:axis] + obj.shape[axis + 1:]
+            if out:
+                #Test for shape agreement
+                if len(out.shape) > len(shape):
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter for reduction operation %s' +
+                        ' has too many dimensions', self.name)
+                elif len(out.shape) < len(shape):
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter for reduction operation %s' +
+                        ' does not have enough dimensions', self.name)
+                elif out.shape != shape:
+                    raise operationerrfmt(space.w_ValueError,
+                        'output parameter shape mismatch, expecting [%s]' +
+                        ' , got [%s]',
+                        ",".join([str(x) for x in shape]),
+                        ",".join([str(x) for x in out.shape]),
+                        )
+                #Test for dtype agreement, perhaps create an itermediate
+                #if out.dtype != dtype:
+                #    raise OperationError(space.w_TypeError, space.wrap(
+                #        "mismatched  dtypes"))
+                return self.do_axis_reduce(obj, out.find_dtype(), axis, out)
+            else:
+                result = W_NDimArray(shape, dtype)
+                return self.do_axis_reduce(obj, dtype, axis, result)
+        if out:
+            if len(out.shape)>0:
+                raise operationerrfmt(space.w_ValueError, "output parameter "
+                              "for reduction operation %s has too many"
+                              " dimensions",self.name)
+            arr = ReduceArray(self.func, self.name, self.identity, obj,
+                                                            out.find_dtype())
+            val = loop.compute(arr)
+            assert isinstance(out, Scalar)
+            out.value = val
+        else:
+            arr = ReduceArray(self.func, self.name, self.identity, obj, dtype)
+            val = loop.compute(arr)
+        return val
 
-    def do_axis_reduce(self, obj, dtype, dim, keepdims):
-        from pypy.module.micronumpy.interp_numarray import AxisReduce,\
-             W_NDimArray
-        if keepdims:
-            shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
-        else:
-            shape = obj.shape[:dim] + obj.shape[dim + 1:]
-        result = W_NDimArray(shape, dtype)
+    def do_axis_reduce(self, obj, dtype, axis, result):
+        from pypy.module.micronumpy.interp_numarray import AxisReduce
         arr = AxisReduce(self.func, self.name, self.identity, obj.shape, dtype,
-                         result, obj, dim)
+                         result, obj, axis)
         loop.compute(arr)
         return arr.left
 
@@ -176,24 +229,55 @@
         self.bool_result = bool_result
 
     def call(self, space, args_w):
-        from pypy.module.micronumpy.interp_numarray import (Call1,
-            convert_to_array, Scalar)
-
-        [w_obj] = args_w
+        from pypy.module.micronumpy.interp_numarray import (Call1, BaseArray,
+            convert_to_array, Scalar, shape_agreement)
+        if len(args_w)<2:
+            [w_obj] = args_w
+            out = None
+        else:
+            [w_obj, out] = args_w
+            if space.is_w(out, space.w_None):
+                out = None
         w_obj = convert_to_array(space, w_obj)
         calc_dtype = find_unaryop_result_dtype(space,
                                   w_obj.find_dtype(),
                                   promote_to_float=self.promote_to_float,
                                   promote_bools=self.promote_bools)
-        if self.bool_result:
+        if out:
+            if not isinstance(out, BaseArray):
+                raise OperationError(space.w_TypeError, space.wrap(
+                                                'output must be an array'))
+            res_dtype = out.find_dtype()
+        elif self.bool_result:
             res_dtype = interp_dtype.get_dtype_cache(space).w_booldtype
         else:
             res_dtype = calc_dtype
         if isinstance(w_obj, Scalar):
-            return space.wrap(self.func(calc_dtype, w_obj.value.convert_to(calc_dtype)))
-
-        w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype, res_dtype,
-                      w_obj)
+            arr = self.func(calc_dtype, w_obj.value.convert_to(calc_dtype))
+            if isinstance(out,Scalar):
+                out.value = arr
+            elif isinstance(out, BaseArray):
+                out.fill(space, arr)
+            else:
+                out = arr
+            return space.wrap(out)
+        if out:
+            assert isinstance(out, BaseArray) # For translation
+            broadcast_shape =  shape_agreement(space, w_obj.shape, out.shape)
+            if not broadcast_shape or broadcast_shape != out.shape:
+                raise operationerrfmt(space.w_ValueError,
+                    'output parameter shape mismatch, could not broadcast [%s]' +
+                    ' to [%s]',
+                    ",".join([str(x) for x in w_obj.shape]),
+                    ",".join([str(x) for x in out.shape]),
+                    )
+            w_res = Call1(self.func, self.name, out.shape, calc_dtype,
+                                         res_dtype, w_obj, out)
+            #Force it immediately
+            w_res.get_concrete()
+        else:
+            w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
+                                         res_dtype, w_obj)
         w_obj.add_invalidates(w_res)
         return w_res
 
@@ -210,34 +294,63 @@
         self.func = func
         self.comparison_func = comparison_func
 
+    @jit.unroll_safe
     def call(self, space, args_w):
         from pypy.module.micronumpy.interp_numarray import (Call2,
-            convert_to_array, Scalar, shape_agreement)
-
-        [w_lhs, w_rhs] = args_w
+            convert_to_array, Scalar, shape_agreement, BaseArray)
+        if len(args_w) > 2:
+            [w_lhs, w_rhs, w_out] = args_w
+        else:
+            [w_lhs, w_rhs] = args_w
+            w_out = None
         w_lhs = convert_to_array(space, w_lhs)
         w_rhs = convert_to_array(space, w_rhs)
-        calc_dtype = find_binop_result_dtype(space,
-            w_lhs.find_dtype(), w_rhs.find_dtype(),
-            int_only=self.int_only,
-            promote_to_float=self.promote_to_float,
-            promote_bools=self.promote_bools,
-        )
+        if space.is_w(w_out, space.w_None) or w_out is None:
+            out = None
+            calc_dtype = find_binop_result_dtype(space,
+                w_lhs.find_dtype(), w_rhs.find_dtype(),
+                int_only=self.int_only,
+                promote_to_float=self.promote_to_float,
+                promote_bools=self.promote_bools,

From noreply at buildbot.pypy.org  Tue Apr 17 11:25:03 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 11:25:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: (antocuni,
 amaury and arigato around): bah,
 space.bool_w does not work generally. Use space.nonzero instead
Message-ID: <20120417092503.45B10820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54455:cf29881e5624
Date: 2012-04-17 11:24 +0200
http://bitbucket.org/pypy/pypy/changeset/cf29881e5624/

Log:	(antocuni, amaury and arigato around): bah, space.bool_w does not
	work generally. Use space.nonzero instead

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -415,7 +415,7 @@
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
             w_result = space.get_and_call_function(w_descr, w_container, w_item)
-            return space.wrap(space.bool_w(w_result))
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -697,10 +697,15 @@
     def test_bool___contains__(self):
         class X(object):
             def __contains__(self, item):
-                return 42
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
         x = X()
         res = 'foo' in x
         assert res is True
+        res = 'bar' in x
+        assert res is True
         #
         class CannotConvertToBool(object):
             def __nonzero__(self):

From noreply at buildbot.pypy.org  Tue Apr 17 11:30:53 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 11:30:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Check that we get the exact exception
 raised, not an unrelated TypeError.
Message-ID: <20120417093053.33CC8820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54456:f4cd131f5c40
Date: 2012-04-17 11:30 +0200
http://bitbucket.org/pypy/pypy/changeset/f4cd131f5c40/

Log:	Check that we get the exact exception raised, not an unrelated
	TypeError.

diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -707,14 +707,16 @@
         res = 'bar' in x
         assert res is True
         #
+        class MyError(Exception):
+            pass
         class CannotConvertToBool(object):
             def __nonzero__(self):
-                raise TypeError
+                raise MyError
         class X(object):
             def __contains__(self, item):
                 return CannotConvertToBool()
         x = X()
-        raises(TypeError, "'foo' in x")
+        raises(MyError, "'foo' in x")
         
             
 

From noreply at buildbot.pypy.org  Tue Apr 17 11:59:14 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 11:59:14 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: We really need to know if we're starting
 a new or restarting an aborted
Message-ID: <20120417095914.DF670820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54457:93da8b4ece49
Date: 2012-04-17 11:58 +0200
http://bitbucket.org/pypy/pypy/changeset/93da8b4ece49/

Log:	We really need to know if we're starting a new or restarting an
	aborted transaction.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -14,7 +14,7 @@
 @specialize.memo()
 def _get_stm_callback(func, argcls):
     def _stm_callback(llarg, retry_counter):
-        llop.stm_start_transaction(lltype.Void)
+        llop.stm_start_transaction(lltype.Void, retry_counter)
         if we_are_translated():
             llarg = rffi.cast(rclass.OBJECTPTR, llarg)
             arg = cast_base_ptr_to_instance(argcls, llarg)
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -632,7 +632,7 @@
     from pypy.translator.stm import stmgcintf
     stmgcintf.StmOperations.del_tls()
 
-def op_stm_start_transaction():
+def op_stm_start_transaction(retry_counter):
     pass
 
 def op_stm_commit_transaction():
diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -121,7 +121,7 @@
         #
         from pypy.rpython.memory.gc.stmtls import StmGCTLS
         self.main_thread_tls = StmGCTLS(self, in_main_thread=True)
-        self.main_thread_tls.start_transaction()
+        self.main_thread_tls.start_transaction(-1)
 
     def setup_thread(self):
         from pypy.rpython.memory.gc.stmtls import StmGCTLS
@@ -195,8 +195,8 @@
         if gen > 0:
             debug_print("XXX not doing a global collect()")
 
-    def start_transaction(self):
-        self.get_tls().start_transaction()
+    def start_transaction(self, retry_counter):
+        self.get_tls().start_transaction(retry_counter)
 
     def commit_transaction(self):
         self.get_tls().stop_transaction()
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -117,19 +117,24 @@
                 if value is not self:
                     del StmGCTLS.nontranslated_dict[key]
         self.stm_operations.leave_transactional_mode()
-        self.start_transaction()
+        self.start_transaction(-1)
 
-    def start_transaction(self):
+    def start_transaction(self, retry_counter):
         """Start a transaction: performs any pending cleanups, and set
         up a fresh state for allocating.  Called at the start of
         each transaction, and at the start of the main thread."""
         # Note that the calls to enter() and
         # end_of_transaction_collection() are not balanced: if a
-        # transaction is aborted, the latter might never be called.
+        # transaction is aborted, the latter might never be called
+        # and we get back to here with retry_counter > 0.
         # Be ready here to clean up any state.
         self._cleanup_state()
-        if self is not self.gc.main_thread_tls:
-            self.gc.root_walker.clear_current_stack_roots()
+        rw = self.gc.root_walker
+        if retry_counter > 0:
+            rw.set_current_stack_roots_limit(self.stack_root_limit)
+        else:
+            self.stack_root_limit = rw.get_current_stack_roots_limit()
+        #
         if self.nursery_free:
             clear_size = self.nursery_free - self.nursery_start
         else:
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -135,8 +135,8 @@
 class FakeRootWalker:
     def walk_current_stack_roots(self, *args):
         pass     # no stack roots in this test file
-    def clear_current_stack_roots(self):
-        pass
+    def get_current_stack_roots_limit(self):
+        return "some limit"
 
 
 class StmGCTests:
@@ -189,7 +189,7 @@
         self.gc.stm_operations.threadnum = threadnum
         if threadnum not in self.gc.stm_operations._tls_dict:
             self.gc.setup_thread()
-            self.gc.start_transaction()
+            self.gc.start_transaction(0)
     def gcsize(self, S):
         return (llmemory.raw_malloc_usage(llmemory.sizeof(self.gc.HDR)) +
                 llmemory.raw_malloc_usage(llmemory.sizeof(S)))
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -38,6 +38,8 @@
             P = lltype.typeOf(self.current_stack[i])
             self.current_stack[i] = llmemory.cast_adr_to_ptr(roots[i], P)
         lltype.free(roots, flavor='raw')
+    def get_current_stack_roots_limit(self):
+        return "some limit"
 
 class FakeGC:
     from pypy.rpython.memory.support import AddressDict, null_address_dict
@@ -83,7 +85,7 @@
         self.gctls_main = StmGCTLS(self.gc, in_main_thread=True)
         self.gctls_thrd = StmGCTLS(self.gc, in_main_thread=False)
         self.gc.main_thread_tls = self.gctls_main
-        self.gctls_main.start_transaction()
+        self.gctls_main.start_transaction(-1)
         self.gc.root_walker.current_stack = self.current_stack
 
     def stack_add(self, p):
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -42,7 +42,7 @@
             [s_gc], annmodel.s_None)
         self.stm_start_ptr = getfn(
             self.gcdata.gc.start_transaction.im_func,
-            [s_gc], annmodel.s_None)
+            [s_gc, annmodel.SomeInteger()], annmodel.s_None)
         self.stm_commit_ptr = getfn(
             self.gcdata.gc.commit_transaction.im_func,
             [s_gc], annmodel.s_None)
@@ -84,7 +84,9 @@
         hop.genop('cast_adr_to_ptr', [v_globaladr], resultvar=op.result)
 
     def gct_stm_start_transaction(self, hop):
-        hop.genop("direct_call", [self.stm_start_ptr, self.c_const_gc])
+        v_int = hop.spaceop.args[0]
+        hop.genop("direct_call", [self.stm_start_ptr,
+                                  self.c_const_gc, v_int])
 
     def gct_stm_commit_transaction(self, hop):
         hop.genop("direct_call", [self.stm_commit_ptr, self.c_const_gc])
@@ -170,8 +172,10 @@
                            stackgcdata.root_stack_base,
                            stackgcdata.root_stack_top)
 
-    def clear_current_stack_roots(self):
+    def get_current_stack_roots_limit(self):
+        return self.stackgcdata.root_stack_top
+
+    def set_current_stack_roots_limit(self, limit):
         """Used when we start a transaction: there might be garbage left
         behind by the previous aborted transaction."""
-        stackgcdata = self.stackgcdata
-        stackgcdata.root_stack_top = stackgcdata.root_stack_base
+        self.stackgcdata.root_stack_top = limit

From noreply at buildbot.pypy.org  Tue Apr 17 12:16:29 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Apr 2012 12:16:29 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: oops, forgot to unpin this
Message-ID: <20120417101629.0D78B820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54458:f698795d565a
Date: 2012-04-17 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/f698795d565a/

Log:	oops, forgot to unpin this

diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -785,6 +785,7 @@
         assert allocated_size >= needed_size
 
         if gc_buf:
+            rgc.unpin(gc_buf)
             if allocated_size != needed_size:
                 gc_buf = rgc.ll_shrink_array(gc_buf, needed_size)
             return hlstrtype(gc_buf)

From noreply at buildbot.pypy.org  Tue Apr 17 14:03:25 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 14:03:25 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix the optimization for sys.exc_info().
Message-ID: <20120417120325.C7BAF82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3k
Changeset: r54461:062d8a9acdf2
Date: 2012-04-17 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/062d8a9acdf2/

Log:	Fix the optimization for sys.exc_info().

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -114,13 +114,10 @@
     #       BINARY_SUBSCR
     # or:
     #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST any integer or None
     #       LOAD_CONST <=2
-    #       SLICE_2
-    # or:
-    #       CALL_FUNCTION/CALL_METHOD
-    #       LOAD_CONST any integer
-    #       LOAD_CONST <=2
-    #       SLICE_3
+    #       BUILD_SLICE 2
+    #       BINARY_SUBSCR
     need_all_three_args = True
     co = frame.getcode().co_code
     p = frame.last_instr
@@ -130,16 +127,16 @@
             lo = ord(co[p+4])
             hi = ord(co[p+5])
             w_constant = frame.getconstant_w((hi * 256) | lo)
-            if space.isinstance_w(w_constant, space.w_int):
-                constant = space.int_w(w_constant)
-                if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+            if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+                if space.isinstance_w(w_constant, space.w_int):
+                    constant = space.int_w(w_constant)
                     if -3 <= constant <= 1 and constant != -1:
                         need_all_three_args = False
-                elif ord(co[p+6]) == stdlib_opcode.SLICE+2:
-                    if constant <= 2:
-                        need_all_three_args = False
-                elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
-                      ord(co[p+9]) == stdlib_opcode.SLICE+3):
+            elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
+                  ord(co[p+9]) == stdlib_opcode.BUILD_SLICE and
+                  ord(co[p+12]) == stdlib_opcode.BINARY_SUBSCR):
+                if (space.is_w(w_constant, space.w_None) or
+                    space.isinstance_w(w_constant, space.w_int)):
                     lo = ord(co[p+7])
                     hi = ord(co[p+8])
                     w_constant = frame.getconstant_w((hi * 256) | lo)

From noreply at buildbot.pypy.org  Tue Apr 17 14:03:23 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 14:03:23 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Backed out changeset 93da8b4ece49
Message-ID: <20120417120323.488EB820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54459:12759f6412ee
Date: 2012-04-17 13:02 +0200
http://bitbucket.org/pypy/pypy/changeset/12759f6412ee/

Log:	Backed out changeset 93da8b4ece49

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -14,7 +14,7 @@
 @specialize.memo()
 def _get_stm_callback(func, argcls):
     def _stm_callback(llarg, retry_counter):
-        llop.stm_start_transaction(lltype.Void, retry_counter)
+        llop.stm_start_transaction(lltype.Void)
         if we_are_translated():
             llarg = rffi.cast(rclass.OBJECTPTR, llarg)
             arg = cast_base_ptr_to_instance(argcls, llarg)
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -632,7 +632,7 @@
     from pypy.translator.stm import stmgcintf
     stmgcintf.StmOperations.del_tls()
 
-def op_stm_start_transaction(retry_counter):
+def op_stm_start_transaction():
     pass
 
 def op_stm_commit_transaction():
diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -121,7 +121,7 @@
         #
         from pypy.rpython.memory.gc.stmtls import StmGCTLS
         self.main_thread_tls = StmGCTLS(self, in_main_thread=True)
-        self.main_thread_tls.start_transaction(-1)
+        self.main_thread_tls.start_transaction()
 
     def setup_thread(self):
         from pypy.rpython.memory.gc.stmtls import StmGCTLS
@@ -195,8 +195,8 @@
         if gen > 0:
             debug_print("XXX not doing a global collect()")
 
-    def start_transaction(self, retry_counter):
-        self.get_tls().start_transaction(retry_counter)
+    def start_transaction(self):
+        self.get_tls().start_transaction()
 
     def commit_transaction(self):
         self.get_tls().stop_transaction()
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -117,24 +117,19 @@
                 if value is not self:
                     del StmGCTLS.nontranslated_dict[key]
         self.stm_operations.leave_transactional_mode()
-        self.start_transaction(-1)
+        self.start_transaction()
 
-    def start_transaction(self, retry_counter):
+    def start_transaction(self):
         """Start a transaction: performs any pending cleanups, and set
         up a fresh state for allocating.  Called at the start of
         each transaction, and at the start of the main thread."""
         # Note that the calls to enter() and
         # end_of_transaction_collection() are not balanced: if a
-        # transaction is aborted, the latter might never be called
-        # and we get back to here with retry_counter > 0.
+        # transaction is aborted, the latter might never be called.
         # Be ready here to clean up any state.
         self._cleanup_state()
-        rw = self.gc.root_walker
-        if retry_counter > 0:
-            rw.set_current_stack_roots_limit(self.stack_root_limit)
-        else:
-            self.stack_root_limit = rw.get_current_stack_roots_limit()
-        #
+        if self is not self.gc.main_thread_tls:
+            self.gc.root_walker.clear_current_stack_roots()
         if self.nursery_free:
             clear_size = self.nursery_free - self.nursery_start
         else:
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -135,8 +135,8 @@
 class FakeRootWalker:
     def walk_current_stack_roots(self, *args):
         pass     # no stack roots in this test file
-    def get_current_stack_roots_limit(self):
-        return "some limit"
+    def clear_current_stack_roots(self):
+        pass
 
 
 class StmGCTests:
@@ -189,7 +189,7 @@
         self.gc.stm_operations.threadnum = threadnum
         if threadnum not in self.gc.stm_operations._tls_dict:
             self.gc.setup_thread()
-            self.gc.start_transaction(0)
+            self.gc.start_transaction()
     def gcsize(self, S):
         return (llmemory.raw_malloc_usage(llmemory.sizeof(self.gc.HDR)) +
                 llmemory.raw_malloc_usage(llmemory.sizeof(S)))
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -38,8 +38,6 @@
             P = lltype.typeOf(self.current_stack[i])
             self.current_stack[i] = llmemory.cast_adr_to_ptr(roots[i], P)
         lltype.free(roots, flavor='raw')
-    def get_current_stack_roots_limit(self):
-        return "some limit"
 
 class FakeGC:
     from pypy.rpython.memory.support import AddressDict, null_address_dict
@@ -85,7 +83,7 @@
         self.gctls_main = StmGCTLS(self.gc, in_main_thread=True)
         self.gctls_thrd = StmGCTLS(self.gc, in_main_thread=False)
         self.gc.main_thread_tls = self.gctls_main
-        self.gctls_main.start_transaction(-1)
+        self.gctls_main.start_transaction()
         self.gc.root_walker.current_stack = self.current_stack
 
     def stack_add(self, p):
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -42,7 +42,7 @@
             [s_gc], annmodel.s_None)
         self.stm_start_ptr = getfn(
             self.gcdata.gc.start_transaction.im_func,
-            [s_gc, annmodel.SomeInteger()], annmodel.s_None)
+            [s_gc], annmodel.s_None)
         self.stm_commit_ptr = getfn(
             self.gcdata.gc.commit_transaction.im_func,
             [s_gc], annmodel.s_None)
@@ -84,9 +84,7 @@
         hop.genop('cast_adr_to_ptr', [v_globaladr], resultvar=op.result)
 
     def gct_stm_start_transaction(self, hop):
-        v_int = hop.spaceop.args[0]
-        hop.genop("direct_call", [self.stm_start_ptr,
-                                  self.c_const_gc, v_int])
+        hop.genop("direct_call", [self.stm_start_ptr, self.c_const_gc])
 
     def gct_stm_commit_transaction(self, hop):
         hop.genop("direct_call", [self.stm_commit_ptr, self.c_const_gc])
@@ -172,10 +170,8 @@
                            stackgcdata.root_stack_base,
                            stackgcdata.root_stack_top)
 
-    def get_current_stack_roots_limit(self):
-        return self.stackgcdata.root_stack_top
-
-    def set_current_stack_roots_limit(self, limit):
+    def clear_current_stack_roots(self):
         """Used when we start a transaction: there might be garbage left
         behind by the previous aborted transaction."""
-        self.stackgcdata.root_stack_top = limit
+        stackgcdata = self.stackgcdata
+        stackgcdata.root_stack_top = stackgcdata.root_stack_base

From noreply at buildbot.pypy.org  Tue Apr 17 14:03:24 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 14:03:24 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Save and restore the root_stack_top
 explicitly with the help of C code.
Message-ID: <20120417120324.8685282F4F@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54460:7a642543dd09
Date: 2012-04-17 13:28 +0200
http://bitbucket.org/pypy/pypy/changeset/7a642543dd09/

Log:	Save and restore the root_stack_top explicitly with the help of C
	code.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -2,7 +2,7 @@
 from pypy.rlib.objectmodel import specialize, we_are_translated
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib.debug import ll_assert
-from pypy.rpython.lltypesystem import rffi, lltype, rclass
+from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.annlowlevel import (cast_base_ptr_to_instance,
                                       cast_instance_to_base_ptr,
@@ -35,6 +35,7 @@
     if we_are_translated():
         llarg = cast_instance_to_base_ptr(arg)
         llarg = rffi.cast(rffi.VOIDP, llarg)
+        adr_of_top = llop.gc_adr_of_root_stack_top(llmemory.Address)
     else:
         # only for tests: we want (1) to test the calls to the C library,
         # but also (2) to work with multiple Python threads, so we acquire
@@ -43,9 +44,11 @@
         _global_lock.acquire()
         lltype.TLS.stm_callback_arg = arg
         llarg = lltype.nullptr(rffi.VOIDP.TO)
+        adr_of_top = llmemory.NULL
+    #
     callback = _get_stm_callback(func, argcls)
     llcallback = llhelper(StmOperations.CALLBACK_TX, callback)
-    StmOperations.perform_transaction(llcallback, llarg)
+    StmOperations.perform_transaction(llcallback, llarg, adr_of_top)
     keepalive_until_here(arg)
     if not we_are_translated():
         _global_lock.release()
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -128,8 +128,6 @@
         # transaction is aborted, the latter might never be called.
         # Be ready here to clean up any state.
         self._cleanup_state()
-        if self is not self.gc.main_thread_tls:
-            self.gc.root_walker.clear_current_stack_roots()
         if self.nursery_free:
             clear_size = self.nursery_free - self.nursery_start
         else:
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -135,8 +135,6 @@
 class FakeRootWalker:
     def walk_current_stack_roots(self, *args):
         pass     # no stack roots in this test file
-    def clear_current_stack_roots(self):
-        pass
 
 
 class StmGCTests:
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -2,6 +2,7 @@
 from pypy.rpython.memory.gctransform.framework import BaseRootWalker
 from pypy.rpython.memory.gctransform.framework import sizeofaddr
 from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython import rmodel
 from pypy.annotation import model as annmodel
 from pypy.rlib.debug import fatalerror_notb
 from pypy.rlib.nonconst import NonConstant
@@ -50,6 +51,17 @@
     def build_root_walker(self):
         return StmShadowStackRootWalker(self)
 
+    def _gc_adr_of_gcdata_attr(self, hop, attrname):
+        x = self.root_walker.stackgcdata
+        c_const_stackgcdata = rmodel.inputconst(lltype.typeOf(x), x)
+        op = hop.spaceop
+        ofs = llmemory.offsetof(c_const_stackgcdata.concretetype.TO,
+                                attrname)
+        c_ofs = rmodel.inputconst(lltype.Signed, ofs)
+        v_gcdata_adr = hop.genop('cast_ptr_to_adr', [c_const_stackgcdata],
+                                 resulttype=llmemory.Address)
+        hop.genop('adr_add', [v_gcdata_adr, c_ofs], resultvar=op.result)
+
     def gct_stm_descriptor_init(self, hop):
         hop.genop("direct_call", [self.setup_secondary_thread_ptr,
                                   self.c_const_gc])
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -658,14 +658,20 @@
   return d->end_time;
 }
 
-void* stm_perform_transaction(void*(*callback)(void*, long), void *arg)
+void* stm_perform_transaction(void*(*callback)(void*, long), void *arg,
+                              void *save_and_restore)
 {
   void *result;
   jmp_buf _jmpbuf;
   volatile long v_counter = 0;
   long counter;
+  volatile void *saved_value;
   assert(active_thread_descriptor == NULL);
+  if (save_and_restore)
+    saved_value = *(void**)save_and_restore;
   setjmp(_jmpbuf);
+  if (save_and_restore)
+    *(void**)save_and_restore = saved_value;
   begin_transaction(&_jmpbuf);
   counter = v_counter;
   v_counter = counter + 1;
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -39,7 +39,7 @@
 #endif
 
 
-void* stm_perform_transaction(void*(*)(void*, long), void*);
+void* stm_perform_transaction(void*(*)(void*, long), void*, void*);
 void stm_enter_transactional_mode(void);
 void stm_leave_transactional_mode(void);
 void stm_try_inevitable(STM_CCHARP1(why));
diff --git a/pypy/translator/stm/stmgcintf.py b/pypy/translator/stm/stmgcintf.py
--- a/pypy/translator/stm/stmgcintf.py
+++ b/pypy/translator/stm/stmgcintf.py
@@ -80,7 +80,8 @@
 
     try_inevitable = smexternal('stm_try_inevitable', [], lltype.Void)
     perform_transaction = smexternal('stm_perform_transaction',
-                                     [CALLBACK_TX, rffi.VOIDP], rffi.VOIDP)
+                                     [CALLBACK_TX, rffi.VOIDP,
+                                      llmemory.Address], rffi.VOIDP)
     thread_id        = smexternal('stm_thread_id',       [], lltype.Signed)
     abort_and_retry  = smexternal('stm_abort_and_retry', [], lltype.Void)
 

From noreply at buildbot.pypy.org  Tue Apr 17 16:18:38 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Apr 2012 16:18:38 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: add raw storage to
	test_ztranslation
Message-ID: <20120417141838.65778820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54462:c16601d836d0
Date: 2012-04-17 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/c16601d836d0/

Log:	add raw storage to test_ztranslation

diff --git a/pypy/jit/backend/x86/test/test_ztranslation.py b/pypy/jit/backend/x86/test/test_ztranslation.py
--- a/pypy/jit/backend/x86/test/test_ztranslation.py
+++ b/pypy/jit/backend/x86/test/test_ztranslation.py
@@ -11,6 +11,7 @@
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64
 from pypy.config.translationoption import DEFL_GC
 from pypy.rlib import rgc
+from pypy.rlib.rerased_raw import UntypedStorage
 
 class TestTranslationX86(CCompiledMixin):
     CPUClass = getcpuclass()
@@ -42,7 +43,7 @@
             return abs(x)
 
         jitdriver = JitDriver(greens = [],
-                              reds = ['total', 'frame', 'j'],
+                              reds = ['total', 'raw_storage', 'frame', 'j'],
                               virtualizables = ['frame'])
         def f(i, j):
             for param, _ in unroll_parameters:
@@ -53,14 +54,16 @@
             total = 0
             frame = Frame(i)
             j = float(j)
+            raw_storage = UntypedStorage("fi")
+            raw_storage.setfloat(0, 0.712)
             while frame.i > 3:
-                jitdriver.can_enter_jit(frame=frame, total=total, j=j)
-                jitdriver.jit_merge_point(frame=frame, total=total, j=j)
+                jitdriver.jit_merge_point(frame=frame, total=total, j=j,
+                                          raw_storage=raw_storage)
                 total += frame.i
                 if frame.i >= 20:
                     frame.i -= 2
                 frame.i -= 1
-                j *= -0.712
+                j *= -raw_storage.getfloat(0)
                 if j + (-j):    raise ValueError
                 k = myabs(j)
                 if k - abs(j):  raise ValueError

From noreply at buildbot.pypy.org  Tue Apr 17 16:21:16 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Apr 2012 16:21:16 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: copy this stuff to a
 descr, so caching is done by someone else
Message-ID: <20120417142116.712F2820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54463:1011b0cf4476
Date: 2012-04-17 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/1011b0cf4476/

Log:	copy this stuff to a descr, so caching is done by someone else

diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -22,8 +22,6 @@
                  gcdescr=None):
         assert type(opts) is not bool
         self.opts = opts
-        self.arraydescr_cache = {}
-
         from pypy.jit.backend.llsupport.gc import get_ll_description
         AbstractCPU.__init__(self)
         self.rtyper = rtyper
@@ -293,14 +291,12 @@
                                                  extrainfo, ffi_flags)
 
     def copy_and_change_descr_typeinfo_to_ptr(self, descr):
-        key = (descr.basesize, descr.itemsize, descr.lendescr)
-        try:
-            return self.arraydescr_cache[key]
-        except KeyError:
-            new_descr = ArrayDescr(descr.basesize, descr.itemsize,
-                                   descr.lendescr, FLAG_POINTER)
-            self.arraydescr_cache[key] = new_descr
-            return new_descr
+        if hasattr(descr, 'typeinfo_to_ptr_descr'):
+            return descr.typeinfo_to_ptr_descr
+        new_descr = ArrayDescr(descr.basesize, descr.itemsize,
+                               descr.lendescr, FLAG_POINTER)
+        descr.typeinfo_to_ptr_descr = new_descr
+        return new_descr
 
     def get_overflow_error(self):
         ovf_vtable = self.cast_adr_to_int(self._ovf_error_vtable)

From noreply at buildbot.pypy.org  Tue Apr 17 16:49:41 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 17 Apr 2012 16:49:41 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: validate more fd's
Message-ID: <20120417144941.DAE96820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54464:b972b2e27935
Date: 2012-04-17 17:49 +0300
http://bitbucket.org/pypy/pypy/changeset/b972b2e27935/

Log:	validate more fd's

diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -397,6 +397,7 @@
         os_dup = self.llexternal(underscore_on_windows+'dup', [rffi.INT], rffi.INT)
 
         def dup_llimpl(fd):
+            rposix.validate_fd(fd)
             newfd = rffi.cast(lltype.Signed, os_dup(rffi.cast(rffi.INT, fd)))
             if newfd == -1:
                 raise OSError(rposix.get_errno(), "dup failed")
@@ -411,6 +412,7 @@
                                   [rffi.INT, rffi.INT], rffi.INT)
 
         def dup2_llimpl(fd, newfd):
+            rposix.validate_fd(fd)
             error = rffi.cast(lltype.Signed, os_dup2(rffi.cast(rffi.INT, fd),
                                              rffi.cast(rffi.INT, newfd)))
             if error == -1:
@@ -891,6 +893,7 @@
         def os_read_llimpl(fd, count):
             if count < 0:
                 raise OSError(errno.EINVAL, None)
+            rposix.validate_fd(fd)
             raw_buf, gc_buf = rffi.alloc_buffer(count)
             try:
                 void_buf = rffi.cast(rffi.VOIDP, raw_buf)
@@ -1048,6 +1051,7 @@
         os_fchdir = self.llexternal('fchdir', [rffi.INT], rffi.INT)
 
         def fchdir_llimpl(fd):
+            rposix.validate_fd(fd)
             res = rffi.cast(rffi.SIGNED, os_fchdir(rffi.cast(rffi.INT, fd)))
             if res < 0:
                 raise OSError(rposix.get_errno(), "fchdir failed")
@@ -1360,6 +1364,7 @@
         os_isatty = self.llexternal(underscore_on_windows+'isatty', [rffi.INT], rffi.INT)
 
         def isatty_llimpl(fd):
+            rposix.validate_fd(fd)
             res = rffi.cast(lltype.Signed, os_isatty(rffi.cast(rffi.INT, fd)))
             return res != 0
 
@@ -1537,6 +1542,7 @@
         os_umask = self.llexternal(underscore_on_windows+'umask', [rffi.MODE_T], rffi.MODE_T)
 
         def umask_llimpl(fd):
+            rposix.validate_fd(fd)
             res = os_umask(rffi.cast(rffi.MODE_T, fd))
             return rffi.cast(lltype.Signed, res)
 

From noreply at buildbot.pypy.org  Tue Apr 17 16:54:53 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Apr 2012 16:54:53 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: fixes for cpyext
Message-ID: <20120417145453.D9CEB820B3@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54465:194eecac5a7b
Date: 2012-04-17 10:54 -0400
http://bitbucket.org/pypy/pypy/changeset/194eecac5a7b/

Log:	fixes for cpyext

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -46,7 +46,7 @@
         isinstance(w_obj, tupleobject.W_TupleObject)):
         return w_obj
     try:
-        return tupleobject.W_TupleObject(space.fixedview(w_obj))
+        return space.newtuple(space.fixedview(w_obj))
     except OperationError:
         raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
 
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -32,15 +32,12 @@
         lltype.free(ar, flavor='raw')
 
     def test_setitem(self, space, api):
-        atuple = space.newtuple([space.wrap(0), space.wrap("hello")])
+        atuple = api.PyTuple_New(2)
         assert api.PyTuple_Size(atuple) == 2
-        assert space.eq_w(space.getitem(atuple, space.wrap(0)), space.wrap(0))
-        assert space.eq_w(space.getitem(atuple, space.wrap(1)), space.wrap("hello"))
         w_obj = space.wrap(1)
         api.Py_IncRef(w_obj)
         api.PyTuple_SetItem(atuple, 1, w_obj)
         assert api.PyTuple_Size(atuple) == 2
-        assert space.eq_w(space.getitem(atuple, space.wrap(0)), space.wrap(0))
         assert space.eq_w(space.getitem(atuple, space.wrap(1)), space.wrap(1))
 
     def test_getslice(self, space, api):
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -150,7 +150,7 @@
             def __init__(self):
                 self.foobar = 32
                 super(UnicodeSubclass2, self).__init__()
-        
+
         newobj = UnicodeSubclass2()
         assert newobj.get_val() == 42
         assert newobj.foobar == 32
@@ -282,7 +282,7 @@
         class C(object):
             pass
         assert module.name_by_heaptype(C) == "C"
-        
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):
@@ -295,7 +295,7 @@
 
         py_type = rffi.cast(PyTypeObjectPtr, ref)
         assert py_type.c_tp_alloc
-        assert from_ref(space, py_type.c_tp_mro).wrappeditems is w_class.mro_w
+        assert space.eq_w(from_ref(space, py_type.c_tp_mro), space.newtuple(w_class.mro_w))
 
         api.Py_DecRef(ref)
 
@@ -320,7 +320,7 @@
         w_obj = api._PyType_Lookup(w_type, space.wrap("__invalid"))
         assert w_obj is None
         assert api.PyErr_Occurred() is None
-    
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def test_some_slots(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -121,7 +121,7 @@
                                                 utf_8, None)
         assert space.eq_w(encoded, encoded_obj)
         self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString,
-               space.newtuple([1, 2, 3]), None, None)
+               space.newtuple([space.wrap(1), space.wrap(2), space.wrap(3)]), None, None)
         self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString,
                space.wrap(''), None, None)
         ascii = rffi.str2charp('ascii')
@@ -176,13 +176,13 @@
         w_res = api.PyUnicode_AsUTF8String(w_u)
         assert space.type(w_res) is space.w_str
         assert space.unwrap(w_res) == 'sp\xc3\xa4m'
-    
+
     def test_decode_utf8(self, space, api):
         u = rffi.str2charp(u'sp&#65533;m'.encode("utf-8"))
         w_u = api.PyUnicode_DecodeUTF8(u, 5, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == u'sp&#65533;m'
-        
+
         w_u = api.PyUnicode_DecodeUTF8(u, 2, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == 'sp'
@@ -314,7 +314,7 @@
         ustr = "abcdef"
         w_ustr = space.wrap(ustr.decode("ascii"))
         result = api.PyUnicode_AsASCIIString(w_ustr)
-        
+
         assert space.eq_w(space.wrap(ustr), result)
 
         w_ustr = space.wrap(u"abcd\xe9f")

From noreply at buildbot.pypy.org  Tue Apr 17 17:07:09 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Apr 2012 17:07:09 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: 2 more fixes
Message-ID: <20120417150709.B7BB1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54466:cf93e89d4822
Date: 2012-04-17 11:06 -0400
http://bitbucket.org/pypy/pypy/changeset/cf93e89d4822/

Log:	2 more fixes

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -59,7 +59,7 @@
         w_res = w_obj.getitem(index)
     else:
         assert isinstance(w_obj, tupleobject.W_TupleObject)
-        w_res = w_obj.wrappeditems[index]
+        w_res = w_obj.getitem(index)
     return borrow_from(w_obj, w_res)
 
 @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
@@ -72,7 +72,7 @@
     if isinstance(w_obj, listobject.W_ListObject):
         return w_obj.length()
     assert isinstance(w_obj, tupleobject.W_TupleObject)
-    return len(w_obj.wrappeditems)
+    return w_obj.length()
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
 def PySequence_GetSlice(space, w_obj, start, end):

From noreply at buildbot.pypy.org  Tue Apr 17 17:12:30 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 17 Apr 2012 17:12:30 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: one mroe fix
Message-ID: <20120417151230.6719F820B3@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54467:da02bab14d8c
Date: 2012-04-17 11:12 -0400
http://bitbucket.org/pypy/pypy/changeset/da02bab14d8c/

Log:	one mroe fix

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -59,7 +59,7 @@
         w_res = w_obj.getitem(index)
     else:
         assert isinstance(w_obj, tupleobject.W_TupleObject)
-        w_res = w_obj.getitem(index)
+        w_res = w_obj.getitem(space, index)
     return borrow_from(w_obj, w_res)
 
 @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:24 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify
Message-ID: <20120417155824.BD681820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54468:e2298258f8a6
Date: 2012-04-17 12:19 +0200
http://bitbucket.org/pypy/pypy/changeset/e2298258f8a6/

Log:	py3k-ify

diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -411,7 +411,7 @@
             buf[3] = b'L'
         except TypeError:
             skip("memoryview(array) returns a read-only buffer on CPython")
-        assert a.tostring() == 'helLo'
+        assert a.tostring() == b'helLo'
 
     def test_buffer_keepalive(self):
         buf = memoryview(self.array('b', b'text'))
@@ -744,12 +744,12 @@
                 self.append(8)
 
             def fromunicode(self, lst):
-                self.append(u'9')
+                self.append('9')
 
             def extend(self, lst):
                 self.append(10)
 
-        assert repr(mya('u', u'hi')) == "array('u', 'hi')"
+        assert repr(mya('u', 'hi')) == "array('u', 'hi')"
         assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])"
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 
@@ -762,7 +762,7 @@
         assert repr(a) == "array('b', [8])"
 
         a = mya('u')
-        a.fromunicode(u'hi')
+        a.fromunicode('hi')
         assert repr(a) == "array('u', '9')"
 
         a = mya('i')
@@ -784,7 +784,7 @@
         assert mya('u', 'hi').tobytes() == 'str'
         assert mya('u', 'hi').tounicode() == 'unicode'
 
-        assert repr(mya('u', u'hi')) == "array('u', 'hi')"
+        assert repr(mya('u', 'hi')) == "array('u', 'hi')"
         assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])"
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:26 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify
Message-ID: <20120417155826.0C63A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54469:a6e00f858173
Date: 2012-04-17 14:34 +0200
http://bitbucket.org/pypy/pypy/changeset/a6e00f858173/

Log:	py3k-ify

diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -804,7 +804,7 @@
         it = d.iteritems()
         d[7] = 8
         # 'd' is now length 4
-        raises(RuntimeError, it.next)
+        raises(RuntimeError, it.__next__)
 
     def test_iter_dict_strategy_only_change_1(self):
         d = {1: 2, 3: 4, 5: 6}
@@ -812,6 +812,8 @@
         class Foo(object):
             def __eq__(self, other):
                 return False
+            def __hash__(self):
+                return 0
         assert d.get(Foo()) is None    # this changes the strategy of 'd'
         lst = list(it)  # but iterating still works
         assert sorted(lst) == [(1, 2), (3, 4), (5, 6)]
@@ -821,10 +823,15 @@
         it = d.iteritems()
         d['foo'] = 'bar'
         del d[1]
+        # on default the strategy changes and thus we get the RuntimeError
+        # (commented below). On py3k, we Int and String strategies don't work
+        # yet, and thus we get the "correct" behavior
+        items = list(it)
+        assert set(items) == set([(3, 4), (5, 6), ('foo', 'bar')])
         # 'd' is still length 3, but its strategy changed.  we are
         # getting a RuntimeError because iterating over the old storage
         # gives us (1, 2), but 1 is not in the dict any longer.
-        raises(RuntimeError, list, it)
+        #raises(RuntimeError, list, it)
 
 
 class FakeString(str):

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:27 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: add a py3k_skip helper,
 for tests which can be skipped now on py3k,
 but should be fixed sooner or later
Message-ID: <20120417155827.6B0C1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54470:485bd020e6f7
Date: 2012-04-17 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/485bd020e6f7/

Log:	add a py3k_skip helper, for tests which can be skipped now on py3k,
	but should be fixed sooner or later

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -29,6 +29,9 @@
 def pytest_configure(config):
     global option
     option = config.option
+    def py3k_skip(message):
+        py.test.skip('[py3k] %s' % message)
+    py.test.py3k_skip = py3k_skip
 
 def _set_platform(opt, opt_str, value, parser):
     from pypy.config.translationoption import PLATFORMS
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -158,6 +158,8 @@
         assert self.space.listview_int(w_d) == [1, 2]
 
     def test_keys_on_string_int_dict(self):
+        py.test.py3k_skip("StringDictStrategy not supported yet")
+        
         w = self.space.wrap
         w_d = self.space.newdict()
         w_d.initialize_content([(w(1), w("a")), (w(2), w("b"))])

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:28 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:28 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: add an applevel py3k_skip,
 and py3k_skip some multidict tests
Message-ID: <20120417155828.B0605820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54471:8868ac4174b0
Date: 2012-04-17 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/8868ac4174b0/

Log:	add an applevel py3k_skip, and py3k_skip some multidict tests

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -116,6 +116,8 @@
                   space.wrap(appsupport.app_raises))
     space.setitem(space.builtin.w_dict, space.wrap('skip'),
                   space.wrap(appsupport.app_skip))
+    space.setitem(space.builtin.w_dict, space.wrap('py3k_skip'),
+                  space.wrap(appsupport.app_py3k_skip))
     space.raises_w = appsupport.raises_w.__get__(space)
     space.eq_w = appsupport.eq_w.__get__(space)
     return space
diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py
--- a/pypy/objspace/std/test/test_dictmultiobject.py
+++ b/pypy/objspace/std/test/test_dictmultiobject.py
@@ -143,6 +143,7 @@
         assert space.eq_w(w_d.getitem_str("b"), space.w_None)
 
     def test_listview_str_dict(self):
+        py.test.py3k_skip("StringDictStrategy not supported yet")
         w = self.space.wrap
 
         w_d = self.space.newdict()
@@ -151,6 +152,7 @@
         assert self.space.listview_str(w_d) == ["a", "b"]
 
     def test_listview_int_dict(self):
+        py.test.py3k_skip("IntDictStrategy not supported yet")
         w = self.space.wrap
         w_d = self.space.newdict()
         w_d.initialize_content([(w(1), w("a")), (w(2), w("b"))])
@@ -781,6 +783,7 @@
         return r[r.find("(") + 1: r.find(")")]
 
     def test_empty_to_string(self):
+        py3k_skip("StringDictStrategy not supported yet")
         d = {}
         assert "EmptyDictStrategy" in self.get_strategy(d)
         d["a"] = 1
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -261,6 +261,13 @@
 
 app_skip = gateway.interp2app_temp(pypyskip)
 
+def py3k_pypyskip(space, w_message): 
+    """skip a test at app-level. """ 
+    msg = space.unwrap(w_message) 
+    py.test.skip('[py3k] %s' % msg)
+
+app_py3k_skip = gateway.interp2app_temp(py3k_pypyskip)
+
 def raises_w(space, w_ExpectedException, *args, **kwds):
     try:
         excinfo = py.test.raises(OperationError, *args, **kwds)

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:29 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:29 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: skip one more test
Message-ID: <20120417155829.F06DE820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54472:bf0e6b700818
Date: 2012-04-17 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/bf0e6b700818/

Log:	skip one more test

diff --git a/pypy/objspace/std/test/test_stringobject.py b/pypy/objspace/std/test/test_stringobject.py
--- a/pypy/objspace/std/test/test_stringobject.py
+++ b/pypy/objspace/std/test/test_stringobject.py
@@ -1,3 +1,4 @@
+import py
 from pypy.objspace.std import stringobject
 from pypy.objspace.std.stringobject import W_StringObject
 from pypy.conftest import gettestobjspace
@@ -88,6 +89,7 @@
         assert self.space.eq_w(space.getitem(w_str, w_slice), wb('el'))
 
     def test_listview_str(self):
+        py.test.py3k_skip("listview_str not supported yet")
         w_str = self.space.wrap('abcd')
         assert self.space.listview_str(w_str) == list("abcd")
 

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:31 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:31 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: skip micronumpy tests,
 we are not interested in them right now
Message-ID: <20120417155831.423D3820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54473:8f32ba40036f
Date: 2012-04-17 16:49 +0200
http://bitbucket.org/pypy/pypy/changeset/8f32ba40036f/

Log:	skip micronumpy tests, we are not interested in them right now

diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,3 +1,4 @@
+import py
 from pypy.conftest import gettestobjspace
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, Scalar
@@ -11,6 +12,7 @@
 
 class BaseNumpyAppTest(object):
     def setup_class(cls):
+        py.test.py3k_skip("micronumpy not supported on py3k")
         if option.runappdirect:
             if '__pypy__' not in sys.builtin_module_names:
                 import numpy

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:32 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k_skip the failing zzpickle tests. Some
 are only needed for stackless, I'll fix them once the core py3k is working
Message-ID: <20120417155832.CF7FA820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54474:0961d07b95b5
Date: 2012-04-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/0961d07b95b5/

Log:	py3k_skip the failing zzpickle tests. Some are only needed for
	stackless, I'll fix them once the core py3k is working

diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -124,6 +124,7 @@
         assert map is result
     
     def test_pickle_non_top_reachable_func(self):
+        py3k_skip("not supported yet")
         def func():
             return 42
         global a
@@ -155,6 +156,7 @@
         assert not (cell != result)
 
     def test_pickle_frame(self):
+        py3k_skip("not supported yet")
         #import sys
         # avoid creating a closure for now
         def f():
@@ -186,6 +188,7 @@
         assert f1.f_trace is f2.f_trace
 
     def test_pickle_frame_with_exc(self):
+        py3k_skip("not supported yet")
         #import sys
         # avoid creating a closure for now
         self = None
@@ -207,6 +210,7 @@
         assert read_exc_type(f2) is ValueError
 
     def test_pickle_frame_clos(self):
+        py3k_skip("not supported yet")
         # similar to above, therefore skipping the asserts.
         # we just want to see that the closure works
         import sys # this is the difference!
@@ -224,6 +228,7 @@
         f2     = pickle.loads(pckl)
 
     def test_pickle_traceback(self):
+        py3k_skip("not supported yet")
         def f():
             try:
                 raise Exception()
@@ -252,6 +257,7 @@
         assert mod is result
     
     def test_pickle_moduledict(self):
+        py3k_skip("not supported yet")
         import pickle
         moddict  = pickle.__dict__
         pckl     = pickle.dumps(moddict)
@@ -282,6 +288,7 @@
         assert a == result
     
     def test_pickle_method(self):
+        py3k_skip("not supported yet")
         class myclass(object):
             def f(self):
                 return 42
@@ -303,6 +310,7 @@
             del sys.modules['mod']
     
     def test_pickle_staticmethod(self):
+        py3k_skip("not supported yet")
         class myclass(object):
             def f():
                 return 42
@@ -314,6 +322,7 @@
         assert method() == result()
     
     def test_pickle_classmethod(self):
+        py3k_skip("not supported yet")
         class myclass(object):
             def f(cls):
                 return cls
@@ -394,6 +403,7 @@
         assert list(e) == list(result)
 
     def test_pickle_xrangeiter(self):
+        py3k_skip("not supported yet")
         import pickle
         riter  = iter(xrange(5))
         next(riter)
@@ -404,6 +414,7 @@
         assert list(result) == [2,3,4]
 
     def test_pickle_generator(self):
+        py3k_skip("not supported yet")
         import types
         mod = types.ModuleType('mod')
         import sys
@@ -427,6 +438,7 @@
             del sys.modules['mod']
 
     def test_pickle_generator_blk(self):
+        py3k_skip("not supported yet")
         # same as above but with the generator inside a block
         import types
         mod = types.ModuleType('mod')

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:34 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:34 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: don't try to support _continuation before
 having the core py3k working
Message-ID: <20120417155834.237C7820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54475:cc66208f7aaf
Date: 2012-04-17 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/cc66208f7aaf/

Log:	don't try to support _continuation before having the core py3k
	working

diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
--- a/pypy/module/_continuation/test/test_zpickle.py
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -3,6 +3,7 @@
 
 class AppTestCopy:
     def setup_class(cls):
+        py3k_skip("_continuation not supported yet")
         cls.space = gettestobjspace(usemodules=('_continuation',),
                                     CALL_METHOD=True)
         cls.space.config.translation.continuation = True

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:35 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:35 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: we no longer have W_IntObject, but longs
Message-ID: <20120417155835.655DE820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54476:fd87f3f71c7a
Date: 2012-04-17 17:14 +0200
http://bitbucket.org/pypy/pypy/changeset/fd87f3f71c7a/

Log:	we no longer have W_IntObject, but longs

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -9,6 +9,7 @@
 from pypy.rlib.debug import check_annotation
 from pypy.objspace.std import stringobject
 from pypy.objspace.std.intobject import W_IntObject
+from pypy.objspace.std.longobject import W_LongObject
 from pypy.objspace.std.listobject import get_positive_index
 from pypy.objspace.std.listtype import get_list_index
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
@@ -349,7 +350,7 @@
     w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_isspace__String(space, w_str)
 
-def bytearray_insert__Bytearray_Int_ANY(space, w_bytearray, w_idx, w_other):
+def bytearray_insert__Bytearray_Long_ANY(space, w_bytearray, w_idx, w_other):
     where = space.int_w(w_idx)
     length = len(w_bytearray.data)
     index = get_positive_index(where, length)

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:36 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:36 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: we no longer have W_IntObject, but longs
Message-ID: <20120417155836.A5D9B820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54477:57c1bff7b005
Date: 2012-04-17 17:35 +0200
http://bitbucket.org/pypy/pypy/changeset/57c1bff7b005/

Log:	we no longer have W_IntObject, but longs

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -76,7 +76,7 @@
 def _getitem_slice_multistep(data, start, step, slicelength):
     return [data[start + i*step] for i in range(slicelength)]
 
-def contains__Bytearray_Int(space, w_bytearray, w_char):
+def contains__Bytearray_Long(space, w_bytearray, w_char):
     char = space.int_w(w_char)
     if not 0 <= char < 256:
         raise OperationError(space.w_ValueError,
@@ -229,7 +229,7 @@
 def _to_bytes(space, w_bytearray):
     return space.wrapbytes(''.join(w_bytearray.data))
 
-def str_count__Bytearray_Int_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
+def str_count__Bytearray_Long_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
     char = w_char.intval
     bytearray = w_bytearray.data
     length = len(bytearray)
@@ -358,7 +358,7 @@
     w_bytearray.data.insert(index, val)
     return space.w_None
 
-def bytearray_pop__Bytearray_Int(space, w_bytearray, w_idx):
+def bytearray_pop__Bytearray_Long(space, w_bytearray, w_idx):
     index = space.int_w(w_idx)
     try:
         result = w_bytearray.data.pop(index)

From noreply at buildbot.pypy.org  Tue Apr 17 17:58:37 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Tue, 17 Apr 2012 17:58:37 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge heads
Message-ID: <20120417155837.EE940820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54478:ddf49f45ca25
Date: 2012-04-17 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/ddf49f45ca25/

Log:	merge heads

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -114,13 +114,10 @@
     #       BINARY_SUBSCR
     # or:
     #       CALL_FUNCTION/CALL_METHOD
+    #       LOAD_CONST any integer or None
     #       LOAD_CONST <=2
-    #       SLICE_2
-    # or:
-    #       CALL_FUNCTION/CALL_METHOD
-    #       LOAD_CONST any integer
-    #       LOAD_CONST <=2
-    #       SLICE_3
+    #       BUILD_SLICE 2
+    #       BINARY_SUBSCR
     need_all_three_args = True
     co = frame.getcode().co_code
     p = frame.last_instr
@@ -130,16 +127,16 @@
             lo = ord(co[p+4])
             hi = ord(co[p+5])
             w_constant = frame.getconstant_w((hi * 256) | lo)
-            if space.isinstance_w(w_constant, space.w_int):
-                constant = space.int_w(w_constant)
-                if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+            if ord(co[p+6]) == stdlib_opcode.BINARY_SUBSCR:
+                if space.isinstance_w(w_constant, space.w_int):
+                    constant = space.int_w(w_constant)
                     if -3 <= constant <= 1 and constant != -1:
                         need_all_three_args = False
-                elif ord(co[p+6]) == stdlib_opcode.SLICE+2:
-                    if constant <= 2:
-                        need_all_three_args = False
-                elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
-                      ord(co[p+9]) == stdlib_opcode.SLICE+3):
+            elif (ord(co[p+6]) == stdlib_opcode.LOAD_CONST and
+                  ord(co[p+9]) == stdlib_opcode.BUILD_SLICE and
+                  ord(co[p+12]) == stdlib_opcode.BINARY_SUBSCR):
+                if (space.is_w(w_constant, space.w_None) or
+                    space.isinstance_w(w_constant, space.w_int)):
                     lo = ord(co[p+7])
                     hi = ord(co[p+8])
                     w_constant = frame.getconstant_w((hi * 256) | lo)

From noreply at buildbot.pypy.org  Tue Apr 17 18:01:49 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Apr 2012 18:01:49 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: add tests for C
	backend
Message-ID: <20120417160149.5D0E6820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54479:1046cc37c7ce
Date: 2012-04-17 18:00 +0200
http://bitbucket.org/pypy/pypy/changeset/1046cc37c7ce/

Log:	add tests for C backend

diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -57,7 +57,7 @@
         storage = rerased_raw.UntypedStorage("ooi")
         assert storage.getshape() == "ooi"
 
-class TestUntypedStorageLLType(LLRtypeMixin, BaseRtypingTest):
+class BaseTestUntypedStorage(BaseRtypingTest):
     def test_int(self):
         def f(x):
             storage = rerased_raw.UntypedStorage("i")
@@ -164,7 +164,7 @@
             return storage.getshape()
 
         llres = self.interpret(f, [])
-        assert hlstr(llres) == "ooi"
+        assert self.ll_to_string(llres) == "ooi"
 
     def test_const(self):
         class A(object):
@@ -183,3 +183,6 @@
 
         res = self.interpret(f, [0])
         assert res == 21
+
+class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
+    pass
diff --git a/pypy/translator/c/test/test_genc.py b/pypy/translator/c/test/test_genc.py
--- a/pypy/translator/c/test/test_genc.py
+++ b/pypy/translator/c/test/test_genc.py
@@ -534,3 +534,4 @@
     else:
         assert 0, "the call was not found in the C source"
     assert 'PYPY_INHIBIT_TAIL_CALL();' in lines[i+1]
+
diff --git a/pypy/translator/c/test/test_untyped_storage.py b/pypy/translator/c/test/test_untyped_storage.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/c/test/test_untyped_storage.py
@@ -0,0 +1,14 @@
+
+from pypy.rlib.test.test_rerased_raw import BaseTestUntypedStorage
+from pypy.translator.c.test.test_genc import compile
+
+class CCompiledMixin(object):
+    def interpret(self, f, args):
+        fn = compile(f, [type(arg) for arg in args], gcpolicy='boehm')
+        return fn(*args)
+
+    def ll_to_string(self, x):
+        return x
+
+class TestUntypedStorage(CCompiledMixin, BaseTestUntypedStorage):
+    pass

From noreply at buildbot.pypy.org  Tue Apr 17 18:01:50 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Apr 2012 18:01:50 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: merge
Message-ID: <20120417160150.A5151820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54480:91315032494a
Date: 2012-04-17 18:01 +0200
http://bitbucket.org/pypy/pypy/changeset/91315032494a/

Log:	merge

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -46,7 +46,7 @@
         isinstance(w_obj, tupleobject.W_TupleObject)):
         return w_obj
     try:
-        return tupleobject.W_TupleObject(space.fixedview(w_obj))
+        return space.newtuple(space.fixedview(w_obj))
     except OperationError:
         raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m)))
 
@@ -59,7 +59,7 @@
         w_res = w_obj.getitem(index)
     else:
         assert isinstance(w_obj, tupleobject.W_TupleObject)
-        w_res = w_obj.wrappeditems[index]
+        w_res = w_obj.getitem(space, index)
     return borrow_from(w_obj, w_res)
 
 @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL)
@@ -72,7 +72,7 @@
     if isinstance(w_obj, listobject.W_ListObject):
         return w_obj.length()
     assert isinstance(w_obj, tupleobject.W_TupleObject)
-    return len(w_obj.wrappeditems)
+    return w_obj.length()
 
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
 def PySequence_GetSlice(space, w_obj, start, end):
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -32,15 +32,12 @@
         lltype.free(ar, flavor='raw')
 
     def test_setitem(self, space, api):
-        atuple = space.newtuple([space.wrap(0), space.wrap("hello")])
+        atuple = api.PyTuple_New(2)
         assert api.PyTuple_Size(atuple) == 2
-        assert space.eq_w(space.getitem(atuple, space.wrap(0)), space.wrap(0))
-        assert space.eq_w(space.getitem(atuple, space.wrap(1)), space.wrap("hello"))
         w_obj = space.wrap(1)
         api.Py_IncRef(w_obj)
         api.PyTuple_SetItem(atuple, 1, w_obj)
         assert api.PyTuple_Size(atuple) == 2
-        assert space.eq_w(space.getitem(atuple, space.wrap(0)), space.wrap(0))
         assert space.eq_w(space.getitem(atuple, space.wrap(1)), space.wrap(1))
 
     def test_getslice(self, space, api):
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -150,7 +150,7 @@
             def __init__(self):
                 self.foobar = 32
                 super(UnicodeSubclass2, self).__init__()
-        
+
         newobj = UnicodeSubclass2()
         assert newobj.get_val() == 42
         assert newobj.foobar == 32
@@ -282,7 +282,7 @@
         class C(object):
             pass
         assert module.name_by_heaptype(C) == "C"
-        
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):
@@ -295,7 +295,7 @@
 
         py_type = rffi.cast(PyTypeObjectPtr, ref)
         assert py_type.c_tp_alloc
-        assert from_ref(space, py_type.c_tp_mro).wrappeditems is w_class.mro_w
+        assert space.eq_w(from_ref(space, py_type.c_tp_mro), space.newtuple(w_class.mro_w))
 
         api.Py_DecRef(ref)
 
@@ -320,7 +320,7 @@
         w_obj = api._PyType_Lookup(w_type, space.wrap("__invalid"))
         assert w_obj is None
         assert api.PyErr_Occurred() is None
-    
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def test_some_slots(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -121,7 +121,7 @@
                                                 utf_8, None)
         assert space.eq_w(encoded, encoded_obj)
         self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString,
-               space.newtuple([1, 2, 3]), None, None)
+               space.newtuple([space.wrap(1), space.wrap(2), space.wrap(3)]), None, None)
         self.raises(space, api, TypeError, api.PyUnicode_AsEncodedString,
                space.wrap(''), None, None)
         ascii = rffi.str2charp('ascii')
@@ -176,13 +176,13 @@
         w_res = api.PyUnicode_AsUTF8String(w_u)
         assert space.type(w_res) is space.w_str
         assert space.unwrap(w_res) == 'sp\xc3\xa4m'
-    
+
     def test_decode_utf8(self, space, api):
         u = rffi.str2charp(u'sp&#65533;m'.encode("utf-8"))
         w_u = api.PyUnicode_DecodeUTF8(u, 5, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == u'sp&#65533;m'
-        
+
         w_u = api.PyUnicode_DecodeUTF8(u, 2, None)
         assert space.type(w_u) is space.w_unicode
         assert space.unwrap(w_u) == 'sp'
@@ -314,7 +314,7 @@
         ustr = "abcdef"
         w_ustr = space.wrap(ustr.decode("ascii"))
         result = api.PyUnicode_AsASCIIString(w_ustr)
-        
+
         assert space.eq_w(space.wrap(ustr), result)
 
         w_ustr = space.wrap(u"abcd\xe9f")

From noreply at buildbot.pypy.org  Tue Apr 17 18:51:56 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Apr 2012 18:51:56 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: finish the blog post
Message-ID: <20120417165156.B1426820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r4194:f1cb993277b4
Date: 2012-04-17 18:50 +0200
http://bitbucket.org/pypy/extradoc/changeset/f1cb993277b4/

Log:	finish the blog post

diff --git a/blog/draft/numpy-status-update-3.rst b/blog/draft/numpy-status-update-3.rst
--- a/blog/draft/numpy-status-update-3.rst
+++ b/blog/draft/numpy-status-update-3.rst
@@ -35,7 +35,22 @@
   float single-dimension numpy arrays. However, results are encouraging,
   given that our assembler generator is far from ideal:
 
-  +
+  +---------------+----------+----------+------+----------------+------------+
+  |               | Numpy    | PyPy SSE | PyPy | GCC non-looped | GCC looped |
+  +---------------+----------+----------+------+----------------+------------+
+  | ``a+b``       | 0.6s     | 0.3s     | 0.4s | 0.3s           | 0.25s      |
+  +---------------+----------+----------+------+----------------+------------+
+  | ``a+b+c``     | 1.9s     | 0.35s    | 0.5s | 0.7s           | 0.32s      |
+  +---------------+----------+----------+------+----------------+------------+
+  | ``a+b+c+d+e`` | 3.2s     | 0.36s    | 0.8s | 1.7s           | 0.51s      |
+  +---------------+----------+----------+------+----------------+------------+
+
+  The `benchmark repo`_ is available. GCC was run with ``-O3``, no further
+  options specified. PyPy was run with default options, the SSE branch is under
+  ``backend-vector-ops``, but **it's not working completely** yet.
+
+  One might argue that C and Python is not the same code - indeed it is not.
+  It just shows some possible approach to writing numeric code.
 
 Next step would be to just continue implementing missing features such as
 
@@ -49,3 +64,4 @@
 
 .. _`pycon`: http://us.pycon.org
 .. _`numpypy status page`: http://buildbot.pypy.org/numpy-status/latest.html
+.. _`benchmark repo`: https://bitbucket.org/fijal/hack2/src/fa3119d8ade6/bench/numeric

From noreply at buildbot.pypy.org  Tue Apr 17 18:51:57 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 17 Apr 2012 18:51:57 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20120417165157.E5690820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r4195:e6c1d174b99a
Date: 2012-04-17 18:51 +0200
http://bitbucket.org/pypy/extradoc/changeset/e6c1d174b99a/

Log:	merge

diff --git a/planning/stm.txt b/planning/stm.txt
--- a/planning/stm.txt
+++ b/planning/stm.txt
@@ -8,8 +8,25 @@
 |
 
 
-Overview
---------
+Python Interface
+----------------
+
+Planned interface refactorings:
+
+* inspired by "concurrent.futures" from Python 3.2: have
+  transaction.add() return a Future instance with a method result().
+  If called inside another transaction, it will suspend it until the
+  result is available, i.e. until the first transaction commits.  Can
+  be used even if the result is not used, just to ensure some ordering.
+  XXX but can that be emulated in transaction.py???
+
+* (later, maybe) allow nested transactions: either by calling
+  transaction.run() inside transactions too, or with actual objects
+  that store collections of transactions.
+
+
+Overview of the GC
+------------------
 
 A saner approach (and likely better results that now): integrate with
 the GC.  Here is the basic plan.

From noreply at buildbot.pypy.org  Tue Apr 17 19:23:57 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 19:23:57 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Debugging.
Message-ID: <20120417172357.80AB0820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54481:4643020b9d08
Date: 2012-04-17 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/4643020b9d08/

Log:	Debugging.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -304,6 +304,8 @@
         it was not done so far.
         """
         obj = root.address[0]
+        size = self.gc.get_size(obj)
+        # ^^^ moved here in order to crash early if 'obj' is invalid
         hdr = self.gc.header(obj)
         #
         # If 'obj' is not in the nursery, we set GCFLAG_VISITED
@@ -327,7 +329,6 @@
             #
             # Case of GCFLAG_HAS_SHADOW.  See comments below.
             size_gc_header = self.gc.gcheaderbuilder.size_gc_header
-            size = self.gc.get_size(obj)
             totalsize = size_gc_header + size
             hdr.tid &= ~GCFLAG_HAS_SHADOW
             newobj = hdr.version
@@ -345,7 +346,6 @@
             # First visit to 'obj': we must move this YOUNG obj out of the
             # nursery.
             size_gc_header = self.gc.gcheaderbuilder.size_gc_header
-            size = self.gc.get_size(obj)
             totalsize = size_gc_header + size
             #
             # Common case: allocate a new nonmovable location for it.

From noreply at buildbot.pypy.org  Tue Apr 17 19:23:58 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 19:23:58 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: A failing test.
Message-ID: <20120417172358.C27EC820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54482:1bb21a8ad460
Date: 2012-04-17 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/1bb21a8ad460/

Log:	A failing test.

diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -370,6 +370,16 @@
         sr2.s1 = t
         self.gc.commit_transaction()
 
+    def test_commit_with_ref_to_local_copy(self):
+        tr, tr_adr = self.malloc(SR)
+        self.select_thread(1)
+        sr_adr = self.gc.stm_writebarrier(tr_adr)
+        assert sr_adr != tr_adr
+        sr = llmemory.cast_adr_to_ptr(sr_adr, lltype.Ptr(SR))
+        sr.sr2 = sr
+        self.gc.commit_transaction()
+        assert sr.sr2 == tr
+
     def test_commit_transaction_no_references(self):
         py.test.skip("rewrite me")
         s, s_adr = self.malloc(S)

From noreply at buildbot.pypy.org  Tue Apr 17 19:24:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 19:24:00 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fixes
Message-ID: <20120417172400.09E66820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54483:91ef9bbd969f
Date: 2012-04-17 19:23 +0200
http://bitbucket.org/pypy/pypy/changeset/91ef9bbd969f/

Log:	Fixes

diff --git a/pypy/translator/c/src/allocator.h b/pypy/translator/c/src/allocator.h
--- a/pypy/translator/c/src/allocator.h
+++ b/pypy/translator/c/src/allocator.h
@@ -1,7 +1,12 @@
-#if defined(RPY_STM)
+#if defined(RPY_STM) && defined(RPY_STM_ASSERT)
+#  define TRIVIAL_MALLOC_DEBUG
+#endif
 
 
-/* XXX no special malloc function, use the thread-safe system-provided one */
+#if defined(NO_OBMALLOC) || (defined(RPY_STM)&&!defined(TRIVIAL_MALLOC_DEBUG))
+
+
+/* no special malloc function, use the thread-safe system-provided one */
 #define PyObject_Malloc malloc
 #define PyObject_Realloc realloc
 #define PyObject_Free free
@@ -26,11 +31,6 @@
 #elif defined(LINUXMEMCHK)
 #  include "linuxmemchk.c"
 
-#elif defined(NO_OBMALLOC)
-  void *PyObject_Malloc(size_t n) { return malloc(n); }
-  void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); }
-  void PyObject_Free(void *p) { free(p); }
-
 #else
 #  ifndef WITH_PYMALLOC
 #    define WITH_PYMALLOC
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -665,7 +665,7 @@
   jmp_buf _jmpbuf;
   volatile long v_counter = 0;
   long counter;
-  volatile void *saved_value;
+  void *volatile saved_value;
   assert(active_thread_descriptor == NULL);
   if (save_and_restore)
     saved_value = *(void**)save_and_restore;

From noreply at buildbot.pypy.org  Tue Apr 17 19:29:27 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 19:29:27 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix the test of 1bb21a8ad460.
Message-ID: <20120417172927.644FA820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54484:ecb25dcd128e
Date: 2012-04-17 19:29 +0200
http://bitbucket.org/pypy/pypy/changeset/ecb25dcd128e/

Log:	Fix the test of 1bb21a8ad460.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -170,6 +170,7 @@
         """
         #
         debug_start("gc-local")
+        self.ending_transaction = end_of_transaction
         #
         # Move away the previous sharedarea_tls and start a new one.
         from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
@@ -312,11 +313,19 @@
         if not self.is_in_nursery(obj):
             # we ignore both GLOBAL objects and objects which have already
             # been VISITED
-            if hdr.tid & (GCFLAG_GLOBAL|GCFLAG_VISITED) == 0:
+            if hdr.tid & (GCFLAG_GLOBAL | GCFLAG_VISITED) == 0:
                 ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0,
                           "local GCFLAG_WAS_COPIED without GCFLAG_VISITED")
                 hdr.tid |= GCFLAG_VISITED
                 self.pending.append(obj)
+            else:
+                # xxx optimize this case: at the end of transactions,
+                # replace references to the local copy with references
+                # to the global copy
+                if (self.ending_transaction > 0 and
+                    hdr.tid & GCFLAG_GLOBAL == 0 and
+                    hdr.tid & GCFLAG_WAS_COPIED != 0):
+                    root.address[0] = hdr.version
             return
         #
         # If 'obj' was already forwarded, change it to its forwarding address.

From noreply at buildbot.pypy.org  Tue Apr 17 19:57:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 17 Apr 2012 19:57:41 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Optimize.
Message-ID: <20120417175741.28BA4820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54485:6c3b8426808f
Date: 2012-04-17 19:36 +0200
http://bitbucket.org/pypy/pypy/changeset/6c3b8426808f/

Log:	Optimize.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -170,7 +170,11 @@
         """
         #
         debug_start("gc-local")
-        self.ending_transaction = end_of_transaction
+        #
+        if end_of_transaction:
+            self.detect_flag_combination = GCFLAG_WAS_COPIED | GCFLAG_VISITED
+        else:
+            self.detect_flag_combination = -1
         #
         # Move away the previous sharedarea_tls and start a new one.
         from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
@@ -313,19 +317,19 @@
         if not self.is_in_nursery(obj):
             # we ignore both GLOBAL objects and objects which have already
             # been VISITED
-            if hdr.tid & (GCFLAG_GLOBAL | GCFLAG_VISITED) == 0:
-                ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0,
-                          "local GCFLAG_WAS_COPIED without GCFLAG_VISITED")
+            flag_combination = hdr.tid & (GCFLAG_GLOBAL |
+                                          GCFLAG_WAS_COPIED |
+                                          GCFLAG_VISITED)
+            if flag_combination == 0:
                 hdr.tid |= GCFLAG_VISITED
                 self.pending.append(obj)
-            else:
-                # xxx optimize this case: at the end of transactions,
-                # replace references to the local copy with references
+            elif flag_combination == self.detect_flag_combination:
+                # At a normal time, self.detect_flag_combination is -1
+                # and this case is never seen.  At end of transactions,
+                # detect_flag_combination is GCFLAG_WAS_COPIED|GCFLAG_VISITED.
+                # Replace references to the local copy with references
                 # to the global copy
-                if (self.ending_transaction > 0 and
-                    hdr.tid & GCFLAG_GLOBAL == 0 and
-                    hdr.tid & GCFLAG_WAS_COPIED != 0):
-                    root.address[0] = hdr.version
+                root.address[0] = hdr.version
             return
         #
         # If 'obj' was already forwarded, change it to its forwarding address.

From noreply at buildbot.pypy.org  Tue Apr 17 22:43:40 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 17 Apr 2012 22:43:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix a crash PyIter_Next() when
 space.next() raises an exception.
Message-ID: <20120417204340.B3B1F820B3@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54486:52c0ca3486b4
Date: 2012-04-17 22:28 +0200
http://bitbucket.org/pypy/pypy/changeset/52c0ca3486b4/

Log:	Fix a crash PyIter_Next() when space.next() raises an exception.

diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()

From noreply at buildbot.pypy.org  Tue Apr 17 22:43:42 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 17 Apr 2012 22:43:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: cpyext: fix interpretation of the
 "direction" parameter of PyUnicode_Tailmatch.
Message-ID: <20120417204342.1194182F4F@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54487:11d96d5e877f
Date: 2012-04-17 22:42 +0200
http://bitbucket.org/pypy/pypy/changeset/11d96d5e877f/

Log:	cpyext: fix interpretation of the "direction" parameter of
	PyUnicode_Tailmatch.

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,8 +453,8 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, 1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)

From noreply at buildbot.pypy.org  Tue Apr 17 23:34:50 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 17 Apr 2012 23:34:50 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: minimal change to call
 validate_fd in app_posix.py 's fdopen()
Message-ID: <20120417213450.B69D9820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54488:2626cbaaeb24
Date: 2012-04-17 23:47 +0300
http://bitbucket.org/pypy/pypy/changeset/2626cbaaeb24/

Log:	minimal change to call validate_fd in app_posix.py 's fdopen()

diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -93,6 +93,7 @@
     # XXX XXX XXX you want do check whether the modes are compatible
     # otherwise you get funny results
     os_flags, universal, reading, writing, basemode, binary = decode_mode(mode)
+    rposix.validate_fd(fd)
     _setfd_binary(fd)
     stream = DiskFile(fd)
     return construct_stream_tower(stream, buffering, universal, reading,

From noreply at buildbot.pypy.org  Tue Apr 17 23:34:51 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 17 Apr 2012 23:34:51 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix exception,
 add test that proves the exception cannot be raised without
 crashing pypy
Message-ID: <20120417213451.F0A83820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54489:e2357d1f1715
Date: 2012-04-18 00:34 +0300
http://bitbucket.org/pypy/pypy/changeset/e2357d1f1715/

Log:	fix exception, add test that proves the exception cannot be raised
	without crashing pypy

diff --git a/pypy/rlib/rwin32.py b/pypy/rlib/rwin32.py
--- a/pypy/rlib/rwin32.py
+++ b/pypy/rlib/rwin32.py
@@ -79,6 +79,7 @@
         for name in """FORMAT_MESSAGE_ALLOCATE_BUFFER FORMAT_MESSAGE_FROM_SYSTEM
                        MAX_PATH
                        WAIT_OBJECT_0 WAIT_TIMEOUT INFINITE
+                       ERROR_INVALID_HANDLE
                     """.split():
             locals()[name] = rffi_platform.ConstantInteger(name)
 
@@ -131,7 +132,7 @@
         validate_fd(fd)
         handle = _get_osfhandle(fd)
         if handle == INVALID_HANDLE_VALUE:
-            raise WindowsError(errno.EBADF, "Invalid file handle")
+            raise WindowsError(ERROR_INVALID_HANDLE, "Invalid file handle")
         return handle
 
     def build_winerror_to_errno():
diff --git a/pypy/rlib/test/test_rwin32.py b/pypy/rlib/test/test_rwin32.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rwin32.py
@@ -0,0 +1,24 @@
+from pypy.rlib import rwin32
+from pypy.tool.udir import udir
+
+
+def test_get_osfhandle():
+    fid = open(str(udir.join('validate_test.txt')), 'w')
+    fd = fid.fileno()
+    rwin32.get_osfhandle(fd)
+    fid.close()
+    raises(OSError, rwin32.get_osfhandle, fd)
+    rwin32.get_osfhandle(0)
+
+def test_get_osfhandle_raising():
+    #try to test what kind of exception get_osfhandle raises w/out fd validation
+    skip('Crashes python')
+    fid = open(str(udir.join('validate_test.txt')), 'w')
+    fd = fid.fileno()
+    fid.close()
+    def validate_fd(fd):
+        return 1
+    _validate_fd = rwin32.validate_fd
+    rwin32.validate_fd = validate_fd
+    raises(WindowsError, rwin32.get_osfhandle, fd)
+    rwin32.validate_fd = _validate_fd

From noreply at buildbot.pypy.org  Tue Apr 17 23:50:23 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 17 Apr 2012 23:50:23 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: skip on non-windows
Message-ID: <20120417215023.51B87820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54490:1888669f91c0
Date: 2012-04-18 00:49 +0300
http://bitbucket.org/pypy/pypy/changeset/1888669f91c0/

Log:	skip on non-windows

diff --git a/pypy/rlib/test/test_rwin32.py b/pypy/rlib/test/test_rwin32.py
--- a/pypy/rlib/test/test_rwin32.py
+++ b/pypy/rlib/test/test_rwin32.py
@@ -1,3 +1,7 @@
+import os
+if os.name <> 'nt':
+    skip('tests for win32 only')
+
 from pypy.rlib import rwin32
 from pypy.tool.udir import udir
 

From notifications-noreply at bitbucket.org  Wed Apr 18 02:32:05 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Wed, 18 Apr 2012 00:32:05 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120418003205.9126.80259@bitbucket12.managed.contegix.com>

You have received a notification from pydcollins.

Hi, I forked pypy. My fork is at https://bitbucket.org/pydcollins/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From notifications-noreply at bitbucket.org  Wed Apr 18 02:45:40 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Wed, 18 Apr 2012 00:45:40 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120418004540.9096.10613@bitbucket03.managed.contegix.com>

You have received a notification from Felipe Cruz.

Hi, I forked pypy. My fork is at https://bitbucket.org/loogica/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Wed Apr 18 13:10:44 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 18 Apr 2012 13:10:44 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification/py3k: first clean merge from
 stdlib-unification
Message-ID: <20120418111044.47A99820B3@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification/py3k
Changeset: r54491:affde7897fe1
Date: 2012-04-18 13:09 +0200
http://bitbucket.org/pypy/pypy/changeset/affde7897fe1/

Log:	first clean merge from stdlib-unification

diff too long, truncating to 10000 out of 205402 lines

diff --git a/lib-python/2.7/UserDict.py b/lib-python/2.7/UserDict.py
--- a/lib-python/2.7/UserDict.py
+++ b/lib-python/2.7/UserDict.py
@@ -80,8 +80,12 @@
     def __iter__(self):
         return iter(self.data)
 
-import _abcoll
-_abcoll.MutableMapping.register(IterableUserDict)
+try:
+    import _abcoll
+except ImportError:
+    pass    # e.g. no '_weakref' module on this pypy
+else:
+    _abcoll.MutableMapping.register(IterableUserDict)
 
 
 class DictMixin:
diff --git a/lib-python/2.7/_threading_local.py b/lib-python/2.7/_threading_local.py
--- a/lib-python/2.7/_threading_local.py
+++ b/lib-python/2.7/_threading_local.py
@@ -155,7 +155,7 @@
         object.__setattr__(self, '_local__args', (args, kw))
         object.__setattr__(self, '_local__lock', RLock())
 
-        if (args or kw) and (cls.__init__ is object.__init__):
+        if (args or kw) and (cls.__init__ == object.__init__):
             raise TypeError("Initialization arguments are not supported")
 
         # We need to create the thread dict in anticipation of
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -7,6 +7,7 @@
 
 __version__ = "1.1.0"
 
+import _ffi
 from _ctypes import Union, Structure, Array
 from _ctypes import _Pointer
 from _ctypes import CFuncPtr as _CFuncPtr
@@ -350,16 +351,17 @@
         self._FuncPtr = _FuncPtr
 
         if handle is None:
-            self._handle = _dlopen(self._name, mode)
+            self._handle = _ffi.CDLL(name, mode)
         else:
             self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %x at %x>" % \
+        return "<%s '%s', handle %r at %x>" % \
                (self.__class__.__name__, self._name,
-                (self._handle & (_sys.maxint*2 + 1)),
+                (self._handle),
                 id(self) & (_sys.maxint*2 + 1))
 
+
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
             raise AttributeError(name)
@@ -487,9 +489,12 @@
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
-    return _cast(obj, obj, typ)
+    try:
+        c_void_p.from_param(obj)
+    except TypeError, e:
+        raise ArgumentError(str(e))
+    return _cast_addr(obj, obj, typ)
 
 _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
diff --git a/lib-python/2.7/ctypes/test/__init__.py b/lib-python/2.7/ctypes/test/__init__.py
--- a/lib-python/2.7/ctypes/test/__init__.py
+++ b/lib-python/2.7/ctypes/test/__init__.py
@@ -206,3 +206,16 @@
         result = unittest.TestResult()
         test(result)
         return result
+
+def xfail(method):
+    """
+    Poor's man xfail: remove it when all the failures have been fixed
+    """
+    def new_method(self, *args, **kwds):
+        try:
+            method(self, *args, **kwds)
+        except:
+            pass
+        else:
+            self.assertTrue(False, "DID NOT RAISE")
+    return new_method
diff --git a/lib-python/2.7/ctypes/test/test_arrays.py b/lib-python/2.7/ctypes/test/test_arrays.py
--- a/lib-python/2.7/ctypes/test/test_arrays.py
+++ b/lib-python/2.7/ctypes/test/test_arrays.py
@@ -1,12 +1,23 @@
 import unittest
 from ctypes import *
+from test.test_support import impl_detail
 
 formats = "bBhHiIlLqQfd"
 
+# c_longdouble commented out for PyPy, look at the commend in test_longdouble
 formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \
-          c_long, c_ulonglong, c_float, c_double, c_longdouble
+          c_long, c_ulonglong, c_float, c_double #, c_longdouble
 
 class ArrayTestCase(unittest.TestCase):
+
+    @impl_detail('long double not supported by PyPy', pypy=False)
+    def test_longdouble(self):
+        """
+        This test is empty. It's just here to remind that we commented out
+        c_longdouble in "formats". If pypy will ever supports c_longdouble, we
+        should kill this test and uncomment c_longdouble inside formats.
+        """
+
     def test_simple(self):
         # create classes holding simple numeric types, and check
         # various properties.
diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py
--- a/lib-python/2.7/ctypes/test/test_bitfields.py
+++ b/lib-python/2.7/ctypes/test/test_bitfields.py
@@ -115,17 +115,21 @@
     def test_nonint_types(self):
         # bit fields are not allowed on non-integer types.
         result = self.fail_fields(("a", c_char_p, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         result = self.fail_fields(("a", c_void_p, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         if c_int != c_long:
             result = self.fail_fields(("a", POINTER(c_int), 1))
-            self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int'))
+            self.assertEqual(result[0], TypeError)
+            self.assertIn('bit fields not allowed for type', result[1])
 
         result = self.fail_fields(("a", c_char, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
         try:
             c_wchar
@@ -133,13 +137,15 @@
             pass
         else:
             result = self.fail_fields(("a", c_wchar, 1))
-            self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_wchar'))
+            self.assertEqual(result[0], TypeError)
+            self.assertIn('bit fields not allowed for type', result[1])
 
         class Dummy(Structure):
             _fields_ = []
 
         result = self.fail_fields(("a", Dummy, 1))
-        self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy'))
+        self.assertEqual(result[0], TypeError)
+        self.assertIn('bit fields not allowed for type', result[1])
 
     def test_single_bitfield_size(self):
         for c_typ in int_types:
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -2,6 +2,7 @@
 from binascii import hexlify
 
 from ctypes import *
+from ctypes.test import xfail
 
 def bin(s):
     return hexlify(memoryview(s)).upper()
@@ -21,6 +22,7 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
+    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertTrue(c_short.__ctype_le__ is c_short)
@@ -48,6 +50,7 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
+    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertTrue(c_int.__ctype_le__ is c_int)
@@ -76,6 +79,7 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
+    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertTrue(c_longlong.__ctype_le__ is c_longlong)
@@ -104,6 +108,7 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
+    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertTrue(c_float.__ctype_le__ is c_float)
@@ -122,6 +127,7 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
+    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertTrue(c_double.__ctype_le__ is c_double)
@@ -149,6 +155,7 @@
         self.assertTrue(c_char.__ctype_le__ is c_char)
         self.assertTrue(c_char.__ctype_be__ is c_char)
 
+    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -198,6 +205,7 @@
             pass
         self.assertRaises(TypeError, setattr, S, "_fields_", [("s", T)])
 
+    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -221,6 +229,7 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
+    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_callbacks.py b/lib-python/2.7/ctypes/test/test_callbacks.py
--- a/lib-python/2.7/ctypes/test/test_callbacks.py
+++ b/lib-python/2.7/ctypes/test/test_callbacks.py
@@ -1,5 +1,6 @@
 import unittest
 from ctypes import *
+from ctypes.test import xfail
 import _ctypes_test
 
 class Callbacks(unittest.TestCase):
@@ -98,6 +99,7 @@
 ##        self.check_type(c_char_p, "abc")
 ##        self.check_type(c_char_p, "def")
 
+    @xfail
     def test_pyobject(self):
         o = ()
         from sys import getrefcount as grc
diff --git a/lib-python/2.7/ctypes/test/test_cfuncs.py b/lib-python/2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/2.7/ctypes/test/test_cfuncs.py
@@ -3,8 +3,8 @@
 
 import unittest
 from ctypes import *
-
 import _ctypes_test
+from test.test_support import impl_detail
 
 class CFunctions(unittest.TestCase):
     _dll = CDLL(_ctypes_test.__file__)
@@ -158,12 +158,14 @@
         self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
         self.assertEqual(self.S(), 42)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble(self):
         self._dll.tf_D.restype = c_longdouble
         self._dll.tf_D.argtypes = (c_longdouble,)
         self.assertEqual(self._dll.tf_D(42.), 14.)
         self.assertEqual(self.S(), 42)
-
+        
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble_plus(self):
         self._dll.tf_bD.restype = c_longdouble
         self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
diff --git a/lib-python/2.7/ctypes/test/test_delattr.py b/lib-python/2.7/ctypes/test/test_delattr.py
--- a/lib-python/2.7/ctypes/test/test_delattr.py
+++ b/lib-python/2.7/ctypes/test/test_delattr.py
@@ -6,15 +6,15 @@
 
 class TestCase(unittest.TestCase):
     def test_simple(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, c_int(42), "value")
 
     def test_chararray(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, (c_char * 5)(), "value")
 
     def test_struct(self):
-        self.assertRaises(TypeError,
+        self.assertRaises((TypeError, AttributeError),
                           delattr, X(), "foo")
 
 if __name__ == "__main__":
diff --git a/lib-python/2.7/ctypes/test/test_frombuffer.py b/lib-python/2.7/ctypes/test/test_frombuffer.py
--- a/lib-python/2.7/ctypes/test/test_frombuffer.py
+++ b/lib-python/2.7/ctypes/test/test_frombuffer.py
@@ -2,6 +2,7 @@
 import array
 import gc
 import unittest
+from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("c_int", c_int)]
@@ -10,6 +11,7 @@
         self._init_called = True
 
 class Test(unittest.TestCase):
+    @xfail
     def test_fom_buffer(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer(a)
@@ -35,6 +37,7 @@
         self.assertRaises(TypeError,
                           (c_char * 16).from_buffer, "a" * 16)
 
+    @xfail
     def test_fom_buffer_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer(a, sizeof(c_int))
@@ -43,6 +46,7 @@
         self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int)))
         self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int)))
 
+    @xfail
     def test_from_buffer_copy(self):
         a = array.array("i", range(16))
         x = (c_int * 16).from_buffer_copy(a)
@@ -67,6 +71,7 @@
         x = (c_char * 16).from_buffer_copy("a" * 16)
         self.assertEqual(x[:], "a" * 16)
 
+    @xfail
     def test_fom_buffer_copy_with_offset(self):
         a = array.array("i", range(16))
         x = (c_int * 15).from_buffer_copy(a, sizeof(c_int))
diff --git a/lib-python/2.7/ctypes/test/test_functions.py b/lib-python/2.7/ctypes/test/test_functions.py
--- a/lib-python/2.7/ctypes/test/test_functions.py
+++ b/lib-python/2.7/ctypes/test/test_functions.py
@@ -7,6 +7,8 @@
 
 from ctypes import *
 import sys, unittest
+from ctypes.test import xfail
+from test.test_support import impl_detail
 
 try:
     WINFUNCTYPE
@@ -143,6 +145,7 @@
         self.assertEqual(result, -21)
         self.assertEqual(type(result), float)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdoubleresult(self):
         f = dll._testfunc_D_bhilfD
         f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
@@ -393,6 +396,7 @@
             self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h),
                                  (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9))
 
+    @xfail
     def test_sf1651235(self):
         # see http://www.python.org/sf/1651235
 
diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py
--- a/lib-python/2.7/ctypes/test/test_internals.py
+++ b/lib-python/2.7/ctypes/test/test_internals.py
@@ -33,7 +33,13 @@
         refcnt = grc(s)
         cs = c_char_p(s)
         self.assertEqual(refcnt + 1, grc(s))
-        self.assertSame(cs._objects, s)
+        try:
+            # Moving gcs need to allocate a nonmoving buffer
+            cs._objects._obj
+        except AttributeError:
+            self.assertSame(cs._objects, s)
+        else:
+            self.assertSame(cs._objects._obj, s)
 
     def test_simple_struct(self):
         class X(Structure):
diff --git a/lib-python/2.7/ctypes/test/test_libc.py b/lib-python/2.7/ctypes/test/test_libc.py
--- a/lib-python/2.7/ctypes/test/test_libc.py
+++ b/lib-python/2.7/ctypes/test/test_libc.py
@@ -25,5 +25,14 @@
         lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort))
         self.assertEqual(chars.raw, "   ,,aaaadmmmnpppsss\x00")
 
+    def SKIPPED_test_no_more_xfail(self):
+        # We decided to not explicitly support the whole ctypes-2.7
+        # and instead go for a case-by-case, demand-driven approach.
+        # So this test is skipped instead of failing.
+        import socket
+        import ctypes.test
+        self.assertTrue(not hasattr(ctypes.test, 'xfail'),
+                        "You should incrementally grep for '@xfail' and remove them, they are real failures")
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_loading.py b/lib-python/2.7/ctypes/test/test_loading.py
--- a/lib-python/2.7/ctypes/test/test_loading.py
+++ b/lib-python/2.7/ctypes/test/test_loading.py
@@ -2,7 +2,7 @@
 import sys, unittest
 import os
 from ctypes.util import find_library
-from ctypes.test import is_resource_enabled
+from ctypes.test import is_resource_enabled, xfail
 
 libc_name = None
 if os.name == "nt":
@@ -75,6 +75,7 @@
             self.assertRaises(AttributeError, dll.__getitem__, 1234)
 
     if os.name == "nt":
+        @xfail
         def test_1703286_A(self):
             from _ctypes import LoadLibrary, FreeLibrary
             # On winXP 64-bit, advapi32 loads at an address that does
@@ -85,6 +86,7 @@
             handle = LoadLibrary("advapi32")
             FreeLibrary(handle)
 
+        @xfail
         def test_1703286_B(self):
             # Since on winXP 64-bit advapi32 loads like described
             # above, the (arbitrarily selected) CloseEventLog function
diff --git a/lib-python/2.7/ctypes/test/test_macholib.py b/lib-python/2.7/ctypes/test/test_macholib.py
--- a/lib-python/2.7/ctypes/test/test_macholib.py
+++ b/lib-python/2.7/ctypes/test/test_macholib.py
@@ -52,7 +52,6 @@
                                  '/usr/lib/libSystem.B.dylib')
 
             result = find_lib('z')
-            self.assertTrue(result.startswith('/usr/lib/libz.1'))
             self.assertTrue(result.endswith('.dylib'))
 
             self.assertEqual(find_lib('IOKit'),
diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py
--- a/lib-python/2.7/ctypes/test/test_numbers.py
+++ b/lib-python/2.7/ctypes/test/test_numbers.py
@@ -1,6 +1,7 @@
 from ctypes import *
 import unittest
 import struct
+from ctypes.test import xfail
 
 def valid_ranges(*types):
     # given a sequence of numeric types, collect their _type_
@@ -89,12 +90,14 @@
 ##            self.assertRaises(ValueError, t, l-1)
 ##            self.assertRaises(ValueError, t, h+1)
 
+    @xfail
     def test_from_param(self):
         # the from_param class method attribute always
         # returns PyCArgObject instances
         for t in signed_types + unsigned_types + float_types:
             self.assertEqual(ArgType, type(t.from_param(0)))
 
+    @xfail
     def test_byref(self):
         # calling byref returns also a PyCArgObject instance
         for t in signed_types + unsigned_types + float_types + bool_types:
@@ -102,6 +105,7 @@
             self.assertEqual(ArgType, type(parm))
 
 
+    @xfail
     def test_floats(self):
         # c_float and c_double can be created from
         # Python int, long and float
@@ -115,6 +119,7 @@
             self.assertEqual(t(2L).value, 2.0)
             self.assertEqual(t(f).value, 2.0)
 
+    @xfail
     def test_integers(self):
         class FloatLike(object):
             def __float__(self):
diff --git a/lib-python/2.7/ctypes/test/test_objects.py b/lib-python/2.7/ctypes/test/test_objects.py
--- a/lib-python/2.7/ctypes/test/test_objects.py
+++ b/lib-python/2.7/ctypes/test/test_objects.py
@@ -22,7 +22,7 @@
 
 >>> array[4] = 'foo bar'
 >>> array._objects
-{'4': 'foo bar'}
+{'4': <CArgObject 'foo bar'>}
 >>> array[4]
 'foo bar'
 >>>
@@ -47,9 +47,9 @@
 
 >>> x.array[0] = 'spam spam spam'
 >>> x._objects
-{'0:2': 'spam spam spam'}
+{'0:2': <CArgObject 'spam spam spam'>}
 >>> x.array._b_base_._objects
-{'0:2': 'spam spam spam'}
+{'0:2': <CArgObject 'spam spam spam'>}
 >>>
 
 '''
diff --git a/lib-python/2.7/ctypes/test/test_parameters.py b/lib-python/2.7/ctypes/test/test_parameters.py
--- a/lib-python/2.7/ctypes/test/test_parameters.py
+++ b/lib-python/2.7/ctypes/test/test_parameters.py
@@ -1,5 +1,7 @@
 import unittest, sys
 
+from ctypes.test import xfail
+
 class SimpleTypesTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -49,6 +51,7 @@
         self.assertEqual(CWCHARP.from_param("abc"), "abcabcabc")
 
     # XXX Replace by c_char_p tests
+    @xfail
     def test_cstrings(self):
         from ctypes import c_char_p, byref
 
@@ -86,7 +89,10 @@
 
         pa = c_wchar_p.from_param(c_wchar_p(u"123"))
         self.assertEqual(type(pa), c_wchar_p)
+    if sys.platform == "win32":
+        test_cw_strings = xfail(test_cw_strings)
 
+    @xfail
     def test_int_pointers(self):
         from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer
         LPINT = POINTER(c_int)
diff --git a/lib-python/2.7/ctypes/test/test_pep3118.py b/lib-python/2.7/ctypes/test/test_pep3118.py
--- a/lib-python/2.7/ctypes/test/test_pep3118.py
+++ b/lib-python/2.7/ctypes/test/test_pep3118.py
@@ -1,6 +1,7 @@
 import unittest
 from ctypes import *
 import re, sys
+from ctypes.test import xfail
 
 if sys.byteorder == "little":
     THIS_ENDIAN = "<"
@@ -19,6 +20,7 @@
 
 class Test(unittest.TestCase):
 
+    @xfail
     def test_native_types(self):
         for tp, fmt, shape, itemtp in native_types:
             ob = tp()
@@ -46,6 +48,7 @@
                 print(tp)
                 raise
 
+    @xfail
     def test_endian_types(self):
         for tp, fmt, shape, itemtp in endian_types:
             ob = tp()
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -3,6 +3,7 @@
 from ctypes import *
 import _ctypes_test
 dll = CDLL(_ctypes_test.__file__)
+from ctypes.test import xfail
 
 class X(Structure):
     _fields_ = [("a", c_int), ("b", c_double)]
@@ -21,6 +22,7 @@
     def loads(self, item):
         return pickle.loads(item)
 
+    @xfail
     def test_simple(self):
         for src in [
             c_int(42),
@@ -31,6 +33,7 @@
             self.assertEqual(memoryview(src).tobytes(),
                                  memoryview(dst).tobytes())
 
+    @xfail
     def test_struct(self):
         X.init_called = 0
 
@@ -49,6 +52,7 @@
         self.assertEqual(memoryview(y).tobytes(),
                              memoryview(x).tobytes())
 
+    @xfail
     def test_unpickable(self):
         # ctypes objects that are pointers or contain pointers are
         # unpickable.
@@ -66,6 +70,7 @@
             ]:
             self.assertRaises(ValueError, lambda: self.dumps(item))
 
+    @xfail
     def test_wchar(self):
         pickle.dumps(c_char("x"))
         # Issue 5049
diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py
--- a/lib-python/2.7/ctypes/test/test_python_api.py
+++ b/lib-python/2.7/ctypes/test/test_python_api.py
@@ -1,6 +1,6 @@
 from ctypes import *
 import unittest, sys
-from ctypes.test import is_resource_enabled
+from ctypes.test import is_resource_enabled, xfail
 
 ################################################################
 # This section should be moved into ctypes\__init__.py, when it's ready.
@@ -17,6 +17,7 @@
 
 class PythonAPITestCase(unittest.TestCase):
 
+    @xfail
     def test_PyString_FromStringAndSize(self):
         PyString_FromStringAndSize = pythonapi.PyString_FromStringAndSize
 
@@ -25,6 +26,7 @@
 
         self.assertEqual(PyString_FromStringAndSize("abcdefghi", 3), "abc")
 
+    @xfail
     def test_PyString_FromString(self):
         pythonapi.PyString_FromString.restype = py_object
         pythonapi.PyString_FromString.argtypes = (c_char_p,)
@@ -56,6 +58,7 @@
             del res
             self.assertEqual(grc(42), ref42)
 
+    @xfail
     def test_PyObj_FromPtr(self):
         s = "abc def ghi jkl"
         ref = grc(s)
@@ -81,6 +84,7 @@
         # not enough arguments
         self.assertRaises(TypeError, PyOS_snprintf, buf)
 
+    @xfail
     def test_pyobject_repr(self):
         self.assertEqual(repr(py_object()), "py_object(<NULL>)")
         self.assertEqual(repr(py_object(42)), "py_object(42)")
diff --git a/lib-python/2.7/ctypes/test/test_refcounts.py b/lib-python/2.7/ctypes/test/test_refcounts.py
--- a/lib-python/2.7/ctypes/test/test_refcounts.py
+++ b/lib-python/2.7/ctypes/test/test_refcounts.py
@@ -90,6 +90,7 @@
             return a * b * 2
         f = proto(func)
 
+        gc.collect()
         a = sys.getrefcount(ctypes.c_int)
         f(1, 2)
         self.assertEqual(sys.getrefcount(ctypes.c_int), a)
diff --git a/lib-python/2.7/ctypes/test/test_stringptr.py b/lib-python/2.7/ctypes/test/test_stringptr.py
--- a/lib-python/2.7/ctypes/test/test_stringptr.py
+++ b/lib-python/2.7/ctypes/test/test_stringptr.py
@@ -2,11 +2,13 @@
 from ctypes import *
 
 import _ctypes_test
+from ctypes.test import xfail
 
 lib = CDLL(_ctypes_test.__file__)
 
 class StringPtrTestCase(unittest.TestCase):
 
+    @xfail
     def test__POINTER_c_char(self):
         class X(Structure):
             _fields_ = [("str", POINTER(c_char))]
@@ -27,6 +29,7 @@
 
         self.assertRaises(TypeError, setattr, x, "str", "Hello, World")
 
+    @xfail
     def test__c_char_p(self):
         class X(Structure):
             _fields_ = [("str", c_char_p)]
diff --git a/lib-python/2.7/ctypes/test/test_strings.py b/lib-python/2.7/ctypes/test/test_strings.py
--- a/lib-python/2.7/ctypes/test/test_strings.py
+++ b/lib-python/2.7/ctypes/test/test_strings.py
@@ -31,8 +31,9 @@
         buf.value = "Hello, World"
         self.assertEqual(buf.value, "Hello, World")
 
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("Hello, World"))
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
+        if test_support.check_impl_detail():
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("Hello, World"))
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
         self.assertRaises(ValueError, setattr, buf, "raw", memoryview("x" * 100))
 
     def test_c_buffer_raw(self, memoryview=memoryview):
@@ -40,7 +41,8 @@
 
         buf.raw = memoryview("Hello, World")
         self.assertEqual(buf.value, "Hello, World")
-        self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
+        if test_support.check_impl_detail():
+            self.assertRaises(TypeError, setattr, buf, "value", memoryview("abc"))
         self.assertRaises(ValueError, setattr, buf, "raw", memoryview("x" * 100))
 
     def test_c_buffer_deprecated(self):
diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py
--- a/lib-python/2.7/ctypes/test/test_structures.py
+++ b/lib-python/2.7/ctypes/test/test_structures.py
@@ -194,8 +194,8 @@
         self.assertEqual(X.b.offset, min(8, longlong_align))
 
 
-        d = {"_fields_": [("a", "b"),
-                          ("b", "q")],
+        d = {"_fields_": [("a", c_byte),
+                          ("b", c_longlong)],
              "_pack_": -1}
         self.assertRaises(ValueError, type(Structure), "X", (Structure,), d)
 
diff --git a/lib-python/2.7/ctypes/test/test_varsize_struct.py b/lib-python/2.7/ctypes/test/test_varsize_struct.py
--- a/lib-python/2.7/ctypes/test/test_varsize_struct.py
+++ b/lib-python/2.7/ctypes/test/test_varsize_struct.py
@@ -1,7 +1,9 @@
 from ctypes import *
 import unittest
+from ctypes.test import xfail
 
 class VarSizeTest(unittest.TestCase):
+    @xfail
     def test_resize(self):
         class X(Structure):
             _fields_ = [("item", c_int),
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -72,8 +72,8 @@
         return name
 
 if os.name == "posix" and sys.platform == "darwin":
-    from ctypes.macholib.dyld import dyld_find as _dyld_find
     def find_library(name):
+        from ctypes.macholib.dyld import dyld_find as _dyld_find
         possible = ['lib%s.dylib' % name,
                     '%s.dylib' % name,
                     '%s.framework/%s' % (name, name)]
diff --git a/lib-python/2.7/distutils/command/bdist_wininst.py b/lib-python/2.7/distutils/command/bdist_wininst.py
--- a/lib-python/2.7/distutils/command/bdist_wininst.py
+++ b/lib-python/2.7/distutils/command/bdist_wininst.py
@@ -298,7 +298,8 @@
                              bitmaplen,        # number of bytes in bitmap
                              )
         file.write(header)
-        file.write(open(arcname, "rb").read())
+        with open(arcname, "rb") as arcfile:
+            file.write(arcfile.read())
 
     # create_exe()
 
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -184,7 +184,7 @@
             # the 'libs' directory is for binary installs - we assume that
             # must be the *native* platform.  But we don't really support
             # cross-compiling via a binary install anyway, so we let it go.
-            self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs'))
+            self.library_dirs.append(os.path.join(sys.exec_prefix, 'include'))
             if self.debug:
                 self.build_temp = os.path.join(self.build_temp, "Debug")
             else:
@@ -192,8 +192,13 @@
 
             # Append the source distribution include and library directories,
             # this allows distutils on windows to work in the source tree
-            self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
-            if MSVC_VERSION == 9:
+            if 0:
+                # pypy has no PC directory
+                self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
+            if 1:
+                # pypy has no PCBuild directory
+                pass
+            elif MSVC_VERSION == 9:
                 # Use the .lib files for the correct architecture
                 if self.plat_name == 'win32':
                     suffix = ''
@@ -695,24 +700,14 @@
         shared extension.  On most platforms, this is just 'ext.libraries';
         on Windows and OS/2, we add the Python library (eg. python20.dll).
         """
-        # The python library is always needed on Windows.  For MSVC, this
-        # is redundant, since the library is mentioned in a pragma in
-        # pyconfig.h that MSVC groks.  The other Windows compilers all seem
-        # to need it mentioned explicitly, though, so that's what we do.
-        # Append '_d' to the python import library on debug builds.
+        # The python library is always needed on Windows.
         if sys.platform == "win32":
-            from distutils.msvccompiler import MSVCCompiler
-            if not isinstance(self.compiler, MSVCCompiler):
-                template = "python%d%d"
-                if self.debug:
-                    template = template + '_d'
-                pythonlib = (template %
-                       (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
-                # don't extend ext.libraries, it may be shared with other
-                # extensions, it is a reference to the original list
-                return ext.libraries + [pythonlib]
-            else:
-                return ext.libraries
+            template = "python%d%d"
+            pythonlib = (template %
+                   (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
+            # don't extend ext.libraries, it may be shared with other
+            # extensions, it is a reference to the original list
+            return ext.libraries + [pythonlib]
         elif sys.platform == "os2emx":
             # EMX/GCC requires the python library explicitly, and I
             # believe VACPP does as well (though not confirmed) - AIM Apr01
diff --git a/lib-python/2.7/distutils/command/install.py b/lib-python/2.7/distutils/command/install.py
--- a/lib-python/2.7/distutils/command/install.py
+++ b/lib-python/2.7/distutils/command/install.py
@@ -83,6 +83,13 @@
         'scripts': '$userbase/bin',
         'data'   : '$userbase',
         },
+    'pypy': {
+        'purelib': '$base/site-packages',
+        'platlib': '$base/site-packages',
+        'headers': '$base/include',
+        'scripts': '$base/bin',
+        'data'   : '$base',
+        },
     }
 
 # The keys to an installation scheme; if any new types of files are to be
@@ -467,6 +474,8 @@
 
     def select_scheme (self, name):
         # it's the caller's problem if they supply a bad name!
+        if hasattr(sys, 'pypy_version_info'):
+            name = 'pypy'
         scheme = INSTALL_SCHEMES[name]
         for key in SCHEME_KEYS:
             attrname = 'install_' + key
diff --git a/lib-python/2.7/distutils/cygwinccompiler.py b/lib-python/2.7/distutils/cygwinccompiler.py
--- a/lib-python/2.7/distutils/cygwinccompiler.py
+++ b/lib-python/2.7/distutils/cygwinccompiler.py
@@ -75,6 +75,9 @@
         elif msc_ver == '1500':
             # VS2008 / MSVC 9.0
             return ['msvcr90']
+        elif msc_ver == '1600':
+            # VS2010 / MSVC 10.0
+            return ['msvcr100']
         else:
             raise ValueError("Unknown MS Compiler version %s " % msc_ver)
 
diff --git a/lib-python/2.7/distutils/msvc9compiler.py b/lib-python/2.7/distutils/msvc9compiler.py
--- a/lib-python/2.7/distutils/msvc9compiler.py
+++ b/lib-python/2.7/distutils/msvc9compiler.py
@@ -648,6 +648,7 @@
             temp_manifest = os.path.join(
                     build_temp,
                     os.path.basename(output_filename) + ".manifest")
+            ld_args.append('/MANIFEST')
             ld_args.append('/MANIFESTFILE:' + temp_manifest)
 
             if extra_preargs:
diff --git a/lib-python/2.7/distutils/spawn.py b/lib-python/2.7/distutils/spawn.py
--- a/lib-python/2.7/distutils/spawn.py
+++ b/lib-python/2.7/distutils/spawn.py
@@ -58,7 +58,6 @@
 
 def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0):
     executable = cmd[0]
-    cmd = _nt_quote_args(cmd)
     if search_path:
         # either we find one or it stays the same
         executable = find_executable(executable) or executable
@@ -66,7 +65,8 @@
     if not dry_run:
         # spawn for NT requires a full path to the .exe
         try:
-            rc = os.spawnv(os.P_WAIT, executable, cmd)
+            import subprocess
+            rc = subprocess.call(cmd)
         except OSError, exc:
             # this seems to happen when the command isn't found
             raise DistutilsExecError, \
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -9,563 +9,21 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id$"
+__revision__ = "$Id: sysconfig.py 85358 2010-10-10 09:54:59Z antoine.pitrou $"
 
-import os
-import re
-import string
 import sys
 
-from distutils.errors import DistutilsPlatformError
 
-# These are needed in a couple of spots, so just compute them once.
-PREFIX = os.path.normpath(sys.prefix)
-EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
+# The content of this file is redirected from
+# sysconfig_cpython or sysconfig_pypy.
 
-# Path to the base directory of the project. On Windows the binary may
-# live in project/PCBuild9.  If we're dealing with an x64 Windows build,
-# it'll live in project/PCbuild/amd64.
-project_base = os.path.dirname(os.path.abspath(sys.executable))
-if os.name == "nt" and "pcbuild" in project_base[-8:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir))
-# PC/VS7.1
-if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
-# PC/AMD64
-if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower():
-    project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
-                                                os.path.pardir))
+if '__pypy__' in sys.builtin_module_names:
+    from distutils.sysconfig_pypy import *
+    from distutils.sysconfig_pypy import _config_vars # needed by setuptools
+    from distutils.sysconfig_pypy import _variable_rx # read_setup_file()
+else:
+    from distutils.sysconfig_cpython import *
+    from distutils.sysconfig_cpython import _config_vars # needed by setuptools
+    from distutils.sysconfig_cpython import _variable_rx # read_setup_file()
 
-# python_build: (Boolean) if true, we're either building Python or
-# building an extension with an un-installed Python, so we use
-# different (hard-wired) directories.
-# Setup.local is available for Makefile builds including VPATH builds,
-# Setup.dist is available on Windows
-def _python_build():
-    for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(project_base, "Modules", fn)):
-            return True
-    return False
-python_build = _python_build()
 
-
-def get_python_version():
-    """Return a string containing the major and minor Python version,
-    leaving off the patchlevel.  Sample return values could be '1.5'
-    or '2.2'.
-    """
-    return sys.version[:3]
-
-
-def get_python_inc(plat_specific=0, prefix=None):
-    """Return the directory containing installed Python header files.
-
-    If 'plat_specific' is false (the default), this is the path to the
-    non-platform-specific header files, i.e. Python.h and so on;
-    otherwise, this is the path to platform-specific header files
-    (namely pyconfig.h).
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        if python_build:
-            buildir = os.path.dirname(sys.executable)
-            if plat_specific:
-                # python.h is located in the buildir
-                inc_dir = buildir
-            else:
-                # the source dir is relative to the buildir
-                srcdir = os.path.abspath(os.path.join(buildir,
-                                         get_config_var('srcdir')))
-                # Include is located in the srcdir
-                inc_dir = os.path.join(srcdir, "Include")
-            return inc_dir
-        return os.path.join(prefix, "include", "python" + get_python_version())
-    elif os.name == "nt":
-        return os.path.join(prefix, "include")
-    elif os.name == "os2":
-        return os.path.join(prefix, "Include")
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its C header files "
-            "on platform '%s'" % os.name)
-
-
-def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
-    """Return the directory containing the Python library (standard or
-    site additions).
-
-    If 'plat_specific' is true, return the directory containing
-    platform-specific modules, i.e. any module from a non-pure-Python
-    module distribution; otherwise, return the platform-shared library
-    directory.  If 'standard_lib' is true, return the directory
-    containing standard Python library modules; otherwise, return the
-    directory for site-specific modules.
-
-    If 'prefix' is supplied, use it instead of sys.prefix or
-    sys.exec_prefix -- i.e., ignore 'plat_specific'.
-    """
-    if prefix is None:
-        prefix = plat_specific and EXEC_PREFIX or PREFIX
-
-    if os.name == "posix":
-        libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
-        if standard_lib:
-            return libpython
-        else:
-            return os.path.join(libpython, "site-packages")
-
-    elif os.name == "nt":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            if get_python_version() < "2.2":
-                return prefix
-            else:
-                return os.path.join(prefix, "Lib", "site-packages")
-
-    elif os.name == "os2":
-        if standard_lib:
-            return os.path.join(prefix, "Lib")
-        else:
-            return os.path.join(prefix, "Lib", "site-packages")
-
-    else:
-        raise DistutilsPlatformError(
-            "I don't know where Python installs its library "
-            "on platform '%s'" % os.name)
-
-
-def customize_compiler(compiler):
-    """Do any platform-specific customization of a CCompiler instance.
-
-    Mainly needed on Unix, so we can plug in the information that
-    varies across Unices and is stored in Python's Makefile.
-    """
-    if compiler.compiler_type == "unix":
-        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext) = \
-            get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SO')
-
-        if 'CC' in os.environ:
-            cc = os.environ['CC']
-        if 'CXX' in os.environ:
-            cxx = os.environ['CXX']
-        if 'LDSHARED' in os.environ:
-            ldshared = os.environ['LDSHARED']
-        if 'CPP' in os.environ:
-            cpp = os.environ['CPP']
-        else:
-            cpp = cc + " -E"           # not always
-        if 'LDFLAGS' in os.environ:
-            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        if 'CPPFLAGS' in os.environ:
-            cpp = cpp + ' ' + os.environ['CPPFLAGS']
-            cflags = cflags + ' ' + os.environ['CPPFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-
-        cc_cmd = cc + ' ' + cflags
-        compiler.set_executables(
-            preprocessor=cpp,
-            compiler=cc_cmd,
-            compiler_so=cc_cmd + ' ' + ccshared,
-            compiler_cxx=cxx,
-            linker_so=ldshared,
-            linker_exe=cc)
-
-        compiler.shared_lib_extension = so_ext
-
-
-def get_config_h_filename():
-    """Return full pathname of installed pyconfig.h file."""
-    if python_build:
-        if os.name == "nt":
-            inc_dir = os.path.join(project_base, "PC")
-        else:
-            inc_dir = project_base
-    else:
-        inc_dir = get_python_inc(plat_specific=1)
-    if get_python_version() < '2.2':
-        config_h = 'config.h'
-    else:
-        # The name of the config.h file changed in 2.2
-        config_h = 'pyconfig.h'
-    return os.path.join(inc_dir, config_h)
-
-
-def get_makefile_filename():
-    """Return full pathname of installed Makefile from the Python build."""
-    if python_build:
-        return os.path.join(os.path.dirname(sys.executable), "Makefile")
-    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
-    return os.path.join(lib_dir, "config", "Makefile")
-
-
-def parse_config_h(fp, g=None):
-    """Parse a config.h-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    if g is None:
-        g = {}
-    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
-    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
-    #
-    while 1:
-        line = fp.readline()
-        if not line:
-            break
-        m = define_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            try: v = int(v)
-            except ValueError: pass
-            g[n] = v
-        else:
-            m = undef_rx.match(line)
-            if m:
-                g[m.group(1)] = 0
-    return g
-
-
-# Regexes needed for parsing Makefile (and similar syntaxes,
-# like old-style Setup files).
-_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-def parse_makefile(fn, g=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    from distutils.text_file import TextFile
-    fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1)
-
-    if g is None:
-        g = {}
-    done = {}
-    notdone = {}
-
-    while 1:
-        line = fp.readline()
-        if line is None:  # eof
-            break
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-
-    fp.close()
-
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    g.update(done)
-    return g
-
-
-def expand_makefile_vars(s, vars):
-    """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in
-    'string' according to 'vars' (a dictionary mapping variable names to
-    values).  Variables not present in 'vars' are silently expanded to the
-    empty string.  The variable values in 'vars' should not contain further
-    variable expansions; if 'vars' is the output of 'parse_makefile()',
-    you're fine.  Returns a variable-expanded version of 's'.
-    """
-
-    # This algorithm does multiple expansion, so if vars['foo'] contains
-    # "${bar}", it will expand ${foo} to ${bar}, and then expand
-    # ${bar}... and so forth.  This is fine as long as 'vars' comes from
-    # 'parse_makefile()', which takes care of such expansions eagerly,
-    # according to make's variable expansion semantics.
-
-    while 1:
-        m = _findvar1_rx.search(s) or _findvar2_rx.search(s)
-        if m:
-            (beg, end) = m.span()
-            s = s[0:beg] + vars.get(m.group(1)) + s[end:]
-        else:
-            break
-    return s
-
-
-_config_vars = None
-
-def _init_posix():
-    """Initialize the module as appropriate for POSIX systems."""
-    g = {}
-    # load the installed Makefile:
-    try:
-        filename = get_makefile_filename()
-        parse_makefile(filename, g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # load the installed pyconfig.h:
-    try:
-        filename = get_config_h_filename()
-        parse_config_h(file(filename), g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # On MacOSX we need to check the setting of the environment variable
-    # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
-    # it needs to be compatible.
-    # If it isn't set we set it to the configure-time value
-    if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g:
-        cfg_target = g['MACOSX_DEPLOYMENT_TARGET']
-        cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
-        if cur_target == '':
-            cur_target = cfg_target
-            os.environ['MACOSX_DEPLOYMENT_TARGET'] = cfg_target
-        elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
-            my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
-                % (cur_target, cfg_target))
-            raise DistutilsPlatformError(my_msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if python_build:
-        g['LDSHARED'] = g['BLDSHARED']
-
-    elif get_python_version() < '2.1':
-        # The following two branches are for 1.5.2 compatibility.
-        if sys.platform == 'aix4':          # what about AIX 3.x ?
-            # Linker script is in the config directory, not in Modules as the
-            # Makefile says.
-            python_lib = get_python_lib(standard_lib=1)
-            ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
-            python_exp = os.path.join(python_lib, 'config', 'python.exp')
-
-            g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
-
-        elif sys.platform == 'beos':
-            # Linker script is in the config directory.  In the Makefile it is
-            # relative to the srcdir, which after installation no longer makes
-            # sense.
-            python_lib = get_python_lib(standard_lib=1)
-            linkerscript_path = string.split(g['LDSHARED'])[0]
-            linkerscript_name = os.path.basename(linkerscript_path)
-            linkerscript = os.path.join(python_lib, 'config',
-                                        linkerscript_name)
-
-            # XXX this isn't the right place to do this: adding the Python
-            # library to the link, if needed, should be in the "build_ext"
-            # command.  (It's also needed for non-MS compilers on Windows, and
-            # it's taken care of for them by the 'build_ext.get_libraries()'
-            # method.)
-            g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
-                             (linkerscript, PREFIX, get_python_version()))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_nt():
-    """Initialize the module as appropriate for NT"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-    g['VERSION'] = get_python_version().replace(".", "")
-    g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable))
-
-    global _config_vars
-    _config_vars = g
-
-
-def _init_os2():
-    """Initialize the module as appropriate for OS/2"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    g['SO'] = '.pyd'
-    g['EXE'] = ".exe"
-
-    global _config_vars
-    _config_vars = g
-
-
-def get_config_vars(*args):
-    """With no arguments, return a dictionary of all configuration
-    variables relevant for the current platform.  Generally this includes
-    everything needed to build extensions and install both pure modules and
-    extensions.  On Unix, this means every variable defined in Python's
-    installed Makefile; on Windows and Mac OS it's a much smaller set.
-
-    With arguments, return a list of values that result from looking up
-    each argument in the configuration variable dictionary.
-    """
-    global _config_vars
-    if _config_vars is None:
-        func = globals().get("_init_" + os.name)
-        if func:
-            func()
-        else:
-            _config_vars = {}
-
-        # Normalized versions of prefix and exec_prefix are handy to have;
-        # in fact, these are the standard versions used most places in the
-        # Distutils.
-        _config_vars['prefix'] = PREFIX
-        _config_vars['exec_prefix'] = EXEC_PREFIX
-
-        if sys.platform == 'darwin':
-            kernel_version = os.uname()[2] # Kernel version (8.4.3)
-            major_version = int(kernel_version.split('.')[0])
-
-            if major_version < 8:
-                # On Mac OS X before 10.4, check if -arch and -isysroot
-                # are in CFLAGS or LDFLAGS and remove them if they are.
-                # This is needed when building extensions on a 10.3 system
-                # using a universal build of python.
-                for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-                    flags = _config_vars[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
-                    _config_vars[key] = flags
-
-            else:
-
-                # Allow the user to override the architecture flags using
-                # an environment variable.
-                # NOTE: This name was introduced by Apple in OSX 10.5 and
-                # is used by several scripting languages distributed with
-                # that OS release.
-
-                if 'ARCHFLAGS' in os.environ:
-                    arch = os.environ['ARCHFLAGS']
-                    for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                        flags = _config_vars[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _config_vars[key] = flags
-
-                # If we're on OSX 10.5 or later and the user tries to
-                # compiles an extension using an SDK that is not present
-                # on the current machine it is better to not use an SDK
-                # than to fail.
-                #
-                # The major usecase for this is users using a Python.org
-                # binary installer  on OSX 10.6: that installer uses
-                # the 10.4u SDK, but that SDK is not installed by default
-                # when you install Xcode.
-                #
-                m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS'])
-                if m is not None:
-                    sdk = m.group(1)
-                    if not os.path.exists(sdk):
-                        for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                             # a number of derived variables. These need to be
-                             # patched up as well.
-                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                            flags = _config_vars[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
-                            _config_vars[key] = flags
-
-    if args:
-        vals = []
-        for name in args:
-            vals.append(_config_vars.get(name))
-        return vals
-    else:
-        return _config_vars
-
-def get_config_var(name):
-    """Return the value of a single variable using the dictionary
-    returned by 'get_config_vars()'.  Equivalent to
-    get_config_vars().get(name)
-    """
-    return get_config_vars().get(name)
diff --git a/lib-python/modified-2.7/distutils/sysconfig_cpython.py b/lib-python/2.7/distutils/sysconfig_cpython.py
rename from lib-python/modified-2.7/distutils/sysconfig_cpython.py
rename to lib-python/2.7/distutils/sysconfig_cpython.py
diff --git a/lib-python/modified-2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
rename from lib-python/modified-2.7/distutils/sysconfig_pypy.py
rename to lib-python/2.7/distutils/sysconfig_pypy.py
diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py
--- a/lib-python/2.7/distutils/tests/test_build_ext.py
+++ b/lib-python/2.7/distutils/tests/test_build_ext.py
@@ -293,7 +293,7 @@
         finally:
             os.chdir(old_wd)
         self.assertTrue(os.path.exists(so_file))
-        self.assertEqual(os.path.splitext(so_file)[-1],
+        self.assertEqual(so_file[so_file.index(os.path.extsep):],
                          sysconfig.get_config_var('SO'))
         so_dir = os.path.dirname(so_file)
         self.assertEqual(so_dir, other_tmp_dir)
@@ -302,7 +302,7 @@
         cmd.run()
         so_file = cmd.get_outputs()[0]
         self.assertTrue(os.path.exists(so_file))
-        self.assertEqual(os.path.splitext(so_file)[-1],
+        self.assertEqual(so_file[so_file.index(os.path.extsep):],
                          sysconfig.get_config_var('SO'))
         so_dir = os.path.dirname(so_file)
         self.assertEqual(so_dir, cmd.build_lib)
diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py
--- a/lib-python/2.7/distutils/tests/test_install.py
+++ b/lib-python/2.7/distutils/tests/test_install.py
@@ -2,6 +2,7 @@
 
 import os
 import unittest
+from test import test_support
 
 from test.test_support import run_unittest
 
@@ -40,14 +41,15 @@
             expected = os.path.normpath(expected)
             self.assertEqual(got, expected)
 
-        libdir = os.path.join(destination, "lib", "python")
-        check_path(cmd.install_lib, libdir)
-        check_path(cmd.install_platlib, libdir)
-        check_path(cmd.install_purelib, libdir)
-        check_path(cmd.install_headers,
-                   os.path.join(destination, "include", "python", "foopkg"))
-        check_path(cmd.install_scripts, os.path.join(destination, "bin"))
-        check_path(cmd.install_data, destination)
+        if test_support.check_impl_detail():
+            libdir = os.path.join(destination, "lib", "python")
+            check_path(cmd.install_lib, libdir)
+            check_path(cmd.install_platlib, libdir)
+            check_path(cmd.install_purelib, libdir)
+            check_path(cmd.install_headers,
+                       os.path.join(destination, "include", "python", "foopkg"))
+            check_path(cmd.install_scripts, os.path.join(destination, "bin"))
+            check_path(cmd.install_data, destination)
 
 
 def test_suite():
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -125,7 +125,22 @@
                   }
 
     if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
         executables['ranlib'] = ["ranlib"]
+        executables['linker_so'] += ['-undefined', 'dynamic_lookup']
+
+        for k, v in executables.iteritems():
+            if v and v[0] == 'cc':
+                v += ['-arch', arch]
+
 
     # Needed for the filename generation methods provided by the base
     # class, CCompiler.  NB. whoever instantiates/uses a particular
@@ -309,7 +324,7 @@
             # On OSX users can specify an alternate SDK using
             # '-isysroot', calculate the SDK root if it is specified
             # (and use it further on)
-            cflags = sysconfig.get_config_var('CFLAGS')
+            cflags = sysconfig.get_config_var('CFLAGS') or ''
             m = re.search(r'-isysroot\s+(\S+)', cflags)
             if m is None:
                 sysroot = '/'
diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py
--- a/lib-python/2.7/heapq.py
+++ b/lib-python/2.7/heapq.py
@@ -193,6 +193,8 @@
 
     Equivalent to:  sorted(iterable, reverse=True)[:n]
     """
+    if n < 0: # for consistency with the c impl
+        return []
     it = iter(iterable)
     result = list(islice(it, n))
     if not result:
@@ -209,6 +211,8 @@
 
     Equivalent to:  sorted(iterable)[:n]
     """
+    if n < 0: # for consistency with the c impl
+        return []
     if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
         # For smaller values of n, the bisect method is faster than a minheap.
         # It is also memory efficient, consuming only n elements of space.
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -1024,7 +1024,11 @@
             kwds["buffering"] = True;
         response = self.response_class(*args, **kwds)
 
-        response.begin()
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
         assert response.will_close != _UNKNOWN
         self.__state = _CS_IDLE
 
diff --git a/lib-python/2.7/idlelib/Delegator.py b/lib-python/2.7/idlelib/Delegator.py
--- a/lib-python/2.7/idlelib/Delegator.py
+++ b/lib-python/2.7/idlelib/Delegator.py
@@ -12,6 +12,14 @@
         self.__cache[name] = attr
         return attr
 
+    def __nonzero__(self):
+        # this is needed for PyPy: else, if self.delegate is None, the
+        # __getattr__ above picks NoneType.__nonzero__, which returns
+        # False. Thus, bool(Delegator()) is False as well, but it's not what
+        # we want.  On CPython, bool(Delegator()) is True because NoneType
+        # does not have __nonzero__
+        return True
+
     def resetcache(self):
         for key in self.__cache.keys():
             try:
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -746,8 +746,15 @@
     'varargs' and 'varkw' are the names of the * and ** arguments or None."""
 
     if not iscode(co):
-        raise TypeError('{!r} is not a code object'.format(co))
+        if hasattr(len, 'func_code') and type(co) is type(len.func_code):
+            # PyPy extension: built-in function objects have a func_code too.
+            # There is no co_code on it, but co_argcount and co_varnames and
+            # co_flags are present.
+            pass
+        else:
+            raise TypeError('{!r} is not a code object'.format(co))
 
+    code = getattr(co, 'co_code', '')
     nargs = co.co_argcount
     names = co.co_varnames
     args = list(names[:nargs])
@@ -757,12 +764,12 @@
     for i in range(nargs):
         if args[i][:1] in ('', '.'):
             stack, remain, count = [], [], []
-            while step < len(co.co_code):
-                op = ord(co.co_code[step])
+            while step < len(code):
+                op = ord(code[step])
                 step = step + 1
                 if op >= dis.HAVE_ARGUMENT:
                     opname = dis.opname[op]
-                    value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
+                    value = ord(code[step]) + ord(code[step+1])*256
                     step = step + 2
                     if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
                         remain.append(value)
@@ -809,7 +816,9 @@
 
     if ismethod(func):
         func = func.im_func
-    if not isfunction(func):
+    if not (isfunction(func) or
+            isbuiltin(func) and hasattr(func, 'func_code')):
+            # PyPy extension: this works for built-in functions too
         raise TypeError('{!r} is not a Python function'.format(func))
     args, varargs, varkw = getargs(func.func_code)
     return ArgSpec(args, varargs, varkw, func.func_defaults)
@@ -949,7 +958,7 @@
                 raise TypeError('%s() takes exactly 0 arguments '
                                 '(%d given)' % (f_name, num_total))
         else:
-            raise TypeError('%s() takes no arguments (%d given)' %
+            raise TypeError('%s() takes no argument (%d given)' %
                             (f_name, num_total))
     for arg in args:
         if isinstance(arg, str) and arg in named:
diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py
--- a/lib-python/2.7/json/encoder.py
+++ b/lib-python/2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,23 +17,22 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
 FLOAT_REPR = repr
 
-def encode_basestring(s):
+def raw_encode_basestring(s):
     """Return a JSON representation of a Python string
 
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
+encode_basestring = lambda s: '"' + raw_encode_basestring(s) + '"'
 
-
-def py_encode_basestring_ascii(s):
+def raw_encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,21 +45,19 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+encode_basestring_ascii = lambda s: '"' + raw_encode_basestring_ascii(s) + '"'
 
 
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
-
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
 
@@ -147,6 +137,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = raw_encode_basestring_ascii
+        else:
+            self.encoder = raw_encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +185,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +320,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and self.indent is None and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +375,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +385,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +431,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +440,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +448,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +461,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +492,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/2.7/json/tests/test_unicode.py b/lib-python/2.7/json/tests/test_unicode.py
--- a/lib-python/2.7/json/tests/test_unicode.py
+++ b/lib-python/2.7/json/tests/test_unicode.py
@@ -80,6 +80,12 @@
         # Issue 10038.
         self.assertEqual(type(self.loads('"foo"')), unicode)
 
+    def test_encode_not_utf_8(self):
+        self.assertEqual(self.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(self.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
+
 
 class TestPyUnicode(TestUnicode, PyTest): pass
 class TestCUnicode(TestUnicode, CTest): pass
diff --git a/lib-python/2.7/multiprocessing/forking.py b/lib-python/2.7/multiprocessing/forking.py
--- a/lib-python/2.7/multiprocessing/forking.py
+++ b/lib-python/2.7/multiprocessing/forking.py
@@ -73,15 +73,12 @@
         return getattr, (m.im_self, m.im_func.func_name)
 ForkingPickler.register(type(ForkingPickler.save), _reduce_method)
 
-def _reduce_method_descriptor(m):
-    return getattr, (m.__objclass__, m.__name__)
-ForkingPickler.register(type(list.append), _reduce_method_descriptor)
-ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
-
-#def _reduce_builtin_function_or_method(m):
-#    return getattr, (m.__self__, m.__name__)
-#ForkingPickler.register(type(list().append), _reduce_builtin_function_or_method)
-#ForkingPickler.register(type(int().__add__), _reduce_builtin_function_or_method)
+if type(list.append) is not type(ForkingPickler.save):
+    # Some python implementations have unbound methods even for builtin types
+    def _reduce_method_descriptor(m):
+        return getattr, (m.__objclass__, m.__name__)
+    ForkingPickler.register(type(list.append), _reduce_method_descriptor)
+    ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
 
 try:
     from functools import partial
diff --git a/lib-python/2.7/opcode.py b/lib-python/2.7/opcode.py
--- a/lib-python/2.7/opcode.py
+++ b/lib-python/2.7/opcode.py
@@ -1,4 +1,3 @@
-
 """
 opcode module - potentially shared between dis and other modules which
 operate on bytecodes (e.g. peephole optimizers).
@@ -189,4 +188,10 @@
 def_op('SET_ADD', 146)
 def_op('MAP_ADD', 147)
 
+# pypy modification, experimental bytecode
+def_op('LOOKUP_METHOD', 201)          # Index in name list
+hasname.append(201)
+def_op('CALL_METHOD', 202)            # #args not including 'self'
+def_op('BUILD_LIST_FROM_ARG', 203)
+
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/2.7/pickle.py b/lib-python/2.7/pickle.py
--- a/lib-python/2.7/pickle.py
+++ b/lib-python/2.7/pickle.py
@@ -168,7 +168,7 @@
 
 # Pickling machinery
 
-class Pickler:
+class Pickler(object):
 
     def __init__(self, file, protocol=None):
         """This takes a file-like object for writing a pickle data stream.
@@ -638,6 +638,10 @@
             # else tmp is empty, and we're done
 
     def save_dict(self, obj):
+        modict_saver = self._pickle_moduledict(obj)
+        if modict_saver is not None:
+            return self.save_reduce(*modict_saver)
+
         write = self.write
 
         if self.bin:
@@ -687,6 +691,29 @@
                 write(SETITEM)
             # else tmp is empty, and we're done
 
+    def _pickle_moduledict(self, obj):
+        # save module dictionary as "getattr(module, '__dict__')"
+
+        # build index of module dictionaries
+        try:
+            modict = self.module_dict_ids
+        except AttributeError:
+            modict = {}
+            from sys import modules
+            for mod in modules.values():
+                if isinstance(mod, ModuleType):
+                    modict[id(mod.__dict__)] = mod
+            self.module_dict_ids = modict
+
+        thisid = id(obj)
+        try:
+            themodule = modict[thisid]
+        except KeyError:
+            return None
+        from __builtin__ import getattr
+        return getattr, (themodule, '__dict__')
+
+
     def save_inst(self, obj):
         cls = obj.__class__
 
@@ -727,6 +754,29 @@
 
     dispatch[InstanceType] = save_inst
 
+    def save_function(self, obj):
+        try:
+            return self.save_global(obj)
+        except PicklingError, e:
+            pass
+        # Check copy_reg.dispatch_table
+        reduce = dispatch_table.get(type(obj))
+        if reduce:
+            rv = reduce(obj)
+        else:
+            # Check for a __reduce_ex__ method, fall back to __reduce__
+            reduce = getattr(obj, "__reduce_ex__", None)
+            if reduce:
+                rv = reduce(self.proto)
+            else:
+                reduce = getattr(obj, "__reduce__", None)
+                if reduce:
+                    rv = reduce()
+                else:
+                    raise e
+        return self.save_reduce(obj=obj, *rv)
+    dispatch[FunctionType] = save_function
+
     def save_global(self, obj, name=None, pack=struct.pack):
         write = self.write
         memo = self.memo
@@ -768,7 +818,6 @@
         self.memoize(obj)
 
     dispatch[ClassType] = save_global
-    dispatch[FunctionType] = save_global
     dispatch[BuiltinFunctionType] = save_global
     dispatch[TypeType] = save_global
 
@@ -824,7 +873,7 @@
 
 # Unpickling machinery
 
-class Unpickler:
+class Unpickler(object):
 
     def __init__(self, file):
         """This takes a file-like object for reading a pickle data stream.
diff --git a/lib-python/2.7/pkgutil.py b/lib-python/2.7/pkgutil.py
--- a/lib-python/2.7/pkgutil.py
+++ b/lib-python/2.7/pkgutil.py
@@ -244,7 +244,8 @@
         return mod
 
     def get_data(self, pathname):
-        return open(pathname, "rb").read()
+        with open(pathname, "rb") as f:
+            return f.read()
 
     def _reopen(self):
         if self.file and self.file.closed:
diff --git a/lib-python/2.7/pprint.py b/lib-python/2.7/pprint.py
--- a/lib-python/2.7/pprint.py
+++ b/lib-python/2.7/pprint.py
@@ -144,7 +144,7 @@
             return
 
         r = getattr(typ, "__repr__", None)
-        if issubclass(typ, dict) and r is dict.__repr__:
+        if issubclass(typ, dict) and r == dict.__repr__:
             write('{')
             if self._indent_per_level > 1:
                 write((self._indent_per_level - 1) * ' ')
@@ -173,10 +173,10 @@
             write('}')
             return
 
-        if ((issubclass(typ, list) and r is list.__repr__) or
-            (issubclass(typ, tuple) and r is tuple.__repr__) or
-            (issubclass(typ, set) and r is set.__repr__) or
-            (issubclass(typ, frozenset) and r is frozenset.__repr__)
+        if ((issubclass(typ, list) and r == list.__repr__) or
+            (issubclass(typ, tuple) and r == tuple.__repr__) or
+            (issubclass(typ, set) and r == set.__repr__) or
+            (issubclass(typ, frozenset) and r == frozenset.__repr__)
            ):
             length = _len(object)
             if issubclass(typ, list):
@@ -266,7 +266,7 @@
         return ("%s%s%s" % (closure, sio.getvalue(), closure)), True, False
 
     r = getattr(typ, "__repr__", None)
-    if issubclass(typ, dict) and r is dict.__repr__:
+    if issubclass(typ, dict) and r == dict.__repr__:
         if not object:
             return "{}", True, False
         objid = _id(object)
@@ -291,8 +291,8 @@
         del context[objid]
         return "{%s}" % _commajoin(components), readable, recursive
 
-    if (issubclass(typ, list) and r is list.__repr__) or \
-       (issubclass(typ, tuple) and r is tuple.__repr__):
+    if (issubclass(typ, list) and r == list.__repr__) or \
+       (issubclass(typ, tuple) and r == tuple.__repr__):
         if issubclass(typ, list):
             if not object:
                 return "[]", True, False
diff --git a/lib-python/2.7/pydoc.py b/lib-python/2.7/pydoc.py
--- a/lib-python/2.7/pydoc.py
+++ b/lib-python/2.7/pydoc.py
@@ -623,7 +623,9 @@
             head, '#ffffff', '#7799ee',
             '<a href=".">index</a><br>' + filelink + docloc)
 
-        modules = inspect.getmembers(object, inspect.ismodule)
+        def isnonbuiltinmodule(obj):
+            return inspect.ismodule(obj) and obj is not __builtin__
+        modules = inspect.getmembers(object, isnonbuiltinmodule)
 
         classes, cdict = [], {}
         for key, value in inspect.getmembers(object, inspect.isclass):
diff --git a/lib-python/2.7/random.py b/lib-python/2.7/random.py
--- a/lib-python/2.7/random.py
+++ b/lib-python/2.7/random.py
@@ -41,7 +41,6 @@
 
 from __future__ import division
 from warnings import warn as _warn
-from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethodType
 from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil
 from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin
 from os import urandom as _urandom
@@ -240,8 +239,7 @@
 
         return self.randrange(a, b+1)
 
-    def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L<<BPF,
-                   _Method=_MethodType, _BuiltinMethod=_BuiltinMethodType):
+    def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L<<BPF):
         """Return a random int in the range [0,n)
 
         Handles the case where n has more bits than returned
@@ -256,7 +254,8 @@
             # Only call self.getrandbits if the original random() builtin method
             # has not been overridden or if a new getrandbits() was supplied.
             # This assures that the two methods correspond.
-            if type(self.random) is _BuiltinMethod or type(getrandbits) is _Method:
+            if (self.random == super(Random, self).random or
+                getrandbits != super(Random, self).getrandbits):
                 k = int(1.00001 + _log(n-1, 2.0))   # 2**k > n-1 > 2**(k-2)
                 r = getrandbits(k)
                 while r >= n:
diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py
--- a/lib-python/2.7/site.py
+++ b/lib-python/2.7/site.py
@@ -75,7 +75,6 @@
 USER_SITE = None
 USER_BASE = None
 
-
 def makepath(*paths):
     dir = os.path.join(*paths)
     try:
@@ -91,7 +90,10 @@
         if hasattr(m, '__loader__'):
             continue   # don't mess with a PEP 302-supplied __file__
         try:
-            m.__file__ = os.path.abspath(m.__file__)
+            prev = m.__file__
+            new = os.path.abspath(m.__file__)
+            if prev != new:
+                m.__file__ = new
         except (AttributeError, OSError):
             pass
 
@@ -289,6 +291,7 @@
     will find its `site-packages` subdirectory depending on the system
     environment, and will return a list of full paths.
     """
+    is_pypy = '__pypy__' in sys.builtin_module_names
     sitepackages = []
     seen = set()
 
@@ -299,6 +302,10 @@
 
         if sys.platform in ('os2emx', 'riscos'):
             sitepackages.append(os.path.join(prefix, "Lib", "site-packages"))
+        elif is_pypy:
+            from distutils.sysconfig import get_python_lib
+            sitedir = get_python_lib(standard_lib=False, prefix=prefix)
+            sitepackages.append(sitedir)
         elif os.sep == '/':
             sitepackages.append(os.path.join(prefix, "lib",
                                         "python" + sys.version[:3],
@@ -435,22 +442,33 @@
                 if key == 'q':
                     break
 
+##def setcopyright():
+##    """Set 'copyright' and 'credits' in __builtin__"""
+##    __builtin__.copyright = _Printer("copyright", sys.copyright)
+##    if sys.platform[:4] == 'java':
+##        __builtin__.credits = _Printer(
+##            "credits",
+##            "Jython is maintained by the Jython developers (www.jython.org).")
+##    else:
+##        __builtin__.credits = _Printer("credits", """\
+##    Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
+##    for supporting Python development.  See www.python.org for more information.""")
+##    here = os.path.dirname(os.__file__)
+##    __builtin__.license = _Printer(
+##        "license", "See http://www.python.org/%.3s/license.html" % sys.version,
+##        ["LICENSE.txt", "LICENSE"],
+##        [os.path.join(here, os.pardir), here, os.curdir])
+
 def setcopyright():
-    """Set 'copyright' and 'credits' in __builtin__"""
+    # XXX this is the PyPy-specific version.  Should be unified with the above.
     __builtin__.copyright = _Printer("copyright", sys.copyright)
-    if sys.platform[:4] == 'java':
-        __builtin__.credits = _Printer(
-            "credits",
-            "Jython is maintained by the Jython developers (www.jython.org).")
-    else:
-        __builtin__.credits = _Printer("credits", """\
-    Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
-    for supporting Python development.  See www.python.org for more information.""")
-    here = os.path.dirname(os.__file__)
+    __builtin__.credits = _Printer(
+        "credits",
+        "PyPy is maintained by the PyPy developers: http://pypy.org/")
     __builtin__.license = _Printer(
-        "license", "See http://www.python.org/%.3s/license.html" % sys.version,
-        ["LICENSE.txt", "LICENSE"],
-        [os.path.join(here, os.pardir), here, os.curdir])
+        "license",
+        "See https://bitbucket.org/pypy/pypy/src/default/LICENSE")
+
 
 
 class _Helper(object):
@@ -476,7 +494,7 @@
     if sys.platform == 'win32':
         import locale, codecs
         enc = locale.getdefaultlocale()[1]
-        if enc.startswith('cp'):            # "cp***" ?
+        if enc is not None and enc.startswith('cp'):            # "cp***" ?
             try:
                 codecs.lookup(enc)
             except LookupError:
@@ -532,9 +550,18 @@
                 "'import usercustomize' failed; use -v for traceback"
 
 
+def import_builtin_stuff():
+    """PyPy specific: pre-import a few built-in modules, because
+    some programs actually rely on them to be in sys.modules :-("""
+    import exceptions
+    if 'zipimport' in sys.builtin_module_names:
+        import zipimport
+
+
 def main():
     global ENABLE_USER_SITE
 
+    import_builtin_stuff()
     abs__file__()
     known_paths = removeduppaths()
     if (os.name == "posix" and sys.path and
diff --git a/lib-python/2.7/socket.py b/lib-python/2.7/socket.py
--- a/lib-python/2.7/socket.py
+++ b/lib-python/2.7/socket.py
@@ -46,8 +46,6 @@
 
 import _socket
 from _socket import *
-from functools import partial
-from types import MethodType
 
 try:
     import _ssl
@@ -159,11 +157,6 @@
 if sys.platform == "riscos":
     _socketmethods = _socketmethods + ('sleeptaskw',)
 
-# All the method names that must be delegated to either the real socket
-# object or the _closedsocket object.
-_delegate_methods = ("recv", "recvfrom", "recv_into", "recvfrom_into",
-                     "send", "sendto")
-
 class _closedsocket(object):
     __slots__ = []
     def _dummy(*args):
@@ -180,22 +173,43 @@
 
     __doc__ = _realsocket.__doc__
 
-    __slots__ = ["_sock", "__weakref__"] + list(_delegate_methods)
-
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
         if _sock is None:
             _sock = _realsocket(family, type, proto)
         self._sock = _sock
-        for method in _delegate_methods:
-            setattr(self, method, getattr(_sock, method))
+        self._io_refs = 0
+        self._closed = False
 
-    def close(self, _closedsocket=_closedsocket,
-              _delegate_methods=_delegate_methods, setattr=setattr):
+    def send(self, data, flags=0):
+        return self._sock.send(data, flags=flags)
+    send.__doc__ = _realsocket.send.__doc__
+
+    def recv(self, buffersize, flags=0):
+        return self._sock.recv(buffersize, flags=flags)
+    recv.__doc__ = _realsocket.recv.__doc__
+
+    def recv_into(self, buffer, nbytes=0, flags=0):
+        return self._sock.recv_into(buffer, nbytes=nbytes, flags=flags)
+    recv_into.__doc__ = _realsocket.recv_into.__doc__
+
+    def recvfrom(self, buffersize, flags=0):
+        return self._sock.recvfrom(buffersize, flags=flags)
+    recvfrom.__doc__ = _realsocket.recvfrom.__doc__
+
+    def recvfrom_into(self, buffer, nbytes=0, flags=0):
+        return self._sock.recvfrom_into(buffer, nbytes=nbytes, flags=flags)
+    recvfrom_into.__doc__ = _realsocket.recvfrom_into.__doc__
+
+    def sendto(self, data, param2, param3=None):
+        if param3 is None:
+            return self._sock.sendto(data, param2)
+        else:
+            return self._sock.sendto(data, param2, param3)
+    sendto.__doc__ = _realsocket.sendto.__doc__
+
+    def close(self):
         # This function should not reference any globals. See issue #808164.
         self._sock = _closedsocket()
-        dummy = self._sock._dummy
-        for method in _delegate_methods:
-            setattr(self, method, dummy)
     close.__doc__ = _realsocket.close.__doc__
 
     def accept(self):
@@ -214,21 +228,49 @@
 
         Return a regular file object corresponding to the socket.  The mode
         and bufsize arguments are as for the built-in open() function."""
-        return _fileobject(self._sock, mode, bufsize)
+        self._io_refs += 1
+        return _fileobject(self, mode, bufsize)
+
+    def _decref_socketios(self):
+        if self._io_refs > 0:
+            self._io_refs -= 1
+        if self._closed:
+            self.close()
+
+    def _real_close(self):
+        # This function should not reference any globals. See issue #808164.
+        self._sock.close()
+
+    def close(self):
+        # This function should not reference any globals. See issue #808164.
+        self._closed = True
+        if self._io_refs <= 0:
+            self._real_close()
 
     family = property(lambda self: self._sock.family, doc="the socket family")
     type = property(lambda self: self._sock.type, doc="the socket type")
     proto = property(lambda self: self._sock.proto, doc="the socket protocol")
 
-def meth(name,self,*args):
-    return getattr(self._sock,name)(*args)
+    # Delegate many calls to the raw socket object.
+    _s = ("def %(name)s(self, %(args)s): return self._sock.%(name)s(%(args)s)\n\n"
+          "%(name)s.__doc__ = _realsocket.%(name)s.__doc__\n")
+    for _m in _socketmethods:
+        # yupi! we're on pypy, all code objects have this interface
+        argcount = getattr(_realsocket, _m).im_func.func_code.co_argcount - 1
+        exec _s % {'name': _m, 'args': ', '.join('arg%d' % i for i in range(argcount))}
+    del _m, _s, argcount
 
-for _m in _socketmethods:
-    p = partial(meth,_m)
-    p.__name__ = _m
-    p.__doc__ = getattr(_realsocket,_m).__doc__
-    m = MethodType(p,None,_socketobject)
-    setattr(_socketobject,_m,m)
+    # Delegation methods with default arguments, that the code above
+    # cannot handle correctly
+    def sendall(self, data, flags=0):
+        self._sock.sendall(data, flags)
+    sendall.__doc__ = _realsocket.sendall.__doc__
+
+    def getsockopt(self, level, optname, buflen=None):
+        if buflen is None:
+            return self._sock.getsockopt(level, optname)
+        return self._sock.getsockopt(level, optname, buflen)
+    getsockopt.__doc__ = _realsocket.getsockopt.__doc__
 
 socket = SocketType = _socketobject
 
@@ -278,8 +320,11 @@
             if self._sock:
                 self.flush()
         finally:
-            if self._close:
-                self._sock.close()
+            if self._sock:
+                if self._close:
+                    self._sock.close()
+                else:
+                    self._sock._decref_socketios()
             self._sock = None
 
     def __del__(self):
diff --git a/lib-python/2.7/sqlite3/test/dbapi.py b/lib-python/2.7/sqlite3/test/dbapi.py
--- a/lib-python/2.7/sqlite3/test/dbapi.py
+++ b/lib-python/2.7/sqlite3/test/dbapi.py
@@ -1,4 +1,4 @@
-#-*- coding: ISO-8859-1 -*-
+#-*- coding: iso-8859-1 -*-
 # pysqlite2/test/dbapi.py: tests for DB-API compliance
 #
 # Copyright (C) 2004-2010 Gerhard H&#65533;ring <gh at ghaering.de>
@@ -332,6 +332,9 @@
             def __init__(self):
                 self.value = 5
 
+            def __iter__(self):
+                return self
+
             def next(self):
                 if self.value == 10:
                     raise StopIteration
@@ -826,7 +829,7 @@
         con = sqlite.connect(":memory:")
         con.close()
         try:
-            con()
+            con("select 1")
             self.fail("Should have raised a ProgrammingError")
         except sqlite.ProgrammingError:
             pass
diff --git a/lib-python/2.7/sqlite3/test/regression.py b/lib-python/2.7/sqlite3/test/regression.py
--- a/lib-python/2.7/sqlite3/test/regression.py
+++ b/lib-python/2.7/sqlite3/test/regression.py
@@ -264,6 +264,28 @@
         """
         self.assertRaises(sqlite.Warning, self.con, 1)
 
+    def CheckUpdateDescriptionNone(self):
+        """
+        Call Cursor.update with an UPDATE query and check that it sets the
+        cursor's description to be None.
+        """
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        cur.execute("UPDATE foo SET id = 3 WHERE id = 1")
+        self.assertEqual(cur.description, None)
+
+    def CheckStatementCache(self):
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        values = [(i,) for i in xrange(5)]
+        cur.executemany("INSERT INTO foo (id) VALUES (?)", values)
+
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+        self.con.commit()
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+
 def suite():
     regression_suite = unittest.makeSuite(RegressionTests, "Check")
     return unittest.TestSuite((regression_suite,))
diff --git a/lib-python/2.7/sqlite3/test/userfunctions.py b/lib-python/2.7/sqlite3/test/userfunctions.py
--- a/lib-python/2.7/sqlite3/test/userfunctions.py
+++ b/lib-python/2.7/sqlite3/test/userfunctions.py
@@ -275,12 +275,14 @@
             pass
 
     def CheckAggrNoStep(self):
+        # XXX it's better to raise OperationalError in order to stop
+        # the query earlier.
         cur = self.con.cursor()
         try:
             cur.execute("select nostep(t) from test")
-            self.fail("should have raised an AttributeError")
-        except AttributeError, e:
-            self.assertEqual(e.args[0], "AggrNoStep instance has no attribute 'step'")
+            self.fail("should have raised an OperationalError")
+        except sqlite.OperationalError, e:
+            self.assertEqual(e.args[0], "user-defined aggregate's 'step' method raised error")
 
     def CheckAggrNoFinalize(self):
         cur = self.con.cursor()
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -86,7 +86,7 @@
 else:
     _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
-from socket import socket, _fileobject, _delegate_methods, error as socket_error
+from socket import socket, _fileobject, error as socket_error
 from socket import getnameinfo as _getnameinfo
 import base64        # for DER-to-PEM translation
 import errno
@@ -103,14 +103,6 @@
                  do_handshake_on_connect=True,
                  suppress_ragged_eofs=True, ciphers=None):
         socket.__init__(self, _sock=sock._sock)
-        # The initializer for socket overrides the methods send(), recv(), etc.
-        # in the instancce, which we don't need -- but we want to provide the
-        # methods defined in SSLSocket.
-        for attr in _delegate_methods:
-            try:
-                delattr(self, attr)
-            except AttributeError:
-                pass
 
         if certfile and not keyfile:
             keyfile = certfile
diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py
--- a/lib-python/2.7/subprocess.py
+++ b/lib-python/2.7/subprocess.py
@@ -803,7 +803,7 @@
             elif stderr == PIPE:
                 errread, errwrite = _subprocess.CreatePipe(None, 0)
             elif stderr == STDOUT:
-                errwrite = c2pwrite
+                errwrite = c2pwrite.handle # pass id to not close it
             elif isinstance(stderr, int):
                 errwrite = msvcrt.get_osfhandle(stderr)
             else:
@@ -818,9 +818,13 @@
 
         def _make_inheritable(self, handle):
             """Return a duplicate of handle, which is inheritable"""
-            return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
+            dupl = _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(),
                                 handle, _subprocess.GetCurrentProcess(), 0, 1,
                                 _subprocess.DUPLICATE_SAME_ACCESS)
+            # If the initial handle was obtained with CreatePipe, close it.
+            if not isinstance(handle, int):
+                handle.Close()
+            return dupl
 
 
         def _find_w9xpopen(self):
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -26,6 +26,16 @@
         'scripts': '{base}/bin',
         'data'   : '{base}',
         },
+    'pypy': {
+        'stdlib': '{base}/lib-python',
+        'platstdlib': '{base}/lib-python',
+        'purelib': '{base}/lib-python',
+        'platlib': '{base}/lib-python',
+        'include': '{base}/include',
+        'platinclude': '{base}/include',
+        'scripts': '{base}/bin',
+        'data'   : '{base}',
+        },
     'nt': {
         'stdlib': '{base}/Lib',
         'platstdlib': '{base}/Lib',
@@ -158,7 +168,9 @@
     return res
 
 def _get_default_scheme():
-    if os.name == 'posix':
+    if '__pypy__' in sys.builtin_module_names:
+        return 'pypy'
+    elif os.name == 'posix':
         # the default scheme for posix is posix_prefix
         return 'posix_prefix'
     return os.name
@@ -182,126 +194,9 @@
     return env_base if env_base else joinuser("~", ".local")
 
 
-def _parse_makefile(filename, vars=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    import re
-    # Regexes needed for parsing Makefile (and similar syntaxes,
-    # like old-style Setup files).
-    _variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-    _findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-    _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-    if vars is None:
-        vars = {}
-    done = {}
-    notdone = {}
-
-    with open(filename) as f:
-        lines = f.readlines()
-
-    for line in lines:
-        if line.startswith('#') or line.strip() == '':
-            continue
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    while notdone:
-        for name in notdone.keys():
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-                else:
-                    done[n] = item = ""
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try: value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        del notdone[name]
-            else:
-                # bogus variable reference; just drop it since we can't deal
-                del notdone[name]
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    vars.update(done)
-    return vars
-
-
-def _get_makefile_filename():
-    if _PYTHON_BUILD:
-        return os.path.join(_PROJECT_BASE, "Makefile")
-    return os.path.join(get_path('platstdlib'), "config", "Makefile")
-
-
 def _init_posix(vars):
     """Initialize the module as appropriate for POSIX systems."""
-    # load the installed Makefile:
-    makefile = _get_makefile_filename()
-    try:
-        _parse_makefile(makefile, vars)
-    except IOError, e:
-        msg = "invalid Python installation: unable to open %s" % makefile
-        if hasattr(e, "strerror"):
-            msg = msg + " (%s)" % e.strerror
-        raise IOError(msg)
-
-    # load the installed pyconfig.h:
-    config_h = get_config_h_filename()
-    try:
-        with open(config_h) as f:
-            parse_config_h(f, vars)
-    except IOError, e:
-        msg = "invalid Python installation: unable to open %s" % config_h
-        if hasattr(e, "strerror"):
-            msg = msg + " (%s)" % e.strerror
-        raise IOError(msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if _PYTHON_BUILD:
-        vars['LDSHARED'] = vars['BLDSHARED']
+    return
 
 def _init_non_posix(vars):
     """Initialize the module as appropriate for NT"""
@@ -474,10 +369,11 @@
                         # patched up as well.
                         'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
 
-                        flags = _CONFIG_VARS[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _CONFIG_VARS[key] = flags
+                        if key in _CONFIG_VARS: 
+                            flags = _CONFIG_VARS[key]
+                            flags = re.sub('-arch\s+\w+\s', ' ', flags)
+                            flags = flags + ' ' + arch
+                            _CONFIG_VARS[key] = flags
 
                 # If we're on OSX 10.5 or later and the user tries to
                 # compiles an extension using an SDK that is not present
diff --git a/lib-python/2.7/tarfile.py b/lib-python/2.7/tarfile.py
--- a/lib-python/2.7/tarfile.py
+++ b/lib-python/2.7/tarfile.py
@@ -1716,9 +1716,6 @@
         except (ImportError, AttributeError):
             raise CompressionError("gzip module is not available")
 
-        if fileobj is None:
-            fileobj = bltn_open(name, mode + "b")
-
         try:
             t = cls.taropen(name, mode,
                 gzip.GzipFile(name, mode, compresslevel, fileobj),
diff --git a/lib-python/2.7/test/list_tests.py b/lib-python/2.7/test/list_tests.py
--- a/lib-python/2.7/test/list_tests.py
+++ b/lib-python/2.7/test/list_tests.py
@@ -45,8 +45,12 @@
         self.assertEqual(str(a2), "[0, 1, 2, [...], 3]")
         self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]")
 
+        if test_support.check_impl_detail():
+            depth = sys.getrecursionlimit() + 100
+        else:
+            depth = 1000 * 1000 # should be enough to exhaust the stack
         l0 = []
-        for i in xrange(sys.getrecursionlimit() + 100):
+        for i in xrange(depth):
             l0 = [l0]
         self.assertRaises(RuntimeError, repr, l0)
 
@@ -472,7 +476,11 @@
         u += "eggs"
         self.assertEqual(u, self.type2test("spameggs"))
 
-        self.assertRaises(TypeError, u.__iadd__, None)
+        def f_iadd(u, x):
+            u += x
+            return u
+
+        self.assertRaises(TypeError, f_iadd, u, None)
 
     def test_imul(self):
         u = self.type2test([0, 1])
diff --git a/lib-python/2.7/test/mapping_tests.py b/lib-python/2.7/test/mapping_tests.py
--- a/lib-python/2.7/test/mapping_tests.py
+++ b/lib-python/2.7/test/mapping_tests.py
@@ -531,7 +531,10 @@
                     self.assertEqual(va, int(ka))
                     kb, vb = tb = b.popitem()
                     self.assertEqual(vb, int(kb))
-                    self.assertTrue(not(copymode < 0 and ta != tb))
+                    if copymode < 0 and test_support.check_impl_detail():
+                        # popitem() is not guaranteed to be deterministic on
+                        # all implementations
+                        self.assertEqual(ta, tb)
                 self.assertTrue(not a)
                 self.assertTrue(not b)
 
diff --git a/lib-python/2.7/test/pickletester.py b/lib-python/2.7/test/pickletester.py
--- a/lib-python/2.7/test/pickletester.py
+++ b/lib-python/2.7/test/pickletester.py
@@ -6,7 +6,7 @@
 import pickletools
 import copy_reg
 
-from test.test_support import TestFailed, have_unicode, TESTFN
+from test.test_support import TestFailed, have_unicode, TESTFN, impl_detail
 
 # Tests that try a number of pickle protocols should have a
 #     for proto in protocols:
@@ -949,6 +949,7 @@
                              "Failed protocol %d: %r != %r"
                              % (proto, obj, loaded))
 
+    @impl_detail("pypy does not store attribute names", pypy=False)
     def test_attribute_name_interning(self):
         # Test that attribute names of pickled objects are interned when
         # unpickling.
@@ -1091,6 +1092,7 @@
         s = StringIO.StringIO("X''.")
         self.assertRaises(EOFError, self.module.load, s)
 
+    @impl_detail("no full restricted mode in pypy", pypy=False)
     def test_restricted(self):
         # issue7128: cPickle failed in restricted mode
         builtins = {self.module.__name__: self.module,
diff --git a/lib-python/2.7/test/regrtest.py b/lib-python/2.7/test/regrtest.py
--- a/lib-python/2.7/test/regrtest.py
+++ b/lib-python/2.7/test/regrtest.py
@@ -1388,7 +1388,26 @@
         test_zipimport
         test_zlib
         """,
-    'openbsd3':
+    'openbsd4':
+        """
+        test_ascii_formatd
+        test_bsddb
+        test_bsddb3
+        test_ctypes
+        test_dl
+        test_epoll
+        test_gdbm
+        test_locale
+        test_normalization
+        test_ossaudiodev
+        test_pep277
+        test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
+        test_multiprocessing
+        """,
+    'openbsd5':
         """
         test_ascii_formatd
         test_bsddb
@@ -1503,13 +1522,7 @@
         return self.expected
 
 if __name__ == '__main__':
-    # findtestdir() gets the dirname out of __file__, so we have to make it
-    # absolute before changing the working directory.
-    # For example __file__ may be relative when running trace or profile.
-    # See issue #9323.
-    __file__ = os.path.abspath(__file__)
-
-    # sanity check
+    # Simplification for findtestdir().
     assert __file__ == os.path.abspath(sys.argv[0])
 
     # When tests are run from the Python build directory, it is best practice
diff --git a/lib-python/2.7/test/seq_tests.py b/lib-python/2.7/test/seq_tests.py
--- a/lib-python/2.7/test/seq_tests.py
+++ b/lib-python/2.7/test/seq_tests.py
@@ -307,12 +307,18 @@
 
     def test_bigrepeat(self):
         import sys
-        if sys.maxint <= 2147483647:
-            x = self.type2test([0])
-            x *= 2**16
-            self.assertRaises(MemoryError, x.__mul__, 2**16)
-            if hasattr(x, '__imul__'):
-                self.assertRaises(MemoryError, x.__imul__, 2**16)
+        # we chose an N such as 2**16 * N does not fit into a cpu word
+        if sys.maxint == 2147483647:
+            # 32 bit system
+            N = 2**16
+        else:
+            # 64 bit system
+            N = 2**48
+        x = self.type2test([0])
+        x *= 2**16
+        self.assertRaises(MemoryError, x.__mul__, N)
+        if hasattr(x, '__imul__'):
+            self.assertRaises(MemoryError, x.__imul__, N)
 
     def test_subscript(self):
         a = self.type2test([10, 11])
diff --git a/lib-python/2.7/test/string_tests.py b/lib-python/2.7/test/string_tests.py
--- a/lib-python/2.7/test/string_tests.py
+++ b/lib-python/2.7/test/string_tests.py
@@ -1024,7 +1024,10 @@
         self.checkequal('abc', 'abc', '__mul__', 1)
         self.checkequal('abcabcabc', 'abc', '__mul__', 3)
         self.checkraises(TypeError, 'abc', '__mul__')
-        self.checkraises(TypeError, 'abc', '__mul__', '')
+        class Mul(object):
+            def mul(self, a, b):
+                return a * b
+        self.checkraises(TypeError, Mul(), 'mul', 'abc', '')
         # XXX: on a 64-bit system, this doesn't raise an overflow error,
         # but either raises a MemoryError, or succeeds (if you have 54TiB)
         #self.checkraises(OverflowError, 10000*'abc', '__mul__', 2000000000)
diff --git a/lib-python/2.7/test/test_abstract_numbers.py b/lib-python/2.7/test/test_abstract_numbers.py
--- a/lib-python/2.7/test/test_abstract_numbers.py
+++ b/lib-python/2.7/test/test_abstract_numbers.py
@@ -40,7 +40,8 @@
 
         c1, c2 = complex(3, 2), complex(4,1)
         # XXX: This is not ideal, but see the comment in math_trunc().
-        self.assertRaises(AttributeError, math.trunc, c1)
+        # Modified to suit PyPy, which gives TypeError in all cases
+        self.assertRaises((AttributeError, TypeError), math.trunc, c1)
         self.assertRaises(TypeError, float, c1)
         self.assertRaises(TypeError, int, c1)
 
diff --git a/lib-python/2.7/test/test_aifc.py b/lib-python/2.7/test/test_aifc.py
--- a/lib-python/2.7/test/test_aifc.py
+++ b/lib-python/2.7/test/test_aifc.py
@@ -1,4 +1,4 @@
-from test.test_support import findfile, run_unittest, TESTFN
+from test.test_support import findfile, run_unittest, TESTFN, impl_detail
 import unittest
 import os
 
@@ -68,6 +68,7 @@
         self.assertEqual(f.getparams(), fout.getparams())
         self.assertEqual(f.readframes(5), fout.readframes(5))
 
+    @impl_detail("PyPy has no audioop module yet", pypy=False)
     def test_compress(self):
         f = self.f = aifc.open(self.sndfilepath)
         fout = self.fout = aifc.open(TESTFN, 'wb')
diff --git a/lib-python/2.7/test/test_array.py b/lib-python/2.7/test/test_array.py
--- a/lib-python/2.7/test/test_array.py
+++ b/lib-python/2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, a.__add__, b)
-
-        self.assertRaises(TypeError, a.__add__, "bad")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, a.__add__, b)
-
-        self.assertRaises(TypeError, a.__iadd__, "bad")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, a.__mul__, "bad")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, a.__imul__, "bad")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
@@ -769,6 +773,7 @@
         p = proxy(s)
         self.assertEqual(p.tostring(), s.tostring())
         s = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, len, p)
 
     def test_bug_782369(self):
diff --git a/lib-python/2.7/test/test_ascii_formatd.py b/lib-python/2.7/test/test_ascii_formatd.py
--- a/lib-python/2.7/test/test_ascii_formatd.py
+++ b/lib-python/2.7/test/test_ascii_formatd.py
@@ -4,6 +4,10 @@
 
 import unittest
 from test.test_support import check_warnings, run_unittest, import_module
+from test.test_support import check_impl_detail
+
+if not check_impl_detail(cpython=True):
+    raise unittest.SkipTest("this test is only for CPython")
 
 # Skip tests if _ctypes module does not exist
 import_module('_ctypes')
diff --git a/lib-python/2.7/test/test_ast.py b/lib-python/2.7/test/test_ast.py
--- a/lib-python/2.7/test/test_ast.py
+++ b/lib-python/2.7/test/test_ast.py
@@ -20,10 +20,24 @@
 # These tests are compiled through "exec"
 # There should be atleast one test per statement
 exec_tests = [
+    # None
+    "None",
     # FunctionDef
     "def f(): pass",
+    # FunctionDef with arg
+    "def f(a): pass",
+    # FunctionDef with arg and default value
+    "def f(a=0): pass",
+    # FunctionDef with varargs
+    "def f(*args): pass",
+    # FunctionDef with kwargs
+    "def f(**kwargs): pass",
+    # FunctionDef with all kind of args
+    "def f(a, b=1, c=None, d=[], e={}, *args, **kwargs): pass",
     # ClassDef
     "class C:pass",
+    # ClassDef, new style class
+    "class C(object): pass",
     # Return
     "def f():return 1",
     # Delete
@@ -68,6 +82,27 @@
     "for a,b in c: pass",
     "[(a,b) for a,b in c]",
     "((a,b) for a,b in c)",
+    "((a,b) for (a,b) in c)",
+    # Multiline generator expression
+    """(
+    (
+    Aa
+    ,
+       Bb
+    )
+    for
+    Aa
+    ,
+    Bb in Cc
+    )""",
+    # dictcomp
+    "{a : b for w in x for m in p if g}",
+    # dictcomp with naked tuple
+    "{a : b for v,w in x}",
+    # setcomp
+    "{r for l in x if g}",
+    # setcomp with naked tuple
+    "{r for l,m in x}",
 ]
 
 # These are compiled through "single"
@@ -80,6 +115,8 @@
 # These are compiled through "eval"
 # It should test all expressions
 eval_tests = [
+  # None
+  "None",
   # BoolOp
   "a and b",
   # BinOp
@@ -90,6 +127,16 @@
   "lambda:None",
   # Dict
   "{ 1:2 }",
+  # Empty dict
+  "{}",
+  # Set
+  "{None,}",
+  # Multiline dict
+  """{
+      1
+        :
+          2
+     }""",
   # ListComp
   "[a for b in c if d]",
   # GeneratorExp
@@ -114,8 +161,14 @@
   "v",
   # List
   "[1,2,3]",
+  # Empty list
+  "[]",
   # Tuple
   "1,2,3",
+  # Tuple
+  "(1,2,3)",
+  # Empty tuple
+  "()",
   # Combination
   "a.b.c.d(a.b[1:2])",
 
@@ -141,6 +194,35 @@
             elif value is not None:
                 self._assertTrueorder(value, parent_pos)
 
+    def test_AST_objects(self):
+        if test_support.check_impl_detail():
+            # PyPy also provides a __dict__ to the ast.AST base class.
+
+            x = ast.AST()
+            try:
+                x.foobar = 21
+            except AttributeError, e:
+                self.assertEquals(e.args[0],
+                                  "'_ast.AST' object has no attribute 'foobar'")
+            else:
+                self.assert_(False)
+
+            try:
+                ast.AST(lineno=2)
+            except AttributeError, e:
+                self.assertEquals(e.args[0],
+                                  "'_ast.AST' object has no attribute 'lineno'")
+            else:
+                self.assert_(False)
+
+        try:
+            ast.AST(2)
+        except TypeError, e:
+            self.assertEquals(e.args[0],
+                              "_ast.AST constructor takes 0 positional arguments")
+        else:
+            self.assert_(False)
+
     def test_snippets(self):
         for input, output, kind in ((exec_tests, exec_results, "exec"),
                                     (single_tests, single_results, "single"),
@@ -169,6 +251,114 @@
         self.assertTrue(issubclass(ast.comprehension, ast.AST))
         self.assertTrue(issubclass(ast.Gt, ast.AST))
 
+    def test_field_attr_existence(self):
+        for name, item in ast.__dict__.iteritems():
+            if isinstance(item, type) and name != 'AST' and name[0].isupper(): # XXX: pypy does not allow abstract ast class instanciation
+                x = item()
+                if isinstance(x, ast.AST):
+                    self.assertEquals(type(x._fields), tuple)
+
+    def test_arguments(self):
+        x = ast.arguments()
+        self.assertEquals(x._fields, ('args', 'vararg', 'kwarg', 'defaults'))
+        try:
+            x.vararg
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'arguments' object has no attribute 'vararg'")
+        else:
+            self.assert_(False)
+        x = ast.arguments(1, 2, 3, 4)
+        self.assertEquals(x.vararg, 2)
+
+    def test_field_attr_writable(self):
+        x = ast.Num()
+        # We can assign to _fields
+        x._fields = 666
+        self.assertEquals(x._fields, 666)
+
+    def test_classattrs(self):
+        x = ast.Num()
+        self.assertEquals(x._fields, ('n',))
+        try:
+            x.n
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'n'")
+        else:
+            self.assert_(False)
+
+        x = ast.Num(42)
+        self.assertEquals(x.n, 42)
+        try:
+            x.lineno
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'lineno'")
+        else:
+            self.assert_(False)
+
+        y = ast.Num()
+        x.lineno = y
+        self.assertEquals(x.lineno, y)
+
+        try:
+            x.foobar
+        except AttributeError, e:
+            self.assertEquals(e.args[0],
+                              "'Num' object has no attribute 'foobar'")
+        else:
+            self.assert_(False)
+
+        x = ast.Num(lineno=2)
+        self.assertEquals(x.lineno, 2)
+
+        x = ast.Num(42, lineno=0)
+        self.assertEquals(x.lineno, 0)
+        self.assertEquals(x._fields, ('n',))
+        self.assertEquals(x.n, 42)
+
+        self.assertRaises(TypeError, ast.Num, 1, 2)
+        self.assertRaises(TypeError, ast.Num, 1, 2, lineno=0)
+
+    def test_module(self):
+        body = [ast.Num(42)]
+        x = ast.Module(body)
+        self.assertEquals(x.body, body)
+
+    def test_nodeclass(self):
+        x = ast.BinOp()
+        self.assertEquals(x._fields, ('left', 'op', 'right'))
+
+        # Zero arguments constructor explicitely allowed
+        x = ast.BinOp()
+        # Random attribute allowed too
+        x.foobarbaz = 5
+        self.assertEquals(x.foobarbaz, 5)
+
+        n1 = ast.Num(1)
+        n3 = ast.Num(3)
+        addop = ast.Add()
+        x = ast.BinOp(n1, addop, n3)
+        self.assertEquals(x.left, n1)
+        self.assertEquals(x.op, addop)
+        self.assertEquals(x.right, n3)
+        
+        x = ast.BinOp(1, 2, 3)
+        self.assertEquals(x.left, 1)
+        self.assertEquals(x.op, 2)
+        self.assertEquals(x.right, 3)
+
+        x = ast.BinOp(1, 2, 3, lineno=0)
+        self.assertEquals(x.lineno, 0)
+
+    def test_nodeclasses(self):
+        x = ast.BinOp(1, 2, 3, lineno=0)
+        self.assertEquals(x.left, 1)
+        self.assertEquals(x.op, 2)
+        self.assertEquals(x.right, 3)
+        self.assertEquals(x.lineno, 0)
+
     def test_nodeclasses(self):
         x = ast.BinOp(1, 2, 3, lineno=0)
         self.assertEqual(x.left, 1)
@@ -178,6 +368,12 @@
 
         # node raises exception when not given enough arguments
         self.assertRaises(TypeError, ast.BinOp, 1, 2)
+        # node raises exception when given too many arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4)
+        # node raises exception when not given enough arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, lineno=0)
+        # node raises exception when given too many arguments
+        self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4, lineno=0)
 
         # can set attributes through kwargs too
         x = ast.BinOp(left=1, op=2, right=3, lineno=0)
@@ -186,8 +382,14 @@
         self.assertEqual(x.right, 3)
         self.assertEqual(x.lineno, 0)
 
+        # Random kwargs also allowed
+        x = ast.BinOp(1, 2, 3, foobarbaz=42)
+        self.assertEquals(x.foobarbaz, 42)
+
+    def test_no_fields(self):
         # this used to fail because Sub._fields was None
         x = ast.Sub()
+        self.assertEquals(x._fields, ())
 
     def test_pickling(self):
         import pickle
@@ -330,8 +532,15 @@
 
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
+('Module', [('Expr', (1, 0), ('Name', (1, 0), 'None', ('Load',)))]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Pass', (1, 9))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',))], None, None, []), [('Pass', (1, 10))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',))], None, None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, []), [('Pass', (1, 14))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, 'kwargs', []), [('Pass', (1, 17))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('Name', (1, 6), 'a', ('Param',)), ('Name', (1, 9), 'b', ('Param',)), ('Name', (1, 14), 'c', ('Param',)), ('Name', (1, 22), 'd', ('Param',)), ('Name', (1, 28), 'e', ('Param',))], 'args', 'kwargs', [('Num', (1, 11), 1), ('Name', (1, 16), 'None', ('Load',)), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 52))], [])]),
 ('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [('Pass', (1, 17))], [])]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
 ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
 ('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
@@ -355,16 +564,26 @@
 ('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]),
 ('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]),
 ('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 12), [('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 20), 'c', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (2, 4), ('Tuple', (3, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), []), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))])]))]),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [])]))]),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))])]))]),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 1), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [])]))]),
 ]
 single_results = [
 ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
 ]
 eval_results = [
+('Expression', ('Name', (1, 0), 'None', ('Load',))),
 ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
 ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
 ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
 ('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, []), ('Name', (1, 7), 'None', ('Load',)))),
 ('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
+('Expression', ('Dict', (1, 0), [], [])),
+('Expression', ('Set', (1, 0), [('Name', (1, 1), 'None', ('Load',))])),
+('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
 ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
@@ -376,7 +595,10 @@
 ('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
 ('Expression', ('Name', (1, 0), 'v', ('Load',))),
 ('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('List', (1, 0), [], ('Load',))),
 ('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 1), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
+('Expression', ('Tuple', (1, 0), [], ('Load',))),
 ('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
 ]
 main()
diff --git a/lib-python/2.7/test/test_builtin.py b/lib-python/2.7/test/test_builtin.py
--- a/lib-python/2.7/test/test_builtin.py
+++ b/lib-python/2.7/test/test_builtin.py
@@ -3,7 +3,8 @@
 import platform
 import unittest
 from test.test_support import fcmp, have_unicode, TESTFN, unlink, \
-                              run_unittest, check_py3k_warnings
+                              run_unittest, check_py3k_warnings, \
+                              check_impl_detail
 import warnings
 from operator import neg
 
@@ -247,12 +248,14 @@
         self.assertRaises(TypeError, compile)
         self.assertRaises(ValueError, compile, 'print 42\n', '<string>', 'badmode')
         self.assertRaises(ValueError, compile, 'print 42\n', '<string>', 'single', 0xff)
-        self.assertRaises(TypeError, compile, chr(0), 'f', 'exec')
+        if check_impl_detail(cpython=True):
+            self.assertRaises(TypeError, compile, chr(0), 'f', 'exec')
         self.assertRaises(TypeError, compile, 'pass', '?', 'exec',
                           mode='eval', source='0', filename='tmp')
         if have_unicode:
             compile(unicode('print u"\xc3\xa5"\n', 'utf8'), '', 'exec')
-            self.assertRaises(TypeError, compile, unichr(0), 'f', 'exec')
+            if check_impl_detail(cpython=True):
+                self.assertRaises(TypeError, compile, unichr(0), 'f', 'exec')
             self.assertRaises(ValueError, compile, unicode('a = 1'), 'f', 'bad')
 
 
@@ -395,12 +398,16 @@
         self.assertEqual(eval('dir()', g, m), list('xyz'))
         self.assertEqual(eval('globals()', g, m), g)
         self.assertEqual(eval('locals()', g, m), m)
-        self.assertRaises(TypeError, eval, 'a', m)
+        # on top of CPython, the first dictionary (the globals) has to
+        # be a real dict.  This is not the case on top of PyPy.
+        if check_impl_detail(pypy=False):
+            self.assertRaises(TypeError, eval, 'a', m)
+
         class A:
             "Non-mapping"
             pass
         m = A()
-        self.assertRaises(TypeError, eval, 'a', g, m)
+        self.assertRaises((TypeError, AttributeError), eval, 'a', g, m)
 
         # Verify that dict subclasses work as well
         class D(dict):
@@ -491,9 +498,10 @@
         execfile(TESTFN, globals, locals)
         self.assertEqual(locals['z'], 2)
 
+        self.assertRaises(TypeError, execfile, TESTFN, {}, ())
         unlink(TESTFN)
         self.assertRaises(TypeError, execfile)
-        self.assertRaises(TypeError, execfile, TESTFN, {}, ())
+        self.assertRaises((TypeError, IOError), execfile, TESTFN, {}, ())
         import os
         self.assertRaises(IOError, execfile, os.curdir)
         self.assertRaises(IOError, execfile, "I_dont_exist")
@@ -1108,7 +1116,8 @@
             def __cmp__(self, other):
                 raise RuntimeError
             __hash__ = None # Invalid cmp makes this unhashable
-        self.assertRaises(RuntimeError, range, a, a + 1, badzero(1))
+        if check_impl_detail(cpython=True):
+            self.assertRaises(RuntimeError, range, a, a + 1, badzero(1))
 
         # Reject floats.
         self.assertRaises(TypeError, range, 1., 1., 1.)
diff --git a/lib-python/2.7/test/test_bytes.py b/lib-python/2.7/test/test_bytes.py
--- a/lib-python/2.7/test/test_bytes.py
+++ b/lib-python/2.7/test/test_bytes.py
@@ -694,6 +694,7 @@
         self.assertEqual(b, b1)
         self.assertTrue(b is b1)
 
+    @test.test_support.impl_detail("undocumented bytes.__alloc__()")
     def test_alloc(self):
         b = bytearray()
         alloc = b.__alloc__()
@@ -821,6 +822,8 @@
         self.assertEqual(b, b"")
         self.assertEqual(c, b"")
 
+    @test.test_support.impl_detail(
+        "resizing semantics of CPython rely on refcounting")
     def test_resize_forbidden(self):
         # #4509: can't resize a bytearray when there are buffer exports, even
         # if it wouldn't reallocate the underlying buffer.
@@ -853,6 +856,26 @@
         self.assertRaises(BufferError, delslice)
         self.assertEqual(b, orig)
 
+    @test.test_support.impl_detail("resizing semantics", cpython=False)
+    def test_resize_forbidden_non_cpython(self):
+        # on non-CPython implementations, we cannot prevent changes to
+        # bytearrays just because there are buffers around.  Instead,
+        # we get (on PyPy) a buffer that follows the changes and resizes.
+        b = bytearray(range(10))
+        for v in [memoryview(b), buffer(b)]:
+            b[5] = 99
+            self.assertIn(v[5], (99, chr(99)))
+            b[5] = 100
+            b += b
+            b += b
+            b += b
+            self.assertEquals(len(v), 80)
+            self.assertIn(v[5], (100, chr(100)))
+            self.assertIn(v[79], (9, chr(9)))
+            del b[10:]
+            self.assertRaises(IndexError, lambda: v[10])
+            self.assertEquals(len(v), 10)
+
     def test_empty_bytearray(self):
         # Issue #7561: operations on empty bytearrays could crash in many
         # situations, due to a fragile implementation of the
diff --git a/lib-python/2.7/test/test_bz2.py b/lib-python/2.7/test/test_bz2.py
--- a/lib-python/2.7/test/test_bz2.py
+++ b/lib-python/2.7/test/test_bz2.py
@@ -50,6 +50,7 @@
         self.filename = TESTFN
 
     def tearDown(self):
+        test_support.gc_collect()
         if os.path.isfile(self.filename):
             os.unlink(self.filename)
 
@@ -246,6 +247,8 @@
         for i in xrange(10000):
             o = BZ2File(self.filename)
             del o
+            if i % 100 == 0:
+                test_support.gc_collect()
 
     def testOpenNonexistent(self):
         # "Test opening a nonexistent file"
@@ -310,6 +313,7 @@
             for t in threads:
                 t.join()
 
+    @test_support.impl_detail()
     def testMixedIterationReads(self):
         # Issue #8397: mixed iteration and reads should be forbidden.
         with bz2.BZ2File(self.filename, 'wb') as f:
diff --git a/lib-python/2.7/test/test_cmd_line_script.py b/lib-python/2.7/test/test_cmd_line_script.py
--- a/lib-python/2.7/test/test_cmd_line_script.py
+++ b/lib-python/2.7/test/test_cmd_line_script.py
@@ -112,6 +112,8 @@
             self._check_script(script_dir, script_name, script_dir, '')
 
     def test_directory_compiled(self):
+        if test.test_support.check_impl_detail(pypy=True):
+            raise unittest.SkipTest("pypy won't load lone .pyc files")
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             compiled_name = compile_script(script_name)
@@ -173,6 +175,8 @@
                                script_name, 'test_pkg')
 
     def test_package_compiled(self):
+        if test.test_support.check_impl_detail(pypy=True):
+            raise unittest.SkipTest("pypy won't load lone .pyc files")
         with temp_dir() as script_dir:
             pkg_dir = os.path.join(script_dir, 'test_pkg')
             make_pkg(pkg_dir)
diff --git a/lib-python/2.7/test/test_code.py b/lib-python/2.7/test/test_code.py
--- a/lib-python/2.7/test/test_code.py
+++ b/lib-python/2.7/test/test_code.py
@@ -82,7 +82,7 @@
 
 import unittest
 import weakref
-import _testcapi
+from test import test_support
 
 
 def consts(t):
@@ -104,7 +104,9 @@
 
 class CodeTest(unittest.TestCase):
 
+    @test_support.impl_detail("test for PyCode_NewEmpty")
     def test_newempty(self):
+        import _testcapi
         co = _testcapi.code_newempty("filename", "funcname", 15)
         self.assertEqual(co.co_filename, "filename")
         self.assertEqual(co.co_name, "funcname")
@@ -132,6 +134,7 @@
         coderef = weakref.ref(f.__code__, callback)
         self.assertTrue(bool(coderef()))
         del f
+        test_support.gc_collect()
         self.assertFalse(bool(coderef()))
         self.assertTrue(self.called)
 
diff --git a/lib-python/2.7/test/test_codeop.py b/lib-python/2.7/test/test_codeop.py
--- a/lib-python/2.7/test/test_codeop.py
+++ b/lib-python/2.7/test/test_codeop.py
@@ -3,7 +3,7 @@
    Nick Mathewson
 """
 import unittest
-from test.test_support import run_unittest, is_jython
+from test.test_support import run_unittest, is_jython, check_impl_detail
 
 from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT
 
@@ -270,7 +270,9 @@
         ai("a = 'a\\\n")
 
         ai("a = 1","eval")
-        ai("a = (","eval")
+        if check_impl_detail():   # on PyPy it asks for more data, which is not
+            ai("a = (","eval")    # completely correct but hard to fix and
+                                  # really a detail (in my opinion <arigo>)
         ai("]","eval")
         ai("())","eval")
         ai("[}","eval")
diff --git a/lib-python/2.7/test/test_coercion.py b/lib-python/2.7/test/test_coercion.py
--- a/lib-python/2.7/test/test_coercion.py
+++ b/lib-python/2.7/test/test_coercion.py
@@ -1,6 +1,7 @@
 import copy
 import unittest
-from test.test_support import run_unittest, TestFailed, check_warnings
+from test.test_support import (
+    run_unittest, TestFailed, check_warnings, check_impl_detail)
 
 
 # Fake a number that implements numeric methods through __coerce__
@@ -306,12 +307,18 @@
         self.assertNotEqual(cmp(u'fish', evil_coercer), 0)
         self.assertNotEqual(cmp(slice(1), evil_coercer), 0)
         # ...but that this still works
-        class WackyComparer(object):
-            def __cmp__(slf, other):
-                self.assertTrue(other == 42, 'expected evil_coercer, got %r' % other)
-                return 0
-            __hash__ = None # Invalid cmp makes this unhashable
-        self.assertEqual(cmp(WackyComparer(), evil_coercer), 0)
+        if check_impl_detail():
+            # NB. I (arigo) would consider the following as implementation-
+            # specific.  For example, in CPython, if we replace 42 with 42.0
+            # both below and in CoerceTo() above, then the test fails.  This
+            # hints that the behavior is really dependent on some obscure
+            # internal details.
+            class WackyComparer(object):
+                def __cmp__(slf, other):
+                    self.assertTrue(other == 42, 'expected evil_coercer, got %r' % other)
+                    return 0
+                __hash__ = None # Invalid cmp makes this unhashable
+            self.assertEqual(cmp(WackyComparer(), evil_coercer), 0)
         # ...and classic classes too, since that code path is a little different
         class ClassicWackyComparer:
             def __cmp__(slf, other):
diff --git a/lib-python/2.7/test/test_compile.py b/lib-python/2.7/test/test_compile.py
--- a/lib-python/2.7/test/test_compile.py
+++ b/lib-python/2.7/test/test_compile.py
@@ -3,6 +3,7 @@
 import _ast
 from test import test_support
 import textwrap
+from test.test_support import check_impl_detail
 
 class TestSpecifics(unittest.TestCase):
 
@@ -90,12 +91,13 @@
         self.assertEqual(m.results, ('z', g))
         exec 'z = locals()' in g, m
         self.assertEqual(m.results, ('z', m))
-        try:
-            exec 'z = b' in m
-        except TypeError:
-            pass
-        else:
-            self.fail('Did not validate globals as a real dict')
+        if check_impl_detail():
+            try:
+                exec 'z = b' in m
+            except TypeError:
+                pass
+            else:
+                self.fail('Did not validate globals as a real dict')
 
         class A:
             "Non-mapping"
diff --git a/lib-python/2.7/test/test_copy.py b/lib-python/2.7/test/test_copy.py
--- a/lib-python/2.7/test/test_copy.py
+++ b/lib-python/2.7/test/test_copy.py
@@ -637,6 +637,7 @@
         self.assertEqual(v[c], d)
         self.assertEqual(len(v), 2)
         del c, d
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
         x, y = C(), C()
         # The underlying containers are decoupled
@@ -666,6 +667,7 @@
         self.assertEqual(v[a].i, b.i)
         self.assertEqual(v[c].i, d.i)
         del c
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
 
     def test_deepcopy_weakvaluedict(self):
@@ -689,6 +691,7 @@
         self.assertTrue(t is d)
         del x, y, z, t
         del d
+        test_support.gc_collect()
         self.assertEqual(len(v), 1)
 
     def test_deepcopy_bound_method(self):
diff --git a/lib-python/2.7/test/test_cpickle.py b/lib-python/2.7/test/test_cpickle.py
--- a/lib-python/2.7/test/test_cpickle.py
+++ b/lib-python/2.7/test/test_cpickle.py
@@ -61,27 +61,27 @@
     error = cPickle.BadPickleGet
 
     def test_recursive_list(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_list,
                           self)
 
     def test_recursive_tuple(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_tuple,
                           self)
 
     def test_recursive_inst(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_inst,
                           self)
 
     def test_recursive_dict(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_dict,
                           self)
 
     def test_recursive_multi(self):
-        self.assertRaises(ValueError,
+        self.assertRaises((ValueError, RuntimeError),
                           AbstractPickleTests.test_recursive_multi,
                           self)
 
diff --git a/lib-python/2.7/test/test_csv.py b/lib-python/2.7/test/test_csv.py
--- a/lib-python/2.7/test/test_csv.py
+++ b/lib-python/2.7/test/test_csv.py
@@ -54,8 +54,10 @@
         self.assertEqual(obj.dialect.skipinitialspace, False)
         self.assertEqual(obj.dialect.strict, False)
         # Try deleting or changing attributes (they are read-only)
-        self.assertRaises(TypeError, delattr, obj.dialect, 'delimiter')
-        self.assertRaises(TypeError, setattr, obj.dialect, 'delimiter', ':')
+        self.assertRaises((TypeError, AttributeError), delattr, obj.dialect,
+                          'delimiter')
+        self.assertRaises((TypeError, AttributeError), setattr, obj.dialect,
+                          'delimiter', ':')
         self.assertRaises(AttributeError, delattr, obj.dialect, 'quoting')
         self.assertRaises(AttributeError, setattr, obj.dialect,
                           'quoting', None)
diff --git a/lib-python/2.7/test/test_deque.py b/lib-python/2.7/test/test_deque.py
--- a/lib-python/2.7/test/test_deque.py
+++ b/lib-python/2.7/test/test_deque.py
@@ -109,7 +109,7 @@
         self.assertEqual(deque('abc', maxlen=4).maxlen, 4)
         self.assertEqual(deque('abc', maxlen=2).maxlen, 2)
         self.assertEqual(deque('abc', maxlen=0).maxlen, 0)
-        with self.assertRaises(AttributeError):
+        with self.assertRaises((AttributeError, TypeError)):
             d = deque('abc')
             d.maxlen = 10
 
@@ -352,7 +352,10 @@
         for match in (True, False):
             d = deque(['ab'])
             d.extend([MutateCmp(d, match), 'c'])
-            self.assertRaises(IndexError, d.remove, 'c')
+            # On CPython we get IndexError: deque mutated during remove().
+            # Why is it an IndexError during remove() only???
+            # On PyPy it is a RuntimeError, as in the other operations.
+            self.assertRaises((IndexError, RuntimeError), d.remove, 'c')
             self.assertEqual(d, deque())
 
     def test_repr(self):
@@ -514,7 +517,7 @@
                 container = reversed(deque([obj, 1]))
             obj.x = iter(container)
             del obj, container
-            gc.collect()
+            test_support.gc_collect()
             self.assertTrue(ref() is None, "Cycle was not collected")
 
 class TestVariousIteratorArgs(unittest.TestCase):
@@ -630,6 +633,7 @@
         p = weakref.proxy(d)
         self.assertEqual(str(p), str(d))
         d = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, str, p)
 
     def test_strange_subclass(self):
diff --git a/lib-python/2.7/test/test_descr.py b/lib-python/2.7/test/test_descr.py
--- a/lib-python/2.7/test/test_descr.py
+++ b/lib-python/2.7/test/test_descr.py
@@ -2,6 +2,7 @@
 import sys
 import types
 import unittest
+import popen2     # trigger early the warning from popen2.py
 
 from copy import deepcopy
 from test import test_support
@@ -1128,7 +1129,7 @@
 
         # Test lookup leaks [SF bug 572567]
         import gc
-        if hasattr(gc, 'get_objects'):
+        if test_support.check_impl_detail():
             class G(object):
                 def __cmp__(self, other):
                     return 0
@@ -1741,6 +1742,10 @@
                 raise MyException
 
         for name, runner, meth_impl, ok, env in specials:
+            if name == '__length_hint__' or name == '__sizeof__':
+                if not test_support.check_impl_detail():
+                    continue
+
             class X(Checker):
                 pass
             for attr, obj in env.iteritems():
@@ -1980,7 +1985,9 @@
         except TypeError, msg:
             self.assertTrue(str(msg).find("weak reference") >= 0)
         else:
-            self.fail("weakref.ref(no) should be illegal")
+            if test_support.check_impl_detail(pypy=False):
+                self.fail("weakref.ref(no) should be illegal")
+            #else: pypy supports taking weakrefs to some more objects
         class Weak(object):
             __slots__ = ['foo', '__weakref__']
         yes = Weak()
@@ -3092,7 +3099,16 @@
         class R(J):
             __slots__ = ["__dict__", "__weakref__"]
 
-        for cls, cls2 in ((G, H), (G, I), (I, H), (Q, R), (R, Q)):
+        if test_support.check_impl_detail(pypy=False):
+            lst = ((G, H), (G, I), (I, H), (Q, R), (R, Q))
+        else:
+            # Not supported in pypy: changing the __class__ of an object
+            # to another __class__ that just happens to have the same slots.
+            # If needed, we can add the feature, but what we'll likely do
+            # then is to allow mostly any __class__ assignment, even if the
+            # classes have different __slots__, because we it's easier.
+            lst = ((Q, R), (R, Q))
+        for cls, cls2 in lst:
             x = cls()
             x.a = 1
             x.__class__ = cls2
@@ -3175,7 +3191,8 @@
             except TypeError:
                 pass
             else:
-                self.fail("%r's __dict__ can be modified" % cls)
+                if test_support.check_impl_detail(pypy=False):
+                    self.fail("%r's __dict__ can be modified" % cls)
 
         # Modules also disallow __dict__ assignment
         class Module1(types.ModuleType, Base):
@@ -4383,13 +4400,10 @@
         self.assertTrue(l.__add__ != [5].__add__)
         self.assertTrue(l.__add__ != l.__mul__)
         self.assertTrue(l.__add__.__name__ == '__add__')
-        if hasattr(l.__add__, '__self__'):
-            # CPython
-            self.assertTrue(l.__add__.__self__ is l)
+        self.assertTrue(l.__add__.__self__ is l)
+        if hasattr(l.__add__, '__objclass__'):   # CPython
             self.assertTrue(l.__add__.__objclass__ is list)
-        else:
-            # Python implementations where [].__add__ is a normal bound method
-            self.assertTrue(l.__add__.im_self is l)
+        else:                                    # PyPy
             self.assertTrue(l.__add__.im_class is list)
         self.assertEqual(l.__add__.__doc__, list.__add__.__doc__)
         try:
@@ -4578,8 +4592,12 @@
             str.split(fake_str)
 
         # call a slot wrapper descriptor
-        with self.assertRaises(TypeError):
-            str.__add__(fake_str, "abc")
+        try:
+            r = str.__add__(fake_str, "abc")
+        except TypeError:
+            pass
+        else:
+            self.assertEqual(r, NotImplemented)
 
 
 class DictProxyTests(unittest.TestCase):
diff --git a/lib-python/2.7/test/test_descrtut.py b/lib-python/2.7/test/test_descrtut.py
--- a/lib-python/2.7/test/test_descrtut.py
+++ b/lib-python/2.7/test/test_descrtut.py
@@ -172,46 +172,12 @@
     AttributeError: 'list' object has no attribute '__methods__'
     >>>
 
-Instead, you can get the same information from the list type:
+Instead, you can get the same information from the list type
+(the following example filters out the numerous method names
+starting with '_'):
 
-    >>> pprint.pprint(dir(list))    # like list.__dict__.keys(), but sorted
-    ['__add__',
-     '__class__',
-     '__contains__',
-     '__delattr__',
-     '__delitem__',
-     '__delslice__',
-     '__doc__',
-     '__eq__',
-     '__format__',
-     '__ge__',
-     '__getattribute__',
-     '__getitem__',
-     '__getslice__',
-     '__gt__',
-     '__hash__',
-     '__iadd__',
-     '__imul__',
-     '__init__',
-     '__iter__',
-     '__le__',
-     '__len__',
-     '__lt__',
-     '__mul__',
-     '__ne__',
-     '__new__',
-     '__reduce__',
-     '__reduce_ex__',
-     '__repr__',
-     '__reversed__',
-     '__rmul__',
-     '__setattr__',
-     '__setitem__',
-     '__setslice__',
-     '__sizeof__',
-     '__str__',
-     '__subclasshook__',
-     'append',
+    >>> pprint.pprint([name for name in dir(list) if not name.startswith('_')])
+    ['append',
      'count',
      'extend',
      'index',
diff --git a/lib-python/2.7/test/test_dict.py b/lib-python/2.7/test/test_dict.py
--- a/lib-python/2.7/test/test_dict.py
+++ b/lib-python/2.7/test/test_dict.py
@@ -319,7 +319,8 @@
                     self.assertEqual(va, int(ka))
                     kb, vb = tb = b.popitem()
                     self.assertEqual(vb, int(kb))
-                    self.assertFalse(copymode < 0 and ta != tb)
+                    if test_support.check_impl_detail():
+                        self.assertFalse(copymode < 0 and ta != tb)
                 self.assertFalse(a)
                 self.assertFalse(b)
 
diff --git a/lib-python/2.7/test/test_dis.py b/lib-python/2.7/test/test_dis.py
--- a/lib-python/2.7/test/test_dis.py
+++ b/lib-python/2.7/test/test_dis.py
@@ -56,8 +56,8 @@
  %-4d         0 LOAD_CONST               1 (0)
               3 POP_JUMP_IF_TRUE        38
               6 LOAD_GLOBAL              0 (AssertionError)
-              9 BUILD_LIST               0
-             12 LOAD_FAST                0 (x)
+              9 LOAD_FAST                0 (x)
+             12 BUILD_LIST_FROM_ARG      0
              15 GET_ITER
         >>   16 FOR_ITER                12 (to 31)
              19 STORE_FAST               1 (s)
diff --git a/lib-python/2.7/test/test_doctest.py b/lib-python/2.7/test/test_doctest.py
--- a/lib-python/2.7/test/test_doctest.py
+++ b/lib-python/2.7/test/test_doctest.py
@@ -782,7 +782,7 @@
     ...     >>> x = 12
     ...     >>> print x//0
     ...     Traceback (most recent call last):
-    ...     ZeroDivisionError: integer division or modulo by zero
+    ...     ZeroDivisionError: integer division by zero
     ...     '''
     >>> test = doctest.DocTestFinder().find(f)[0]
     >>> doctest.DocTestRunner(verbose=False).run(test)
@@ -799,7 +799,7 @@
     ...     >>> print 'pre-exception output', x//0
     ...     pre-exception output
     ...     Traceback (most recent call last):
-    ...     ZeroDivisionError: integer division or modulo by zero
+    ...     ZeroDivisionError: integer division by zero
     ...     '''
     >>> test = doctest.DocTestFinder().find(f)[0]
     >>> doctest.DocTestRunner(verbose=False).run(test)
@@ -810,7 +810,7 @@
         print 'pre-exception output', x//0
     Exception raised:
         ...
-        ZeroDivisionError: integer division or modulo by zero
+        ZeroDivisionError: integer division by zero
     TestResults(failed=1, attempted=2)
 
 Exception messages may contain newlines:
@@ -978,7 +978,7 @@
     Exception raised:
         Traceback (most recent call last):
         ...
-        ZeroDivisionError: integer division or modulo by zero
+        ZeroDivisionError: integer division by zero
     TestResults(failed=1, attempted=1)
 """
     def displayhook(): r"""
@@ -1924,7 +1924,7 @@
     > <doctest foo-b&#228;r at baz[1]>(1)<module>()
     -> calls_set_trace()
     (Pdb) print foo
-    *** NameError: name 'foo' is not defined
+    *** NameError: global name 'foo' is not defined
     (Pdb) continue
     TestResults(failed=0, attempted=2)
 """
@@ -2229,7 +2229,7 @@
           favorite_color
       Exception raised:
           ...
-          NameError: name 'favorite_color' is not defined
+          NameError: global name 'favorite_color' is not defined
       <BLANKLINE>
       <BLANKLINE>
 
@@ -2289,7 +2289,7 @@
         favorite_color
     Exception raised:
         ...
-        NameError: name 'favorite_color' is not defined
+        NameError: global name 'favorite_color' is not defined
     **********************************************************************
     1 items had failures:
        1 of   2 in test_doctest.txt
@@ -2382,7 +2382,7 @@
         favorite_color
     Exception raised:
         ...
-        NameError: name 'favorite_color' is not defined
+        NameError: global name 'favorite_color' is not defined
     TestResults(failed=1, attempted=2)
     >>> doctest.master = None  # Reset master.
 
diff --git a/lib-python/2.7/test/test_dumbdbm.py b/lib-python/2.7/test/test_dumbdbm.py
--- a/lib-python/2.7/test/test_dumbdbm.py
+++ b/lib-python/2.7/test/test_dumbdbm.py
@@ -107,9 +107,11 @@
         f.close()
 
         # Mangle the file by adding \r before each newline
-        data = open(_fname + '.dir').read()
+        with open(_fname + '.dir') as f:
+            data = f.read()
         data = data.replace('\n', '\r\n')
-        open(_fname + '.dir', 'wb').write(data)
+        with open(_fname + '.dir', 'wb') as f:
+            f.write(data)
 
         f = dumbdbm.open(_fname)
         self.assertEqual(f['1'], 'hello')
diff --git a/lib-python/2.7/test/test_extcall.py b/lib-python/2.7/test/test_extcall.py
--- a/lib-python/2.7/test/test_extcall.py
+++ b/lib-python/2.7/test/test_extcall.py
@@ -90,19 +90,19 @@
 
     >>> class Nothing: pass
     ...
-    >>> g(*Nothing())
+    >>> g(*Nothing())                     #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: g() argument after * must be a sequence, not instance
+    TypeError: ...argument after * must be a sequence, not instance
 
     >>> class Nothing:
     ...     def __len__(self): return 5
     ...
 
-    >>> g(*Nothing())
+    >>> g(*Nothing())                     #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: g() argument after * must be a sequence, not instance
+    TypeError: ...argument after * must be a sequence, not instance
 
     >>> class Nothing():
     ...     def __len__(self): return 5
@@ -154,52 +154,50 @@
       ...
     TypeError: g() got multiple values for keyword argument 'x'
 
-    >>> f(**{1:2})
+    >>> f(**{1:2})                             #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: f() keywords must be strings
+    TypeError: ...keywords must be strings
 
     >>> h(**{'e': 2})
     Traceback (most recent call last):
       ...
     TypeError: h() got an unexpected keyword argument 'e'
 
-    >>> h(*h)
+    >>> h(*h)                                  #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after * must be a sequence, not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> dir(*h)
+    >>> dir(*h)                                #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() argument after * must be a sequence, not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> None(*h)
+    >>> None(*h)                               #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: NoneType object argument after * must be a sequence, \
-not function
+    TypeError: ...argument after * must be a sequence, not function
 
-    >>> h(**h)
+    >>> h(**h)                                 #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: h() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> dir(**h)
+    >>> dir(**h)                               #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() argument after ** must be a mapping, not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> None(**h)
+    >>> None(**h)                              #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: NoneType object argument after ** must be a mapping, \
-not function
+    TypeError: ...argument after ** must be a mapping, not function
 
-    >>> dir(b=1, **{'b': 1})
+    >>> dir(b=1, **{'b': 1})                   #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: dir() got multiple values for keyword argument 'b'
+    TypeError: ...got multiple values for keyword argument 'b'
 
 Another helper function
 
@@ -247,10 +245,10 @@
     ...     False
     True
 
-    >>> id(1, **{'foo': 1})
+    >>> id(1, **{'foo': 1})                 #doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: id() takes no keyword arguments
+    TypeError: id() ... keyword argument...
 
 A corner case of keyword dictionary items being deleted during
 the function call setup. See <http://bugs.python.org/issue2016>.
diff --git a/lib-python/2.7/test/test_fcntl.py b/lib-python/2.7/test/test_fcntl.py
--- a/lib-python/2.7/test/test_fcntl.py
+++ b/lib-python/2.7/test/test_fcntl.py
@@ -32,7 +32,7 @@
                         'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5',
                         'freebsd6', 'freebsd7', 'freebsd8',
                         'bsdos2', 'bsdos3', 'bsdos4',
-                        'openbsd', 'openbsd2', 'openbsd3', 'openbsd4'):
+                        'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', 'openbsd5'):
         if struct.calcsize('l') == 8:
             off_t = 'l'
             pid_t = 'i'
diff --git a/lib-python/2.7/test/test_file.py b/lib-python/2.7/test/test_file.py
--- a/lib-python/2.7/test/test_file.py
+++ b/lib-python/2.7/test/test_file.py
@@ -12,7 +12,7 @@
 import io
 import _pyio as pyio
 
-from test.test_support import TESTFN, run_unittest
+from test.test_support import TESTFN, run_unittest, gc_collect
 from UserList import UserList
 
 class AutoFileTests(unittest.TestCase):
@@ -33,6 +33,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testAttributes(self):
@@ -157,7 +158,12 @@
     def testStdin(self):
         # This causes the interpreter to exit on OSF1 v5.1.
         if sys.platform != 'osf1V5':
-            self.assertRaises((IOError, ValueError), sys.stdin.seek, -1)
+            if sys.stdin.isatty():
+                self.assertRaises((IOError, ValueError), sys.stdin.seek, -1)
+            else:
+                print((
+                    '  Skipping sys.stdin.seek(-1): stdin is not a tty.'
+                    ' Test manually.'), file=sys.__stdout__)
         else:
             print((
                 '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
diff --git a/lib-python/2.7/test/test_file2k.py b/lib-python/2.7/test/test_file2k.py
--- a/lib-python/2.7/test/test_file2k.py
+++ b/lib-python/2.7/test/test_file2k.py
@@ -11,7 +11,7 @@
     threading = None
 
 from test import test_support
-from test.test_support import TESTFN, run_unittest
+from test.test_support import TESTFN, run_unittest, gc_collect
 from UserList import UserList
 
 class AutoFileTests(unittest.TestCase):
@@ -32,6 +32,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testAttributes(self):
@@ -116,8 +117,12 @@
 
         for methodname in methods:
             method = getattr(self.f, methodname)
+            args = {'readinto': (bytearray(''),),
+                    'seek':     (0,),
+                    'write':    ('',),
+                    }.get(methodname, ())
             # should raise on closed file
-            self.assertRaises(ValueError, method)
+            self.assertRaises(ValueError, method, *args)
         with test_support.check_py3k_warnings():
             for methodname in deprecated_methods:
                 method = getattr(self.f, methodname)
@@ -216,7 +221,12 @@
     def testStdin(self):
         # This causes the interpreter to exit on OSF1 v5.1.
         if sys.platform != 'osf1V5':
-            self.assertRaises(IOError, sys.stdin.seek, -1)
+            if sys.stdin.isatty():
+                self.assertRaises(IOError, sys.stdin.seek, -1)
+            else:
+                print >>sys.__stdout__, (
+                    '  Skipping sys.stdin.seek(-1): stdin is not a tty.'
+                    ' Test manualy.')
         else:
             print >>sys.__stdout__, (
                 '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
@@ -336,8 +346,9 @@
                 except ValueError:
                     pass
                 else:
-                    self.fail("%s%r after next() didn't raise ValueError" %
-                                     (methodname, args))
+                    if test_support.check_impl_detail():
+                        self.fail("%s%r after next() didn't raise ValueError" %
+                                  (methodname, args))
                 f.close()
 
             # Test to see if harmless (by accident) mixing of read* and
@@ -388,6 +399,7 @@
             if lines != testlines:
                 self.fail("readlines() after next() with empty buffer "
                           "failed. Got %r, expected %r" % (line, testline))
+            f.close()
             # Reading after iteration hit EOF shouldn't hurt either
             f = open(TESTFN)
             try:
@@ -438,6 +450,9 @@
         self.close_count = 0
         self.close_success_count = 0
         self.use_buffering = False
+        # to prevent running out of file descriptors on PyPy,
+        # we only keep the 50 most recent files open
+        self.all_files = [None] * 50
 
     def tearDown(self):
         if self.f:
@@ -453,9 +468,14 @@
 
     def _create_file(self):
         if self.use_buffering:
-            self.f = open(self.filename, "w+", buffering=1024*16)
+            f = open(self.filename, "w+", buffering=1024*16)
         else:
-            self.f = open(self.filename, "w+")
+            f = open(self.filename, "w+")
+        self.f = f
+        self.all_files.append(f)
+        oldf = self.all_files.pop(0)
+        if oldf is not None:
+            oldf.close()
 
     def _close_file(self):
         with self._count_lock:
@@ -496,7 +516,6 @@
 
     def _test_close_open_io(self, io_func, nb_workers=5):
         def worker():
-            self._create_file()
             funcs = itertools.cycle((
                 lambda: io_func(),
                 lambda: self._close_and_reopen_file(),
@@ -508,7 +527,11 @@
                     f()
                 except (IOError, ValueError):
                     pass
+        self._create_file()
         self._run_workers(worker, nb_workers)
+        # make sure that all files can be closed now
+        del self.all_files
+        gc_collect()
         if test_support.verbose:
             # Useful verbose statistics when tuning this test to take
             # less time to run but still ensuring that its still useful.
diff --git a/lib-python/2.7/test/test_fileio.py b/lib-python/2.7/test/test_fileio.py
--- a/lib-python/2.7/test/test_fileio.py
+++ b/lib-python/2.7/test/test_fileio.py
@@ -12,6 +12,7 @@
 
 from test.test_support import TESTFN, check_warnings, run_unittest, make_bad_fd
 from test.test_support import py3k_bytes as bytes
+from test.test_support import gc_collect
 from test.script_helper import run_python
 
 from _io import FileIO as _FileIO
@@ -34,6 +35,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testSeekTell(self):
@@ -104,8 +106,8 @@
         self.assertTrue(f.closed)
 
     def testMethods(self):
-        methods = ['fileno', 'isatty', 'read', 'readinto',
-                   'seek', 'tell', 'truncate', 'write', 'seekable',
+        methods = ['fileno', 'isatty', 'read',
+                   'tell', 'truncate', 'seekable',
                    'readable', 'writable']
         if sys.platform.startswith('atheos'):
             methods.remove('truncate')
@@ -117,6 +119,10 @@
             method = getattr(self.f, methodname)
             # should raise on closed file
             self.assertRaises(ValueError, method)
+        # methods with one argument
+        self.assertRaises(ValueError, self.f.readinto, 0)
+        self.assertRaises(ValueError, self.f.write, 0)
+        self.assertRaises(ValueError, self.f.seek, 0)
 
     def testOpendir(self):
         # Issue 3703: opening a directory should fill the errno
diff --git a/lib-python/2.7/test/test_format.py b/lib-python/2.7/test/test_format.py
--- a/lib-python/2.7/test/test_format.py
+++ b/lib-python/2.7/test/test_format.py
@@ -242,7 +242,7 @@
             try:
                 testformat(formatstr, args)
             except exception, exc:
-                if str(exc) == excmsg:
+                if str(exc) == excmsg or not test_support.check_impl_detail():
                     if verbose:
                         print "yes"
                 else:
@@ -272,13 +272,16 @@
         test_exc(u'no format', u'1', TypeError,
                  "not all arguments converted during string formatting")
 
-        class Foobar(long):
-            def __oct__(self):
-                # Returning a non-string should not blow up.
-                return self + 1
-
-        test_exc('%o', Foobar(), TypeError,
-                 "expected string or Unicode object, long found")
+        if test_support.check_impl_detail():
+            # __oct__() is called if Foobar inherits from 'long', but
+            # not, say, 'object' or 'int' or 'str'.  This seems strange
+            # enough to consider it a complete implementation detail.
+            class Foobar(long):
+                def __oct__(self):
+                    # Returning a non-string should not blow up.
+                    return self + 1
+            test_exc('%o', Foobar(), TypeError,
+                     "expected string or Unicode object, long found")
 
         if maxsize == 2**31-1:
             # crashes 2.2.1 and earlier:
diff --git a/lib-python/2.7/test/test_funcattrs.py b/lib-python/2.7/test/test_funcattrs.py
--- a/lib-python/2.7/test/test_funcattrs.py
+++ b/lib-python/2.7/test/test_funcattrs.py
@@ -14,6 +14,8 @@
         self.b = b
 
     def cannot_set_attr(self, obj, name, value, exceptions):
+        if not test_support.check_impl_detail():
+            exceptions = (TypeError, AttributeError)
         # Helper method for other tests.
         try:
             setattr(obj, name, value)
@@ -286,13 +288,13 @@
     def test_delete_func_dict(self):
         try:
             del self.b.__dict__
-        except TypeError:
+        except (AttributeError, TypeError):
             pass
         else:
             self.fail("deleting function dictionary should raise TypeError")
         try:
             del self.b.func_dict
-        except TypeError:
+        except (AttributeError, TypeError):
             pass
         else:
             self.fail("deleting function dictionary should raise TypeError")
diff --git a/lib-python/2.7/test/test_functools.py b/lib-python/2.7/test/test_functools.py
--- a/lib-python/2.7/test/test_functools.py
+++ b/lib-python/2.7/test/test_functools.py
@@ -45,6 +45,8 @@
         # attributes should not be writable
         if not isinstance(self.thetype, type):
             return
+        if not test_support.check_impl_detail():
+            return
         self.assertRaises(TypeError, setattr, p, 'func', map)
         self.assertRaises(TypeError, setattr, p, 'args', (1, 2))
         self.assertRaises(TypeError, setattr, p, 'keywords', dict(a=1, b=2))
@@ -136,6 +138,7 @@
         p = proxy(f)
         self.assertEqual(f.func, p.func)
         f = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'func')
 
     def test_with_bound_and_unbound_methods(self):
@@ -172,7 +175,7 @@
                       updated=functools.WRAPPER_UPDATES):
         # Check attributes were assigned
         for name in assigned:
-            self.assertTrue(getattr(wrapper, name) is getattr(wrapped, name))
+            self.assertTrue(getattr(wrapper, name) == getattr(wrapped, name), name)
         # Check attributes were updated
         for name in updated:
             wrapper_attr = getattr(wrapper, name)
diff --git a/lib-python/2.7/test/test_generators.py b/lib-python/2.7/test/test_generators.py
--- a/lib-python/2.7/test/test_generators.py
+++ b/lib-python/2.7/test/test_generators.py
@@ -190,7 +190,7 @@
       File "<stdin>", line 1, in ?
       File "<stdin>", line 2, in g
       File "<stdin>", line 2, in f
-    ZeroDivisionError: integer division or modulo by zero
+    ZeroDivisionError: integer division by zero
     >>> k.next()  # and the generator cannot be resumed
     Traceback (most recent call last):
       File "<stdin>", line 1, in ?
@@ -733,14 +733,16 @@
 ...     yield 1
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[0]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[0]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 >>> def f():
 ...     yield 1
 ...     return 22
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[1]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[1]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 "return None" is not the same as "return" in a generator:
 
@@ -749,7 +751,8 @@
 ...     return None
 Traceback (most recent call last):
   ..
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[2]>, line 3)
+  File "<doctest test.test_generators.__test__.syntax[2]>", line 3
+SyntaxError: 'return' with argument inside generator
 
 These are fine:
 
@@ -878,7 +881,9 @@
 ...     if 0:
 ...         yield 2             # because it's a generator (line 10)
 Traceback (most recent call last):
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.syntax[24]>, line 10)
+  ...
+  File "<doctest test.test_generators.__test__.syntax[24]>", line 10
+SyntaxError: 'return' with argument inside generator
 
 This one caused a crash (see SF bug 567538):
 
@@ -1496,6 +1501,10 @@
 """
 
 coroutine_tests = """\
+A helper function to call gc.collect() without printing
+>>> import gc
+>>> def gc_collect(): gc.collect()
+
 Sending a value into a started generator:
 
 >>> def f():
@@ -1570,13 +1579,14 @@
 >>> def f(): return lambda x=(yield): 1
 Traceback (most recent call last):
   ...
-SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.coroutine[22]>, line 1)
+  File "<doctest test.test_generators.__test__.coroutine[22]>", line 1
+SyntaxError: 'return' with argument inside generator
 
 >>> def f(): x = yield = y
 Traceback (most recent call last):
   ...
   File "<doctest test.test_generators.__test__.coroutine[23]>", line 1
-SyntaxError: assignment to yield expression not possible
+SyntaxError: can't assign to yield expression
 
 >>> def f(): (yield bar) = y
 Traceback (most recent call last):
@@ -1665,7 +1675,7 @@
 >>> f().throw("abc")     # throw on just-opened generator
 Traceback (most recent call last):
   ...
-TypeError: exceptions must be classes, or instances, not str
+TypeError: exceptions must be old-style classes or derived from BaseException, not str
 
 Now let's try closing a generator:
 
@@ -1697,7 +1707,7 @@
 
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 exiting
 
 >>> class context(object):
@@ -1708,7 +1718,7 @@
 ...          yield
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 exiting
 
 
@@ -1721,7 +1731,7 @@
 
 >>> g = f()
 >>> g.next()
->>> del g
+>>> del g; gc_collect()
 finally
 
 
@@ -1747,6 +1757,7 @@
 >>> g = f()
 >>> g.next()
 >>> del g
+>>> gc_collect()
 >>> sys.stderr.getvalue().startswith(
 ...     "Exception RuntimeError: 'generator ignored GeneratorExit' in "
 ... )
@@ -1812,6 +1823,9 @@
 references. We add it to the standard suite so the routine refleak-tests
 would trigger if it starts being uncleanable again.
 
+>>> import gc
+>>> def gc_collect(): gc.collect()
+
 >>> import itertools
 >>> def leak():
 ...     class gen:
@@ -1863,9 +1877,10 @@
 ...
 ...     l = Leaker()
 ...     del l
+...     gc_collect()
 ...     err = sys.stderr.getvalue().strip()
 ...     err.startswith(
-...         "Exception RuntimeError: RuntimeError() in <"
+...         "Exception RuntimeError: RuntimeError() in "
 ...     )
 ...     err.endswith("> ignored")
 ...     len(err.splitlines())
diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py
--- a/lib-python/2.7/test/test_genexps.py
+++ b/lib-python/2.7/test/test_genexps.py
@@ -128,8 +128,9 @@
 
 Verify re-use of tuples (a side benefit of using genexps over listcomps)
 
+    >>> from test.test_support import check_impl_detail
     >>> tupleids = map(id, ((i,i) for i in xrange(10)))
-    >>> int(max(tupleids) - min(tupleids))
+    >>> int(max(tupleids) - min(tupleids)) if check_impl_detail() else 0
     0
 
 Verify that syntax error's are raised for genexps used as lvalues
@@ -198,13 +199,13 @@
     >>> g = (10 // i for i in (5, 0, 2))
     >>> g.next()
     2
-    >>> g.next()
+    >>> g.next()       # doctest: +ELLIPSIS
     Traceback (most recent call last):
       File "<pyshell#37>", line 1, in -toplevel-
         g.next()
       File "<pyshell#35>", line 1, in <generator expression>
         g = (10 // i for i in (5, 0, 2))
-    ZeroDivisionError: integer division or modulo by zero
+    ZeroDivisionError: integer division...by zero
     >>> g.next()
     Traceback (most recent call last):
       File "<pyshell#38>", line 1, in -toplevel-
diff --git a/lib-python/2.7/test/test_heapq.py b/lib-python/2.7/test/test_heapq.py
--- a/lib-python/2.7/test/test_heapq.py
+++ b/lib-python/2.7/test/test_heapq.py
@@ -215,6 +215,11 @@
 class TestHeapPython(TestHeap):
     module = py_heapq
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 @skipUnless(c_heapq, 'requires _heapq')
 class TestHeapC(TestHeap):
diff --git a/lib-python/2.7/test/test_import.py b/lib-python/2.7/test/test_import.py
--- a/lib-python/2.7/test/test_import.py
+++ b/lib-python/2.7/test/test_import.py
@@ -7,7 +7,8 @@
 import sys
 import unittest
 from test.test_support import (unlink, TESTFN, unload, run_unittest, rmtree,
-                               is_jython, check_warnings, EnvironmentVarGuard)
+                               is_jython, check_warnings, EnvironmentVarGuard,
+                               impl_detail, check_impl_detail)
 import textwrap
 from test import script_helper
 
@@ -69,7 +70,8 @@
                 self.assertEqual(mod.b, b,
                     "module loaded (%s) but contents invalid" % mod)
             finally:
-                unlink(source)
+                if check_impl_detail(pypy=False):
+                    unlink(source)
 
             try:
                 imp.reload(mod)
@@ -149,13 +151,16 @@
         # Compile & remove .py file, we only need .pyc (or .pyo).
         with open(filename, 'r') as f:
             py_compile.compile(filename)
-        unlink(filename)
+        if check_impl_detail(pypy=False):
+            # pypy refuses to import a .pyc if the .py does not exist
+            unlink(filename)
 
         # Need to be able to load from current dir.
         sys.path.append('')
 
         # This used to crash.
         exec 'import ' + module
+        reload(longlist)
 
         # Cleanup.
         del sys.path[-1]
@@ -326,6 +331,7 @@
         self.assertEqual(mod.code_filename, self.file_name)
         self.assertEqual(mod.func_filename, self.file_name)
 
+    @impl_detail("pypy refuses to import without a .py source", pypy=False)
     def test_module_without_source(self):
         target = "another_module.py"
         py_compile.compile(self.file_name, dfile=target)
diff --git a/lib-python/2.7/test/test_inspect.py b/lib-python/2.7/test/test_inspect.py
--- a/lib-python/2.7/test/test_inspect.py
+++ b/lib-python/2.7/test/test_inspect.py
@@ -4,11 +4,11 @@
 import unittest
 import inspect
 import linecache
-import datetime
 from UserList import UserList
 from UserDict import UserDict
 
 from test.test_support import run_unittest, check_py3k_warnings
+from test.test_support import check_impl_detail
 
 with check_py3k_warnings(
         ("tuple parameter unpacking has been removed", SyntaxWarning),
@@ -74,7 +74,8 @@
 
     def test_excluding_predicates(self):
         self.istest(inspect.isbuiltin, 'sys.exit')
-        self.istest(inspect.isbuiltin, '[].append')
+        if check_impl_detail():
+            self.istest(inspect.isbuiltin, '[].append')
         self.istest(inspect.iscode, 'mod.spam.func_code')
         self.istest(inspect.isframe, 'tb.tb_frame')
         self.istest(inspect.isfunction, 'mod.spam')
@@ -92,9 +93,9 @@
         else:
             self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals))
         if hasattr(types, 'MemberDescriptorType'):
-            self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days')
+            self.istest(inspect.ismemberdescriptor, 'type(lambda: None).func_globals')
         else:
-            self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days))
+            self.assertFalse(inspect.ismemberdescriptor(type(lambda: None).func_globals))
 
     def test_isroutine(self):
         self.assertTrue(inspect.isroutine(mod.spam))
@@ -567,7 +568,8 @@
         else:
             self.fail('Exception not raised')
         self.assertIs(type(ex1), type(ex2))
-        self.assertEqual(str(ex1), str(ex2))
+        if check_impl_detail():
+            self.assertEqual(str(ex1), str(ex2))
 
     def makeCallable(self, signature):
         """Create a function that returns its locals(), excluding the
diff --git a/lib-python/2.7/test/test_int.py b/lib-python/2.7/test/test_int.py
--- a/lib-python/2.7/test/test_int.py
+++ b/lib-python/2.7/test/test_int.py
@@ -1,7 +1,7 @@
 import sys
 
 import unittest
-from test.test_support import run_unittest, have_unicode
+from test.test_support import run_unittest, have_unicode, check_impl_detail
 import math
 
 L = [
@@ -392,9 +392,10 @@
                 try:
                     int(TruncReturnsNonIntegral())
                 except TypeError as e:
-                    self.assertEqual(str(e),
-                                      "__trunc__ returned non-Integral"
-                                      " (type NonIntegral)")
+                    if check_impl_detail(cpython=True):
+                        self.assertEqual(str(e),
+                                          "__trunc__ returned non-Integral"
+                                          " (type NonIntegral)")
                 else:
                     self.fail("Failed to raise TypeError with %s" %
                               ((base, trunc_result_base),))
diff --git a/lib-python/2.7/test/test_io.py b/lib-python/2.7/test/test_io.py
--- a/lib-python/2.7/test/test_io.py
+++ b/lib-python/2.7/test/test_io.py
@@ -2561,6 +2561,31 @@
         """Check that a partial write, when it gets interrupted, properly
         invokes the signal handler, and bubbles up the exception raised
         in the latter."""
+
+        # XXX This test has three flaws that appear when objects are
+        # XXX not reference counted.
+
+        # - if wio.write() happens to trigger a garbage collection,
+        #   the signal exception may be raised when some __del__
+        #   method is running; it will not reach the assertRaises()
+        #   call.
+
+        # - more subtle, if the wio object is not destroyed at once
+        #   and survives this function, the next opened file is likely
+        #   to have the same fileno (since the file descriptor was
+        #   actively closed).  When wio.__del__ is finally called, it
+        #   will close the other's test file...  To trigger this with
+        #   CPython, try adding "global wio" in this function.
+
+        # - This happens only for streams created by the _pyio module,
+        #   because a wio.close() that fails still consider that the
+        #   file needs to be closed again.  You can try adding an
+        #   "assert wio.closed" at the end of the function.
+
+        # Fortunately, a little gc.gollect() seems to be enough to
+        # work around all these issues.
+        support.gc_collect()
+
         read_results = []
         def _read():
             s = os.read(r, 1)
diff --git a/lib-python/2.7/test/test_isinstance.py b/lib-python/2.7/test/test_isinstance.py
--- a/lib-python/2.7/test/test_isinstance.py
+++ b/lib-python/2.7/test/test_isinstance.py
@@ -260,7 +260,18 @@
     # Make sure that calling isinstance with a deeply nested tuple for its
     # argument will raise RuntimeError eventually.
     tuple_arg = (compare_to,)
-    for cnt in xrange(sys.getrecursionlimit()+5):
+
+
+    if test_support.check_impl_detail(cpython=True):
+        RECURSION_LIMIT = sys.getrecursionlimit()
+    else:
+        # on non-CPython implementations, the maximum
+        # actual recursion limit might be higher, but
+        # probably not higher than 99999
+        #
+        RECURSION_LIMIT = 99999
+
+    for cnt in xrange(RECURSION_LIMIT+5):
         tuple_arg = (tuple_arg,)
         fxn(arg, tuple_arg)
 
diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py
--- a/lib-python/2.7/test/test_itertools.py
+++ b/lib-python/2.7/test/test_itertools.py
@@ -137,6 +137,8 @@
                 self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version
                 self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_combinations_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1)
@@ -207,7 +209,10 @@
                 self.assertEqual(result, list(cwr1(values, r)))         # matches first pure python version
                 self.assertEqual(result, list(cwr2(values, r)))         # matches second pure python version
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_combinations_with_replacement_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
+        cwr = combinations_with_replacement
         self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1)
 
@@ -271,6 +276,8 @@
                     self.assertEqual(result, list(permutations(values, None))) # test r as None
                     self.assertEqual(result, list(permutations(values)))       # test default r
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_permutations_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1)
         self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1)
@@ -526,6 +533,9 @@
         self.assertEqual(list(izip()), zip())
         self.assertRaises(TypeError, izip, 3)
         self.assertRaises(TypeError, izip, range(3), 3)
+
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_izip_tuple_reuse(self):
         # Check tuple re-use (implementation detail)
         self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')],
                          zip('abc', 'def'))
@@ -575,6 +585,8 @@
             else:
                 self.fail('Did not raise Type in:  ' + stmt)
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_iziplongest_tuple_reuse(self):
         # Check tuple re-use (implementation detail)
         self.assertEqual([tuple(list(pair)) for pair in izip_longest('abc', 'def')],
                          zip('abc', 'def'))
@@ -683,6 +695,8 @@
             args = map(iter, args)
             self.assertEqual(len(list(product(*args))), expected_len)
 
+    @test_support.impl_detail("tuple reuse is specific to CPython")
+    def test_product_tuple_reuse(self):
         # Test implementation detail:  tuple re-use
         self.assertEqual(len(set(map(id, product('abc', 'def')))), 1)
         self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1)
@@ -771,11 +785,11 @@
         self.assertRaises(ValueError, islice, xrange(10), 1, -5, -1)
         self.assertRaises(ValueError, islice, xrange(10), 1, 10, -1)
         self.assertRaises(ValueError, islice, xrange(10), 1, 10, 0)
-        self.assertRaises(ValueError, islice, xrange(10), 'a')
-        self.assertRaises(ValueError, islice, xrange(10), 'a', 1)
-        self.assertRaises(ValueError, islice, xrange(10), 1, 'a')
-        self.assertRaises(ValueError, islice, xrange(10), 'a', 1, 1)
-        self.assertRaises(ValueError, islice, xrange(10), 1, 'a', 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a')
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a', 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 1, 'a')
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 'a', 1, 1)
+        self.assertRaises((ValueError, TypeError), islice, xrange(10), 1, 'a', 1)
         self.assertEqual(len(list(islice(count(), 1, 10, maxsize))), 1)
 
         # Issue #10323:  Less islice in a predictable state
@@ -855,9 +869,17 @@
         self.assertRaises(TypeError, tee, [1,2], 3, 'x')
 
         # tee object should be instantiable
-        a, b = tee('abc')
-        c = type(a)('def')
-        self.assertEqual(list(c), list('def'))
+        if test_support.check_impl_detail():
+            # XXX I (arigo) would argue that 'type(a)(iterable)' has
+            # ill-defined semantics: it always return a fresh tee object,
+            # but depending on whether 'iterable' is itself a tee object
+            # or not, it is ok or not to continue using 'iterable' after
+            # the call.  I cannot imagine why 'type(a)(non_tee_object)'
+            # would be useful, as 'iter(non_tee_obect)' is equivalent
+            # as far as I can see.
+            a, b = tee('abc')
+            c = type(a)('def')
+            self.assertEqual(list(c), list('def'))
 
         # test long-lagged and multi-way split
         a, b, c = tee(xrange(2000), 3)
@@ -895,6 +917,7 @@
         p = proxy(a)
         self.assertEqual(getattr(p, '__class__'), type(b))
         del a
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, '__class__')
 
     def test_StopIteration(self):
@@ -1317,6 +1340,7 @@
 
 class LengthTransparency(unittest.TestCase):
 
+    @test_support.impl_detail("__length_hint__() API is undocumented")
     def test_repeat(self):
         from test.test_iterlen import len
         self.assertEqual(len(repeat(None, 50)), 50)
diff --git a/lib-python/2.7/test/test_linecache.py b/lib-python/2.7/test/test_linecache.py
--- a/lib-python/2.7/test/test_linecache.py
+++ b/lib-python/2.7/test/test_linecache.py
@@ -54,13 +54,13 @@
 
         # Check whether lines correspond to those from file iteration
         for entry in TESTS:
-            filename = os.path.join(TEST_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             for index, line in enumerate(open(filename)):
                 self.assertEqual(line, getline(filename, index + 1))
 
         # Check module loading
         for entry in MODULES:
-            filename = os.path.join(MODULE_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             for index, line in enumerate(open(filename)):
                 self.assertEqual(line, getline(filename, index + 1))
 
@@ -78,7 +78,7 @@
     def test_clearcache(self):
         cached = []
         for entry in TESTS:
-            filename = os.path.join(TEST_PATH, entry) + '.py'
+            filename = support.findfile( entry + '.py')
             cached.append(filename)
             linecache.getline(filename, 1)
 
diff --git a/lib-python/2.7/test/test_list.py b/lib-python/2.7/test/test_list.py
--- a/lib-python/2.7/test/test_list.py
+++ b/lib-python/2.7/test/test_list.py
@@ -15,6 +15,10 @@
         self.assertEqual(list(''), [])
         self.assertEqual(list('spam'), ['s', 'p', 'a', 'm'])
 
+    # the following test also works with pypy, but eats all your address
+    # space's RAM before raising and takes too long.
+    @test_support.impl_detail("eats all your RAM before working", pypy=False)
+    def test_segfault_1(self):
         if sys.maxsize == 0x7fffffff:
             # This test can currently only work on 32-bit machines.
             # XXX If/when PySequence_Length() returns a ssize_t, it should be
@@ -32,6 +36,7 @@
             #     http://sources.redhat.com/ml/newlib/2002/msg00369.html
             self.assertRaises(MemoryError, list, xrange(sys.maxint // 2))
 
+    def test_segfault_2(self):
         # This code used to segfault in Py2.4a3
         x = []
         x.extend(-y for y in x)
diff --git a/lib-python/2.7/test/test_long.py b/lib-python/2.7/test/test_long.py
--- a/lib-python/2.7/test/test_long.py
+++ b/lib-python/2.7/test/test_long.py
@@ -530,9 +530,10 @@
                 try:
                     long(TruncReturnsNonIntegral())
                 except TypeError as e:
-                    self.assertEqual(str(e),
-                                     "__trunc__ returned non-Integral"
-                                     " (type NonIntegral)")
+                    if test_support.check_impl_detail(cpython=True):
+                        self.assertEqual(str(e),
+                                          "__trunc__ returned non-Integral"
+                                          " (type NonIntegral)")
                 else:
                     self.fail("Failed to raise TypeError with %s" %
                               ((base, trunc_result_base),))
diff --git a/lib-python/2.7/test/test_marshal.py b/lib-python/2.7/test/test_marshal.py
--- a/lib-python/2.7/test/test_marshal.py
+++ b/lib-python/2.7/test/test_marshal.py
@@ -7,20 +7,31 @@
 import unittest
 import os
 
-class IntTestCase(unittest.TestCase):
+class HelperMixin:
+    def helper(self, sample, *extra, **kwargs):
+        expected = kwargs.get('expected', sample)
+        new = marshal.loads(marshal.dumps(sample, *extra))
+        self.assertEqual(expected, new)
+        self.assertEqual(type(expected), type(new))
+        try:
+            with open(test_support.TESTFN, "wb") as f:
+                marshal.dump(sample, f, *extra)
+            with open(test_support.TESTFN, "rb") as f:
+                new = marshal.load(f)
+            self.assertEqual(expected, new)
+            self.assertEqual(type(expected), type(new))
+        finally:
+            test_support.unlink(test_support.TESTFN)
+
+
+class IntTestCase(unittest.TestCase, HelperMixin):
     def test_ints(self):
         # Test the full range of Python ints.
         n = sys.maxint
         while n:
             for expected in (-n, n):
-                s = marshal.dumps(expected)
-                got = marshal.loads(s)
-                self.assertEqual(expected, got)
-                marshal.dump(expected, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(expected, got)
+                self.helper(expected)
             n = n >> 1
-        os.unlink(test_support.TESTFN)
 
     def test_int64(self):
         # Simulate int marshaling on a 64-bit box.  This is most interesting if
@@ -48,28 +59,16 @@
 
     def test_bool(self):
         for b in (True, False):
-            new = marshal.loads(marshal.dumps(b))
-            self.assertEqual(b, new)
-            self.assertEqual(type(b), type(new))
-            marshal.dump(b, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(b, new)
-            self.assertEqual(type(b), type(new))
+            self.helper(b)
 
-class FloatTestCase(unittest.TestCase):
+class FloatTestCase(unittest.TestCase, HelperMixin):
     def test_floats(self):
         # Test a few floats
         small = 1e-25
         n = sys.maxint * 3.7e250
         while n > small:
             for expected in (-n, n):
-                f = float(expected)
-                s = marshal.dumps(f)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-                marshal.dump(f, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
+                self.helper(expected)
             n /= 123.4567
 
         f = 0.0
@@ -85,59 +84,25 @@
         while n < small:
             for expected in (-n, n):
                 f = float(expected)
+                self.helper(f)
+                self.helper(f, 1)
+            n *= 123.4567
 
-                s = marshal.dumps(f)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-
-                s = marshal.dumps(f, 1)
-                got = marshal.loads(s)
-                self.assertEqual(f, got)
-
-                marshal.dump(f, file(test_support.TESTFN, "wb"))
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
-
-                marshal.dump(f, file(test_support.TESTFN, "wb"), 1)
-                got = marshal.load(file(test_support.TESTFN, "rb"))
-                self.assertEqual(f, got)
-            n *= 123.4567
-        os.unlink(test_support.TESTFN)
-
-class StringTestCase(unittest.TestCase):
+class StringTestCase(unittest.TestCase, HelperMixin):
     def test_unicode(self):
         for s in [u"", u"Andr&#65533; Previn", u"abc", u" "*10000]:
-            new = marshal.loads(marshal.dumps(s))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-            marshal.dump(s, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-        os.unlink(test_support.TESTFN)
+            self.helper(s)
 
     def test_string(self):
         for s in ["", "Andr&#65533; Previn", "abc", " "*10000]:
-            new = marshal.loads(marshal.dumps(s))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-            marshal.dump(s, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-            self.assertEqual(type(s), type(new))
-        os.unlink(test_support.TESTFN)
+            self.helper(s)
 
     def test_buffer(self):
         for s in ["", "Andr&#65533; Previn", "abc", " "*10000]:
             with test_support.check_py3k_warnings(("buffer.. not supported",
                                                      DeprecationWarning)):
                 b = buffer(s)
-            new = marshal.loads(marshal.dumps(b))
-            self.assertEqual(s, new)
-            marshal.dump(b, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(s, new)
-        os.unlink(test_support.TESTFN)
+            self.helper(b, expected=s)
 
 class ExceptionTestCase(unittest.TestCase):
     def test_exceptions(self):
@@ -150,7 +115,7 @@
         new = marshal.loads(marshal.dumps(co))
         self.assertEqual(co, new)
 
-class ContainerTestCase(unittest.TestCase):
+class ContainerTestCase(unittest.TestCase, HelperMixin):
     d = {'astring': 'foo at bar.baz.spam',
          'afloat': 7283.43,
          'anint': 2**20,
@@ -161,42 +126,20 @@
          'aunicode': u"Andr&#65533; Previn"
          }
     def test_dict(self):
-        new = marshal.loads(marshal.dumps(self.d))
-        self.assertEqual(self.d, new)
-        marshal.dump(self.d, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(self.d, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(self.d)
 
     def test_list(self):
         lst = self.d.items()
-        new = marshal.loads(marshal.dumps(lst))
-        self.assertEqual(lst, new)
-        marshal.dump(lst, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(lst, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(lst)
 
     def test_tuple(self):
         t = tuple(self.d.keys())
-        new = marshal.loads(marshal.dumps(t))
-        self.assertEqual(t, new)
-        marshal.dump(t, file(test_support.TESTFN, "wb"))
-        new = marshal.load(file(test_support.TESTFN, "rb"))
-        self.assertEqual(t, new)
-        os.unlink(test_support.TESTFN)
+        self.helper(t)
 
     def test_sets(self):
         for constructor in (set, frozenset):
             t = constructor(self.d.keys())
-            new = marshal.loads(marshal.dumps(t))
-            self.assertEqual(t, new)
-            self.assertTrue(isinstance(new, constructor))
-            self.assertNotEqual(id(t), id(new))
-            marshal.dump(t, file(test_support.TESTFN, "wb"))
-            new = marshal.load(file(test_support.TESTFN, "rb"))
-            self.assertEqual(t, new)
-            os.unlink(test_support.TESTFN)
+            self.helper(t)
 
 class BugsTestCase(unittest.TestCase):
     def test_bug_5888452(self):
@@ -226,6 +169,7 @@
         s = 'c' + ('X' * 4*4) + '{' * 2**20
         self.assertRaises(ValueError, marshal.loads, s)
 
+    @test_support.impl_detail('specific recursion check')
     def test_recursion_limit(self):
         # Create a deeply nested structure.
         head = last = []
diff --git a/lib-python/2.7/test/test_memoryio.py b/lib-python/2.7/test/test_memoryio.py
--- a/lib-python/2.7/test/test_memoryio.py
+++ b/lib-python/2.7/test/test_memoryio.py
@@ -617,7 +617,7 @@
         state = memio.__getstate__()
         self.assertEqual(len(state), 3)
         bytearray(state[0]) # Check if state[0] supports the buffer interface.
-        self.assertIsInstance(state[1], int)
+        self.assertIsInstance(state[1], (int, long))
         self.assertTrue(isinstance(state[2], dict) or state[2] is None)
         memio.close()
         self.assertRaises(ValueError, memio.__getstate__)
diff --git a/lib-python/2.7/test/test_memoryview.py b/lib-python/2.7/test/test_memoryview.py
--- a/lib-python/2.7/test/test_memoryview.py
+++ b/lib-python/2.7/test/test_memoryview.py
@@ -26,7 +26,8 @@
     def check_getitem_with_type(self, tp):
         item = self.getitem_type
         b = tp(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         self.assertEqual(m[0], item(b"a"))
         self.assertIsInstance(m[0], bytes)
@@ -43,7 +44,8 @@
         self.assertRaises(TypeError, lambda: m[0.0])
         self.assertRaises(TypeError, lambda: m["a"])
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_getitem(self):
         for tp in self._types:
@@ -65,7 +67,8 @@
         if not self.ro_type:
             return
         b = self.ro_type(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         def setitem(value):
             m[0] = value
@@ -73,14 +76,16 @@
         self.assertRaises(TypeError, setitem, 65)
         self.assertRaises(TypeError, setitem, memoryview(b"a"))
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_setitem_writable(self):
         if not self.rw_type:
             return
         tp = self.rw_type
         b = self.rw_type(self._source)
-        oldrefcount = sys.getrefcount(b)
+        if hasattr(sys, 'getrefcount'):
+            oldrefcount = sys.getrefcount(b)
         m = self._view(b)
         m[0] = tp(b"0")
         self._check_contents(tp, b, b"0bcdef")
@@ -110,13 +115,14 @@
         self.assertRaises(TypeError, setitem, (0,), b"a")
         self.assertRaises(TypeError, setitem, "a", b"a")
         # Trying to resize the memory object
-        self.assertRaises(ValueError, setitem, 0, b"")
-        self.assertRaises(ValueError, setitem, 0, b"ab")
+        self.assertRaises((ValueError, TypeError), setitem, 0, b"")
+        self.assertRaises((ValueError, TypeError), setitem, 0, b"ab")
         self.assertRaises(ValueError, setitem, slice(1,1), b"a")
         self.assertRaises(ValueError, setitem, slice(0,2), b"a")
 
         m = None
-        self.assertEqual(sys.getrefcount(b), oldrefcount)
+        if hasattr(sys, 'getrefcount'):
+            self.assertEqual(sys.getrefcount(b), oldrefcount)
 
     def test_delitem(self):
         for tp in self._types:
@@ -292,6 +298,7 @@
     def _check_contents(self, tp, obj, contents):
         self.assertEqual(obj[1:7], tp(contents))
 
+    @unittest.skipUnless(hasattr(sys, 'getrefcount'), "Reference counting")
     def test_refs(self):
         for tp in self._types:
             m = memoryview(tp(self._source))
diff --git a/lib-python/2.7/test/test_mmap.py b/lib-python/2.7/test/test_mmap.py
--- a/lib-python/2.7/test/test_mmap.py
+++ b/lib-python/2.7/test/test_mmap.py
@@ -119,7 +119,8 @@
     def test_access_parameter(self):
         # Test for "access" keyword parameter
         mapsize = 10
-        open(TESTFN, "wb").write("a"*mapsize)
+        with open(TESTFN, "wb") as f:
+            f.write("a"*mapsize)
         f = open(TESTFN, "rb")
         m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_READ)
         self.assertEqual(m[:], 'a'*mapsize, "Readonly memory map data incorrect.")
@@ -168,9 +169,11 @@
         else:
             self.fail("Able to resize readonly memory map")
         f.close()
+        m.close()
         del m, f
-        self.assertEqual(open(TESTFN, "rb").read(), 'a'*mapsize,
-               "Readonly memory map data file was modified")
+        with open(TESTFN, "rb") as f:
+            self.assertEqual(f.read(), 'a'*mapsize,
+                "Readonly memory map data file was modified")
 
         # Opening mmap with size too big
         import sys
@@ -220,11 +223,13 @@
         self.assertEqual(m[:], 'd' * mapsize,
                "Copy-on-write memory map data not written correctly.")
         m.flush()
-        self.assertEqual(open(TESTFN, "rb").read(), 'c'*mapsize,
-               "Copy-on-write test data file should not be modified.")
+        f.close()
+        with open(TESTFN, "rb") as f:
+            self.assertEqual(f.read(), 'c'*mapsize,
+                "Copy-on-write test data file should not be modified.")
         # Ensuring copy-on-write maps cannot be resized
         self.assertRaises(TypeError, m.resize, 2*mapsize)
-        f.close()
+        m.close()
         del m, f
 
         # Ensuring invalid access parameter raises exception
@@ -287,6 +292,7 @@
         self.assertEqual(m.find('one', 1), 8)
         self.assertEqual(m.find('one', 1, -1), 8)
         self.assertEqual(m.find('one', 1, -2), -1)
+        m.close()
 
 
     def test_rfind(self):
@@ -305,6 +311,7 @@
         self.assertEqual(m.rfind('one', 0, -2), 0)
         self.assertEqual(m.rfind('one', 1, -1), 8)
         self.assertEqual(m.rfind('one', 1, -2), -1)
+        m.close()
 
 
     def test_double_close(self):
@@ -533,7 +540,8 @@
         if not hasattr(mmap, 'PROT_READ'):
             return
         mapsize = 10
-        open(TESTFN, "wb").write("a"*mapsize)
+        with open(TESTFN, "wb") as f:
+            f.write("a"*mapsize)
         f = open(TESTFN, "rb")
         m = mmap.mmap(f.fileno(), mapsize, prot=mmap.PROT_READ)
         self.assertRaises(TypeError, m.write, "foo")
@@ -545,7 +553,8 @@
 
     def test_io_methods(self):
         data = "0123456789"
-        open(TESTFN, "wb").write("x"*len(data))
+        with open(TESTFN, "wb") as f:
+            f.write("x"*len(data))
         f = open(TESTFN, "r+b")
         m = mmap.mmap(f.fileno(), len(data))
         f.close()
@@ -574,6 +583,7 @@
         self.assertEqual(m[:], "012bar6789")
         m.seek(8)
         self.assertRaises(ValueError, m.write, "bar")
+        m.close()
 
     if os.name == 'nt':
         def test_tagname(self):
@@ -611,7 +621,8 @@
             m.close()
 
             # Should not crash (Issue 5385)
-            open(TESTFN, "wb").write("x"*10)
+            with open(TESTFN, "wb") as f:
+                f.write("x"*10)
             f = open(TESTFN, "r+b")
             m = mmap.mmap(f.fileno(), 0)
             f.close()
diff --git a/lib-python/2.7/test/test_module.py b/lib-python/2.7/test/test_module.py
--- a/lib-python/2.7/test/test_module.py
+++ b/lib-python/2.7/test/test_module.py
@@ -1,6 +1,6 @@
 # Test the module type
 import unittest
-from test.test_support import run_unittest, gc_collect
+from test.test_support import run_unittest, gc_collect, check_impl_detail
 
 import sys
 ModuleType = type(sys)
@@ -10,8 +10,10 @@
         # An uninitialized module has no __dict__ or __name__,
         # and __doc__ is None
         foo = ModuleType.__new__(ModuleType)
-        self.assertTrue(foo.__dict__ is None)
-        self.assertRaises(SystemError, dir, foo)
+        self.assertFalse(foo.__dict__)
+        if check_impl_detail():
+            self.assertTrue(foo.__dict__ is None)
+            self.assertRaises(SystemError, dir, foo)
         try:
             s = foo.__name__
             self.fail("__name__ = %s" % repr(s))
diff --git a/lib-python/2.7/test/test_multibytecodec.py b/lib-python/2.7/test/test_multibytecodec.py
--- a/lib-python/2.7/test/test_multibytecodec.py
+++ b/lib-python/2.7/test/test_multibytecodec.py
@@ -42,7 +42,7 @@
         dec = codecs.getdecoder('euc-kr')
         myreplace  = lambda exc: (u'', sys.maxint+1)
         codecs.register_error('test.cjktest', myreplace)
-        self.assertRaises(IndexError, dec,
+        self.assertRaises((IndexError, OverflowError), dec,
                           'apple\x92ham\x93spam', 'test.cjktest')
 
     def test_codingspec(self):
@@ -148,7 +148,8 @@
 class Test_StreamReader(unittest.TestCase):
     def test_bug1728403(self):
         try:
-            open(TESTFN, 'w').write('\xa1')
+            with open(TESTFN, 'w') as f:
+                f.write('\xa1')
             f = codecs.open(TESTFN, encoding='cp949')
             self.assertRaises(UnicodeDecodeError, f.read, 2)
         finally:
diff --git a/lib-python/2.7/test/test_multibytecodec_support.py b/lib-python/2.7/test/test_multibytecodec_support.py
--- a/lib-python/2.7/test/test_multibytecodec_support.py
+++ b/lib-python/2.7/test/test_multibytecodec_support.py
@@ -110,8 +110,8 @@
         def myreplace(exc):
             return (u'x', sys.maxint + 1)
         codecs.register_error("test.cjktest", myreplace)
-        self.assertRaises(IndexError, self.encode, self.unmappedunicode,
-                          'test.cjktest')
+        self.assertRaises((IndexError, OverflowError), self.encode,
+                          self.unmappedunicode, 'test.cjktest')
 
     def test_callback_None_index(self):
         def myreplace(exc):
@@ -330,7 +330,7 @@
                             repr(csetch), repr(unich), exc.reason))
 
 def load_teststring(name):
-    dir = os.path.join(os.path.dirname(__file__), 'cjkencodings')
+    dir = test_support.findfile('cjkencodings')
     with open(os.path.join(dir, name + '.txt'), 'rb') as f:
         encoded = f.read()
     with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f:
diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py
--- a/lib-python/2.7/test/test_multiprocessing.py
+++ b/lib-python/2.7/test/test_multiprocessing.py
@@ -1316,6 +1316,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1605,6 +1606,10 @@
             if len(blocks) > maxblocks:
                 i = random.randrange(maxblocks)
                 del blocks[i]
+            # XXX There should be a better way to release resources for a
+            # single block
+            if i % maxblocks == 0:
+                import gc; gc.collect()
 
         # get the heap object
         heap = multiprocessing.heap.BufferWrapper._heap
@@ -1704,6 +1709,7 @@
         a = Foo()
         util.Finalize(a, conn.send, args=('a',))
         del a           # triggers callback for a
+        test_support.gc_collect()
 
         b = Foo()
         close_b = util.Finalize(b, conn.send, args=('b',))
diff --git a/lib-python/2.7/test/test_mutants.py b/lib-python/2.7/test/test_mutants.py
--- a/lib-python/2.7/test/test_mutants.py
+++ b/lib-python/2.7/test/test_mutants.py
@@ -1,4 +1,4 @@
-from test.test_support import verbose, TESTFN
+from test.test_support import verbose, TESTFN, check_impl_detail
 import random
 import os
 
@@ -137,10 +137,16 @@
     while dict1 and len(dict1) == len(dict2):
         if verbose:
             print ".",
-        if random.random() < 0.5:
-            c = cmp(dict1, dict2)
-        else:
-            c = dict1 == dict2
+        try:
+            if random.random() < 0.5:
+                c = cmp(dict1, dict2)
+            else:
+                c = dict1 == dict2
+        except RuntimeError:
+            # CPython never raises RuntimeError here, but other implementations
+            # might, and it's fine.
+            if check_impl_detail(cpython=True):
+                raise
     if verbose:
         print
 
diff --git a/lib-python/2.7/test/test_optparse.py b/lib-python/2.7/test/test_optparse.py
--- a/lib-python/2.7/test/test_optparse.py
+++ b/lib-python/2.7/test/test_optparse.py
@@ -383,6 +383,7 @@
         self.assertRaises(self.parser.remove_option, ('foo',), None,
                           ValueError, "no such option 'foo'")
 
+    @test_support.impl_detail("sys.getrefcount")
     def test_refleak(self):
         # If an OptionParser is carrying around a reference to a large
         # object, various cycles can prevent it from being GC'd in
diff --git a/lib-python/2.7/test/test_peepholer.py b/lib-python/2.7/test/test_peepholer.py
--- a/lib-python/2.7/test/test_peepholer.py
+++ b/lib-python/2.7/test/test_peepholer.py
@@ -41,7 +41,7 @@
     def test_none_as_constant(self):
         # LOAD_GLOBAL None  -->  LOAD_CONST None
         def f(x):
-            None
+            y = None
             return x
         asm = disassemble(f)
         for elem in ('LOAD_GLOBAL',):
@@ -67,10 +67,13 @@
             self.assertIn(elem, asm)
 
     def test_pack_unpack(self):
+        # On PyPy, "a, b = ..." is even more optimized, by removing
+        # the ROT_TWO.  But the ROT_TWO is not removed if assigning
+        # to more complex expressions, so check that.
         for line, elem in (
             ('a, = a,', 'LOAD_CONST',),
-            ('a, b = a, b', 'ROT_TWO',),
-            ('a, b, c = a, b, c', 'ROT_THREE',),
+            ('a[1], b = a, b', 'ROT_TWO',),
+            ('a, b[2], c = a, b, c', 'ROT_THREE',),
             ):
             asm = dis_single(line)
             self.assertIn(elem, asm)
@@ -78,6 +81,8 @@
             self.assertNotIn('UNPACK_TUPLE', asm)
 
     def test_folding_of_tuples_of_constants(self):
+        # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>"
+        # but on PyPy, it turns into "a=1;b=2;c=3".
         for line, elem in (
             ('a = 1,2,3', '((1, 2, 3))'),
             ('("a","b","c")', "(('a', 'b', 'c'))"),
@@ -86,7 +91,8 @@
             ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'),
             ):
             asm = dis_single(line)
-            self.assertIn(elem, asm)
+            self.assert_(elem in asm or (
+                line == 'a,b,c = 1,2,3' and 'UNPACK_TUPLE' not in asm))
             self.assertNotIn('BUILD_TUPLE', asm)
 
         # Bug 1053819:  Tuple of constants misidentified when presented with:
diff --git a/lib-python/2.7/test/test_pprint.py b/lib-python/2.7/test/test_pprint.py
--- a/lib-python/2.7/test/test_pprint.py
+++ b/lib-python/2.7/test/test_pprint.py
@@ -233,7 +233,16 @@
                                   frozenset([0, 2]),
                                   frozenset([0, 1])])}"""
         cube = test.test_set.cube(3)
-        self.assertEqual(pprint.pformat(cube), cube_repr_tgt)
+        # XXX issues of dictionary order, and for the case below,
+        # order of items in the frozenset([...]) representation.
+        # Whether we get precisely cube_repr_tgt or not is open
+        # to implementation-dependent choices (this test probably
+        # fails horribly in CPython if we tweak the dict order too).
+        got = pprint.pformat(cube)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertEqual(got, cube_repr_tgt)
+        else:
+            self.assertEqual(eval(got), cube)
         cubo_repr_tgt = """\
 {frozenset([frozenset([0, 2]), frozenset([0])]): frozenset([frozenset([frozenset([0,
                                                                                   2]),
@@ -393,7 +402,11 @@
                                                                                         2])])])}"""
 
         cubo = test.test_set.linegraph(cube)
-        self.assertEqual(pprint.pformat(cubo), cubo_repr_tgt)
+        got = pprint.pformat(cubo)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertEqual(got, cubo_repr_tgt)
+        else:
+            self.assertEqual(eval(got), cubo)
 
     def test_depth(self):
         nested_tuple = (1, (2, (3, (4, (5, 6)))))
diff --git a/lib-python/2.7/test/test_pydoc.py b/lib-python/2.7/test/test_pydoc.py
--- a/lib-python/2.7/test/test_pydoc.py
+++ b/lib-python/2.7/test/test_pydoc.py
@@ -267,8 +267,8 @@
         testpairs = (
             ('i_am_not_here', 'i_am_not_here'),
             ('test.i_am_not_here_either', 'i_am_not_here_either'),
-            ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
-            ('i_am_not_here.{}'.format(modname), 'i_am_not_here.{}'.format(modname)),
+            ('test.i_am_not_here.neither_am_i', 'i_am_not_here'),
+            ('i_am_not_here.{}'.format(modname), 'i_am_not_here'),
             ('test.{}'.format(modname), modname),
             )
 
@@ -292,8 +292,8 @@
                     result = run_pydoc(modname)
                 finally:
                     forget(modname)
-                expected = badimport_pattern % (modname, expectedinmsg)
-                self.assertEqual(expected, result)
+                expected = badimport_pattern % (modname, '(.+\\.)?' + expectedinmsg + '(\\..+)?$')
+                self.assertTrue(re.match(expected, result))
 
     def test_input_strip(self):
         missing_module = " test.i_am_not_here "
diff --git a/lib-python/2.7/test/test_pyexpat.py b/lib-python/2.7/test/test_pyexpat.py
--- a/lib-python/2.7/test/test_pyexpat.py
+++ b/lib-python/2.7/test/test_pyexpat.py
@@ -570,6 +570,9 @@
         self.assertEqual(self.n, 4)
 
 class MalformedInputText(unittest.TestCase):
+    # CPython seems to ship its own version of expat, they fixed it on this commit : 
+    # http://svn.python.org/view?revision=74429&view=revision
+    @unittest.skipIf(sys.platform == "darwin", "Expat is broken on Mac OS X 10.6.6")
     def test1(self):
         xml = "\0\r\n"
         parser = expat.ParserCreate()
@@ -579,6 +582,7 @@
         except expat.ExpatError as e:
             self.assertEqual(str(e), 'unclosed token: line 2, column 0')
 
+    @unittest.skipIf(sys.platform == "darwin", "Expat is broken on Mac OS X 10.6.6")
     def test2(self):
         xml = "<?xml version\xc2\x85='1.0'?>\r\n"
         parser = expat.ParserCreate()
diff --git a/lib-python/2.7/test/test_repr.py b/lib-python/2.7/test/test_repr.py
--- a/lib-python/2.7/test/test_repr.py
+++ b/lib-python/2.7/test/test_repr.py
@@ -9,6 +9,7 @@
 import unittest
 
 from test.test_support import run_unittest, check_py3k_warnings
+from test.test_support import check_impl_detail
 from repr import repr as r # Don't shadow builtin repr
 from repr import Repr
 
@@ -145,8 +146,11 @@
         # Functions
         eq(repr(hash), '<built-in function hash>')
         # Methods
-        self.assertTrue(repr(''.split).startswith(
-            '<built-in method split of str object at 0x'))
+        if check_impl_detail(cpython=True):
+            self.assertTrue(repr(''.split).startswith(
+                '<built-in method split of str object at 0x'))
+        elif check_impl_detail(pypy=True):
+            eq(repr(''.split), "<bound method str.split of ''>")
 
     def test_xrange(self):
         eq = self.assertEqual
@@ -185,7 +189,10 @@
     def test_descriptors(self):
         eq = self.assertEqual
         # method descriptors
-        eq(repr(dict.items), "<method 'items' of 'dict' objects>")
+        if check_impl_detail(cpython=True):
+            eq(repr(dict.items), "<method 'items' of 'dict' objects>")
+        elif check_impl_detail(pypy=True):
+            eq(repr(dict.items), "<unbound method dict.items>")
         # XXX member descriptors
         # XXX attribute descriptors
         # XXX slot descriptors
@@ -247,8 +254,14 @@
         eq = self.assertEqual
         touch(os.path.join(self.subpkgname, self.pkgname + os.extsep + 'py'))
         from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import areallylongpackageandmodulenametotestreprtruncation
-        eq(repr(areallylongpackageandmodulenametotestreprtruncation),
-           "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+        # On PyPy, we use %r to format the file name; on CPython it is done
+        # with '%s'.  It seems to me that %r is safer <arigo>.
+        if '__pypy__' in sys.builtin_module_names:
+            eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+               "<module %r from %r>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
+        else:
+            eq(repr(areallylongpackageandmodulenametotestreprtruncation),
+               "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__))
         eq(repr(sys), "<module 'sys' (built-in)>")
 
     def test_type(self):
diff --git a/lib-python/2.7/test/test_runpy.py b/lib-python/2.7/test/test_runpy.py
--- a/lib-python/2.7/test/test_runpy.py
+++ b/lib-python/2.7/test/test_runpy.py
@@ -5,10 +5,15 @@
 import sys
 import re
 import tempfile
-from test.test_support import verbose, run_unittest, forget
+from test.test_support import verbose, run_unittest, forget, check_impl_detail
 from test.script_helper import (temp_dir, make_script, compile_script,
                                 make_pkg, make_zip_script, make_zip_pkg)
 
+if check_impl_detail(pypy=True):
+    no_lone_pyc_file = True
+else:
+    no_lone_pyc_file = False
+
 
 from runpy import _run_code, _run_module_code, run_module, run_path
 # Note: This module can't safely test _run_module_as_main as it
@@ -168,13 +173,14 @@
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", mod_name
-            d2 = run_module(mod_name) # Read from bytecode
-            self.assertIn("x", d2)
-            self.assertTrue(d2["x"] == 1)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", mod_name
+                d2 = run_module(mod_name) # Read from bytecode
+                self.assertIn("x", d2)
+                self.assertTrue(d2["x"] == 1)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose: print "Module executed successfully"
@@ -190,13 +196,14 @@
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", pkg_name
-            d2 = run_module(pkg_name) # Read from bytecode
-            self.assertIn("x", d2)
-            self.assertTrue(d2["x"] == 1)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", pkg_name
+                d2 = run_module(pkg_name) # Read from bytecode
+                self.assertIn("x", d2)
+                self.assertTrue(d2["x"] == 1)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, pkg_name)
         if verbose: print "Package executed successfully"
@@ -244,15 +251,17 @@
             self.assertIn("sibling", d1)
             self.assertIn("nephew", d1)
             del d1 # Ensure __loader__ entry doesn't keep file open
-            __import__(mod_name)
-            os.remove(mod_fname)
-            if verbose: print "Running from compiled:", mod_name
-            d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
-            self.assertIn("__package__", d2)
-            self.assertTrue(d2["__package__"] == pkg_name)
-            self.assertIn("sibling", d2)
-            self.assertIn("nephew", d2)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not no_lone_pyc_file:
+                __import__(mod_name)
+                os.remove(mod_fname)
+                if verbose: print "Running from compiled:", mod_name
+                # Read from bytecode
+                d2 = run_module(mod_name, run_name=run_name)
+                self.assertIn("__package__", d2)
+                self.assertTrue(d2["__package__"] == pkg_name)
+                self.assertIn("sibling", d2)
+                self.assertIn("nephew", d2)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose: print "Module executed successfully"
@@ -345,6 +354,8 @@
                                script_dir, '')
 
     def test_directory_compiled(self):
+        if no_lone_pyc_file:
+            return
         with temp_dir() as script_dir:
             mod_name = '__main__'
             script_name = self._make_test_script(script_dir, mod_name)
diff --git a/lib-python/2.7/test/test_scope.py b/lib-python/2.7/test/test_scope.py
--- a/lib-python/2.7/test/test_scope.py
+++ b/lib-python/2.7/test/test_scope.py
@@ -1,6 +1,6 @@
 import unittest
 from test.test_support import check_syntax_error, check_py3k_warnings, \
-                              check_warnings, run_unittest
+                              check_warnings, run_unittest, gc_collect
 
 
 class ScopeTests(unittest.TestCase):
@@ -432,6 +432,7 @@
 
         for i in range(100):
             f1()
+        gc_collect()
 
         self.assertEqual(Foo.count, 0)
 
diff --git a/lib-python/2.7/test/test_set.py b/lib-python/2.7/test/test_set.py
--- a/lib-python/2.7/test/test_set.py
+++ b/lib-python/2.7/test/test_set.py
@@ -309,6 +309,7 @@
             fo.close()
             test_support.unlink(test_support.TESTFN)
 
+    @test_support.impl_detail(pypy=False)
     def test_do_not_rehash_dict_keys(self):
         n = 10
         d = dict.fromkeys(map(HashCountingInt, xrange(n)))
@@ -559,6 +560,7 @@
         p = weakref.proxy(s)
         self.assertEqual(str(p), str(s))
         s = None
+        test_support.gc_collect()
         self.assertRaises(ReferenceError, str, p)
 
     # C API test only available in a debug build
@@ -590,6 +592,7 @@
         s.__init__(self.otherword)
         self.assertEqual(s, set(self.word))
 
+    @test_support.impl_detail()
     def test_singleton_empty_frozenset(self):
         f = frozenset()
         efs = [frozenset(), frozenset([]), frozenset(()), frozenset(''),
@@ -770,9 +773,10 @@
         for v in self.set:
             self.assertIn(v, self.values)
         setiter = iter(self.set)
-        # note: __length_hint__ is an internal undocumented API,
-        # don't rely on it in your own programs
-        self.assertEqual(setiter.__length_hint__(), len(self.set))
+        if test_support.check_impl_detail():
+            # note: __length_hint__ is an internal undocumented API,
+            # don't rely on it in your own programs
+            self.assertEqual(setiter.__length_hint__(), len(self.set))
 
     def test_pickling(self):
         p = pickle.dumps(self.set)
@@ -1564,7 +1568,7 @@
             for meth in (s.union, s.intersection, s.difference, s.symmetric_difference, s.isdisjoint):
                 for g in (G, I, Ig, L, R):
                     expected = meth(data)
-                    actual = meth(G(data))
+                    actual = meth(g(data))
                     if isinstance(expected, bool):
                         self.assertEqual(actual, expected)
                     else:
diff --git a/lib-python/2.7/test/test_sets.py b/lib-python/2.7/test/test_sets.py
--- a/lib-python/2.7/test/test_sets.py
+++ b/lib-python/2.7/test/test_sets.py
@@ -686,7 +686,9 @@
         set_list = sorted(self.set)
         self.assertEqual(len(dup_list), len(set_list))
         for i, el in enumerate(dup_list):
-            self.assertIs(el, set_list[i])
+            # Object identity is not guarnteed for immutable objects, so we
+            # can't use assertIs here.
+            self.assertEqual(el, set_list[i])
 
     def test_deep_copy(self):
         dup = copy.deepcopy(self.set)
diff --git a/lib-python/2.7/test/test_site.py b/lib-python/2.7/test/test_site.py
--- a/lib-python/2.7/test/test_site.py
+++ b/lib-python/2.7/test/test_site.py
@@ -226,6 +226,10 @@
             self.assertEqual(len(dirs), 1)
             wanted = os.path.join('xoxo', 'Lib', 'site-packages')
             self.assertEqual(dirs[0], wanted)
+        elif '__pypy__' in sys.builtin_module_names:
+            self.assertEquals(len(dirs), 1)
+            wanted = os.path.join('xoxo', 'site-packages')
+            self.assertEquals(dirs[0], wanted)
         elif os.sep == '/':
             self.assertEqual(len(dirs), 2)
             wanted = os.path.join('xoxo', 'lib', 'python' + sys.version[:3],
diff --git a/lib-python/2.7/test/test_socket.py b/lib-python/2.7/test/test_socket.py
--- a/lib-python/2.7/test/test_socket.py
+++ b/lib-python/2.7/test/test_socket.py
@@ -252,6 +252,7 @@
         self.assertEqual(p.fileno(), s.fileno())
         s.close()
         s = None
+        test_support.gc_collect()
         try:
             p.fileno()
         except ReferenceError:
@@ -285,32 +286,34 @@
             s.sendto(u'\u2620', sockname)
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, sockname)
-        self.assertIn('not complex', str(cm.exception))
+        self.assertIn('complex', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', None)
-        self.assertIn('not NoneType', str(cm.exception))
+        self.assertIn('NoneType', str(cm.exception))
         # 3 args
         with self.assertRaises(UnicodeEncodeError):
             s.sendto(u'\u2620', 0, sockname)
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, 0, sockname)
-        self.assertIn('not complex', str(cm.exception))
+        self.assertIn('complex', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 0, None)
-        self.assertIn('not NoneType', str(cm.exception))
+        if test_support.check_impl_detail():
+            self.assertIn('not NoneType', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 'bar', sockname)
-        self.assertIn('an integer is required', str(cm.exception))
+        self.assertIn('integer', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', None, None)
-        self.assertIn('an integer is required', str(cm.exception))
+        if test_support.check_impl_detail():
+            self.assertIn('an integer is required', str(cm.exception))
         # wrong number of args
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo')
-        self.assertIn('(1 given)', str(cm.exception))
+        self.assertIn(' given)', str(cm.exception))
         with self.assertRaises(TypeError) as cm:
             s.sendto('foo', 0, sockname, 4)
-        self.assertIn('(4 given)', str(cm.exception))
+        self.assertIn(' given)', str(cm.exception))
 
 
     def testCrucialConstants(self):
@@ -385,10 +388,10 @@
             socket.htonl(k)
             socket.htons(k)
         for k in bad_values:
-            self.assertRaises(OverflowError, socket.ntohl, k)
-            self.assertRaises(OverflowError, socket.ntohs, k)
-            self.assertRaises(OverflowError, socket.htonl, k)
-            self.assertRaises(OverflowError, socket.htons, k)
+            self.assertRaises((OverflowError, ValueError), socket.ntohl, k)
+            self.assertRaises((OverflowError, ValueError), socket.ntohs, k)
+            self.assertRaises((OverflowError, ValueError), socket.htonl, k)
+            self.assertRaises((OverflowError, ValueError), socket.htons, k)
 
     def testGetServBy(self):
         eq = self.assertEqual
@@ -428,8 +431,8 @@
         if udpport is not None:
             eq(socket.getservbyport(udpport, 'udp'), service)
         # Make sure getservbyport does not accept out of range ports.
-        self.assertRaises(OverflowError, socket.getservbyport, -1)
-        self.assertRaises(OverflowError, socket.getservbyport, 65536)
+        self.assertRaises((OverflowError, ValueError), socket.getservbyport, -1)
+        self.assertRaises((OverflowError, ValueError), socket.getservbyport, 65536)
 
     def testDefaultTimeout(self):
         # Testing default timeout
@@ -608,8 +611,8 @@
         neg_port = port - 65536
         sock = socket.socket()
         try:
-            self.assertRaises(OverflowError, sock.bind, (host, big_port))
-            self.assertRaises(OverflowError, sock.bind, (host, neg_port))
+            self.assertRaises((OverflowError, ValueError), sock.bind, (host, big_port))
+            self.assertRaises((OverflowError, ValueError), sock.bind, (host, neg_port))
             sock.bind((host, port))
         finally:
             sock.close()
@@ -1309,6 +1312,7 @@
             closed = False
             def flush(self): pass
             def close(self): self.closed = True
+            def _decref_socketios(self): pass
 
         # must not close unless we request it: the original use of _fileobject
         # by module socket requires that the underlying socket not be closed until
diff --git a/lib-python/2.7/test/test_sort.py b/lib-python/2.7/test/test_sort.py
--- a/lib-python/2.7/test/test_sort.py
+++ b/lib-python/2.7/test/test_sort.py
@@ -140,7 +140,10 @@
                 return random.random() < 0.5
 
         L = [C() for i in range(50)]
-        self.assertRaises(ValueError, L.sort)
+        try:
+            L.sort()
+        except ValueError:
+            pass
 
     def test_cmpNone(self):
         # Testing None as a comparison function.
@@ -150,8 +153,10 @@
         L.sort(None)
         self.assertEqual(L, range(50))
 
+    @test_support.impl_detail(pypy=False)
     def test_undetected_mutation(self):
         # Python 2.4a1 did not always detect mutation
+        # So does pypy...
         memorywaster = []
         for i in range(20):
             def mutating_cmp(x, y):
@@ -226,7 +231,10 @@
             def __del__(self):
                 del data[:]
                 data[:] = range(20)
-        self.assertRaises(ValueError, data.sort, key=SortKiller)
+        try:
+            data.sort(key=SortKiller)
+        except ValueError:
+            pass
 
     def test_key_with_mutating_del_and_exception(self):
         data = range(10)
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -881,6 +881,8 @@
                 c = socket.socket()
                 c.connect((HOST, port))
                 listener_gone.wait()
+                # XXX why is it necessary?
+                test_support.gc_collect()
                 try:
                     ssl_sock = ssl.wrap_socket(c)
                 except IOError:
@@ -1330,10 +1332,8 @@
 
 def test_main(verbose=False):
     global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT
-    CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir,
-                            "keycert.pem")
-    SVN_PYTHON_ORG_ROOT_CERT = os.path.join(
-        os.path.dirname(__file__) or os.curdir,
+    CERTFILE = test_support.findfile("keycert.pem")
+    SVN_PYTHON_ORG_ROOT_CERT = test_support.findfile(
         "https_svn_python_org_root.pem")
 
     if (not os.path.exists(CERTFILE) or
diff --git a/lib-python/2.7/test/test_str.py b/lib-python/2.7/test/test_str.py
--- a/lib-python/2.7/test/test_str.py
+++ b/lib-python/2.7/test/test_str.py
@@ -422,10 +422,11 @@
         for meth in ('foo'.startswith, 'foo'.endswith):
             with self.assertRaises(TypeError) as cm:
                 meth(['f'])
-            exc = str(cm.exception)
-            self.assertIn('unicode', exc)
-            self.assertIn('str', exc)
-            self.assertIn('tuple', exc)
+            if test_support.check_impl_detail():
+                exc = str(cm.exception)
+                self.assertIn('unicode', exc)
+                self.assertIn('str', exc)
+                self.assertIn('tuple', exc)
 
 def test_main():
     test_support.run_unittest(StrTest)
diff --git a/lib-python/2.7/test/test_struct.py b/lib-python/2.7/test/test_struct.py
--- a/lib-python/2.7/test/test_struct.py
+++ b/lib-python/2.7/test/test_struct.py
@@ -535,7 +535,8 @@
 
     @unittest.skipUnless(IS32BIT, "Specific to 32bit machines")
     def test_crasher(self):
-        self.assertRaises(MemoryError, struct.pack, "357913941c", "a")
+        self.assertRaises((MemoryError, struct.error), struct.pack,
+                          "357913941c", "a")
 
     def test_count_overflow(self):
         hugecount = '{}b'.format(sys.maxsize+1)
diff --git a/lib-python/2.7/test/test_subprocess.py b/lib-python/2.7/test/test_subprocess.py
--- a/lib-python/2.7/test/test_subprocess.py
+++ b/lib-python/2.7/test/test_subprocess.py
@@ -16,11 +16,11 @@
 # Depends on the following external programs: Python
 #
 
-if mswindows:
-    SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
-                                                'os.O_BINARY);')
-else:
-    SETBINARY = ''
+#if mswindows:
+#    SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), '
+#                                                'os.O_BINARY);')
+#else:
+#    SETBINARY = ''
 
 
 try:
@@ -420,8 +420,9 @@
         self.assertStderrEqual(stderr, "")
 
     def test_universal_newlines(self):
-        p = subprocess.Popen([sys.executable, "-c",
-                          'import sys,os;' + SETBINARY +
+        # NB. replaced SETBINARY with the -u flag
+        p = subprocess.Popen([sys.executable, "-u", "-c",
+                          'import sys,os;' + #SETBINARY +
                           'sys.stdout.write("line1\\n");'
                           'sys.stdout.flush();'
                           'sys.stdout.write("line2\\r");'
@@ -448,8 +449,9 @@
 
     def test_universal_newlines_communicate(self):
         # universal newlines through communicate()
-        p = subprocess.Popen([sys.executable, "-c",
-                          'import sys,os;' + SETBINARY +
+        # NB. replaced SETBINARY with the -u flag
+        p = subprocess.Popen([sys.executable, "-u", "-c",
+                          'import sys,os;' + #SETBINARY +
                           'sys.stdout.write("line1\\n");'
                           'sys.stdout.flush();'
                           'sys.stdout.write("line2\\r");'
diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py
--- a/lib-python/2.7/test/test_support.py
+++ b/lib-python/2.7/test/test_support.py
@@ -431,16 +431,20 @@
             rmtree(name)
 
 
-def findfile(file, here=__file__, subdir=None):
+def findfile(file, here=None, subdir=None):
     """Try to find a file on sys.path and the working directory.  If it is not
     found the argument passed to the function is returned (this does not
     necessarily signal failure; could still be the legitimate path)."""
+    import test
     if os.path.isabs(file):
         return file
     if subdir is not None:
         file = os.path.join(subdir, file)
     path = sys.path
-    path = [os.path.dirname(here)] + path
+    if here is None:
+        path = test.__path__ + path
+    else:
+        path = [os.path.dirname(here)] + path
     for dn in path:
         fn = os.path.join(dn, file)
         if os.path.exists(fn): return fn
@@ -1050,15 +1054,33 @@
     guards, default = _parse_guards(guards)
     return guards.get(platform.python_implementation().lower(), default)
 
+# ----------------------------------
+# PyPy extension: you can run::
+#     python ..../test_foo.py --pdb
+# to get a pdb prompt in case of exceptions
 
+ResultClass = unittest.TextTestRunner.resultclass
+
+class TestResultWithPdb(ResultClass):
+
+    def addError(self, testcase, exc_info):
+        ResultClass.addError(self, testcase, exc_info)
+        if '--pdb' in sys.argv:
+            import pdb, traceback
+            traceback.print_tb(exc_info[2])
+            pdb.post_mortem(exc_info[2])
+
+# ----------------------------------
 
 def _run_suite(suite):
     """Run tests from a unittest.TestSuite-derived class."""
     if verbose:
-        runner = unittest.TextTestRunner(sys.stdout, verbosity=2)
+        runner = unittest.TextTestRunner(sys.stdout, verbosity=2,
+                                         resultclass=TestResultWithPdb)
     else:
         runner = BasicTestRunner()
 
+
     result = runner.run(suite)
     if not result.wasSuccessful():
         if len(result.errors) == 1 and not result.failures:
@@ -1071,6 +1093,34 @@
                 err += "; run in verbose mode for details"
         raise TestFailed(err)
 
+# ----------------------------------
+# PyPy extension: you can run::
+#     python ..../test_foo.py --filter bar
+# to run only the test cases whose name contains bar
+
+def filter_maybe(suite):
+    try:
+        i = sys.argv.index('--filter')
+        filter = sys.argv[i+1]
+    except (ValueError, IndexError):
+        return suite
+    tests = []
+    for test in linearize_suite(suite):
+        if filter in test._testMethodName:
+            tests.append(test)
+    return unittest.TestSuite(tests)
+
+def linearize_suite(suite_or_test):
+    try:
+        it = iter(suite_or_test)
+    except TypeError:
+        yield suite_or_test
+        return
+    for subsuite in it:
+        for item in linearize_suite(subsuite):
+            yield item
+
+# ----------------------------------
 
 def run_unittest(*classes):
     """Run tests from unittest.TestCase-derived classes."""
@@ -1086,6 +1136,7 @@
             suite.addTest(cls)
         else:
             suite.addTest(unittest.makeSuite(cls))
+    suite = filter_maybe(suite)
     _run_suite(suite)
 
 
diff --git a/lib-python/2.7/test/test_syntax.py b/lib-python/2.7/test/test_syntax.py
--- a/lib-python/2.7/test/test_syntax.py
+++ b/lib-python/2.7/test/test_syntax.py
@@ -5,7 +5,8 @@
 >>> def f(x):
 ...     global x
 Traceback (most recent call last):
-SyntaxError: name 'x' is local and global (<doctest test.test_syntax[0]>, line 1)
+  File "<doctest test.test_syntax[0]>", line 1
+SyntaxError: name 'x' is local and global
 
 The tests are all raise SyntaxErrors.  They were created by checking
 each C call that raises SyntaxError.  There are several modules that
@@ -375,7 +376,7 @@
 In 2.5 there was a missing exception and an assert was triggered in a debug
 build.  The number of blocks must be greater than CO_MAXBLOCKS.  SF #1565514
 
-   >>> while 1:
+   >>> while 1: # doctest:+SKIP
    ...  while 2:
    ...   while 3:
    ...    while 4:
diff --git a/lib-python/2.7/test/test_sys.py b/lib-python/2.7/test/test_sys.py
--- a/lib-python/2.7/test/test_sys.py
+++ b/lib-python/2.7/test/test_sys.py
@@ -264,6 +264,7 @@
             self.assertEqual(sys.getdlopenflags(), oldflags+1)
             sys.setdlopenflags(oldflags)
 
+    @test.test_support.impl_detail("reference counting")
     def test_refcount(self):
         # n here must be a global in order for this test to pass while
         # tracing with a python function.  Tracing calls PyFrame_FastToLocals
@@ -287,7 +288,7 @@
             is sys._getframe().f_code
         )
 
-    # sys._current_frames() is a CPython-only gimmick.
+    @test.test_support.impl_detail("current_frames")
     def test_current_frames(self):
         have_threads = True
         try:
@@ -383,7 +384,10 @@
         self.assertEqual(len(sys.float_info), 11)
         self.assertEqual(sys.float_info.radix, 2)
         self.assertEqual(len(sys.long_info), 2)
-        self.assertTrue(sys.long_info.bits_per_digit % 5 == 0)
+        if test.test_support.check_impl_detail(cpython=True):
+            self.assertTrue(sys.long_info.bits_per_digit % 5 == 0)
+        else:
+            self.assertTrue(sys.long_info.bits_per_digit >= 1)
         self.assertTrue(sys.long_info.sizeof_digit >= 1)
         self.assertEqual(type(sys.long_info.bits_per_digit), int)
         self.assertEqual(type(sys.long_info.sizeof_digit), int)
@@ -432,6 +436,7 @@
             self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
         self.assertTrue(repr(sys.flags))
 
+    @test.test_support.impl_detail("sys._clear_type_cache")
     def test_clear_type_cache(self):
         sys._clear_type_cache()
 
@@ -473,6 +478,7 @@
         p.wait()
         self.assertIn(executable, ["''", repr(sys.executable)])
 
+ at unittest.skipUnless(test.test_support.check_impl_detail(), "sys.getsizeof()")
 class SizeofTest(unittest.TestCase):
 
     TPFLAGS_HAVE_GC = 1<<14
diff --git a/lib-python/2.7/test/test_sys_settrace.py b/lib-python/2.7/test/test_sys_settrace.py
--- a/lib-python/2.7/test/test_sys_settrace.py
+++ b/lib-python/2.7/test/test_sys_settrace.py
@@ -213,12 +213,16 @@
         "finally"
 def generator_example():
     # any() will leave the generator before its end
-    x = any(generator_function())
+    x = any(generator_function()); gc.collect()
 
     # the following lines were not traced
     for x in range(10):
         y = x
 
+# On CPython, when the generator is decref'ed to zero, we see the trace
+# for the "finally:" portion.  On PyPy, we don't see it before the next
+# garbage collection.  That's why we put gc.collect() on the same line above.
+
 generator_example.events = ([(0, 'call'),
                              (2, 'line'),
                              (-6, 'call'),
@@ -282,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
@@ -323,17 +327,24 @@
         self.run_test(tighterloop_example)
 
     def test_13_genexp(self):
-        self.run_test(generator_example)
-        # issue1265: if the trace function contains a generator,
-        # and if the traced function contains another generator
-        # that is not completely exhausted, the trace stopped.
-        # Worse: the 'finally' clause was not invoked.
-        tracer = Tracer()
-        sys.settrace(tracer.traceWithGenexp)
-        generator_example()
-        sys.settrace(None)
-        self.compare_events(generator_example.__code__.co_firstlineno,
-                            tracer.events, generator_example.events)
+        if self.using_gc:
+            test_support.gc_collect()
+            gc.enable()
+        try:
+            self.run_test(generator_example)
+            # issue1265: if the trace function contains a generator,
+            # and if the traced function contains another generator
+            # that is not completely exhausted, the trace stopped.
+            # Worse: the 'finally' clause was not invoked.
+            tracer = Tracer()
+            sys.settrace(tracer.traceWithGenexp)
+            generator_example()
+            sys.settrace(None)
+            self.compare_events(generator_example.__code__.co_firstlineno,
+                                tracer.events, generator_example.events)
+        finally:
+            if self.using_gc:
+                gc.disable()
 
     def test_14_onliner_if(self):
         def onliners():
diff --git a/lib-python/2.7/test/test_sysconfig.py b/lib-python/2.7/test/test_sysconfig.py
--- a/lib-python/2.7/test/test_sysconfig.py
+++ b/lib-python/2.7/test/test_sysconfig.py
@@ -209,13 +209,22 @@
 
         self.assertEqual(get_platform(), 'macosx-10.4-fat64')
 
-        for arch in ('ppc', 'i386', 'x86_64', 'ppc64'):
+        for arch in ('ppc', 'i386', 'ppc64', 'x86_64'):
             get_config_vars()['CFLAGS'] = ('-arch %s -isysroot '
                                            '/Developer/SDKs/MacOSX10.4u.sdk  '
                                            '-fno-strict-aliasing -fno-common '
                                            '-dynamic -DNDEBUG -g -O3'%(arch,))
 
             self.assertEqual(get_platform(), 'macosx-10.4-%s'%(arch,))
+        
+        # macosx with ARCHFLAGS set and empty _CONFIG_VARS
+        os.environ['ARCHFLAGS'] = '-arch i386'
+        sysconfig._CONFIG_VARS = None
+        
+        # this will attempt to recreate the _CONFIG_VARS based on environment 
+        # variables; used to check a problem with the PyPy's _init_posix
+        # implementation; see: issue 705
+        get_config_vars() 
 
         # linux debian sarge
         os.name = 'posix'
@@ -235,7 +244,7 @@
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
-                  'posix_home', 'posix_prefix', 'posix_user')
+                  'posix_home', 'posix_prefix', 'posix_user', 'pypy')
         self.assertEqual(get_scheme_names(), wanted)
 
     def test_symlink(self):
diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/2.7/test/test_tarfile.py
--- a/lib-python/2.7/test/test_tarfile.py
+++ b/lib-python/2.7/test/test_tarfile.py
@@ -169,6 +169,7 @@
         except tarfile.ReadError:
             self.fail("tarfile.open() failed on empty archive")
         self.assertListEqual(tar.getmembers(), [])
+        tar.close()
 
     def test_null_tarfile(self):
         # Test for issue6123: Allow opening empty archives.
@@ -207,16 +208,21 @@
         fobj = open(self.tarname, "rb")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, os.path.abspath(fobj.name))
+        tar.close()
 
     def test_no_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self.assertRaises(AttributeError, getattr, fobj, "name")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, None)
 
     def test_empty_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         fobj.name = ""
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
@@ -515,6 +521,7 @@
         self.tar = tarfile.open(self.tarname, mode=self.mode, encoding="iso8859-1")
         tarinfo = self.tar.getmember("pax/umlauts-&#65533;&#65533;&#65533;&#65533;&#65533;&#65533;&#65533;")
         self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self.tar.close()
 
 
 class LongnameTest(ReadTest):
@@ -675,6 +682,7 @@
             tar = tarfile.open(tmpname, self.mode)
             tarinfo = tar.gettarinfo(path)
             self.assertEqual(tarinfo.size, 0)
+            tar.close()
         finally:
             os.rmdir(path)
 
@@ -692,6 +700,7 @@
                 tar.gettarinfo(target)
                 tarinfo = tar.gettarinfo(link)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(target)
                 os.remove(link)
@@ -704,6 +713,7 @@
                 tar = tarfile.open(tmpname, self.mode)
                 tarinfo = tar.gettarinfo(path)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(path)
 
@@ -722,6 +732,7 @@
         tar.add(dstname)
         os.chdir(cwd)
         self.assertTrue(tar.getnames() == [], "added the archive to itself")
+        tar.close()
 
     def test_exclude(self):
         tempdir = os.path.join(TEMPDIR, "exclude")
@@ -742,6 +753,7 @@
             tar = tarfile.open(tmpname, "r")
             self.assertEqual(len(tar.getmembers()), 1)
             self.assertEqual(tar.getnames()[0], "empty_dir")
+            tar.close()
         finally:
             shutil.rmtree(tempdir)
 
@@ -947,7 +959,9 @@
             fobj.close()
         elif self.mode.endswith("bz2"):
             dec = bz2.BZ2Decompressor()
-            data = open(tmpname, "rb").read()
+            f = open(tmpname, "rb")
+            data = f.read()
+            f.close()
             data = dec.decompress(data)
             self.assertTrue(len(dec.unused_data) == 0,
                     "found trailing data")
@@ -1026,6 +1040,7 @@
                 "unable to read longname member")
         self.assertEqual(tarinfo.linkname, member.linkname,
                 "unable to read longname member")
+        tar.close()
 
     def test_longname_1023(self):
         self._test(("longnam/" * 127) + "longnam")
@@ -1118,6 +1133,7 @@
         else:
             n = tar.getmembers()[0].name
             self.assertTrue(name == n, "PAX longname creation failed")
+        tar.close()
 
     def test_pax_global_header(self):
         pax_headers = {
@@ -1146,6 +1162,7 @@
                     tarfile.PAX_NUMBER_FIELDS[key](val)
                 except (TypeError, ValueError):
                     self.fail("unable to convert pax header field")
+        tar.close()
 
     def test_pax_extended_header(self):
         # The fields from the pax header have priority over the
@@ -1165,6 +1182,7 @@
         self.assertEqual(t.pax_headers, pax_headers)
         self.assertEqual(t.name, "foo")
         self.assertEqual(t.uid, 123)
+        tar.close()
 
 
 class UstarUnicodeTest(unittest.TestCase):
@@ -1208,6 +1226,7 @@
         tarinfo.name = "foo"
         tarinfo.uname = u"&#65533;&#65533;&#65533;"
         self.assertRaises(UnicodeError, tar.addfile, tarinfo)
+        tar.close()
 
     def test_unicode_argument(self):
         tar = tarfile.open(tarname, "r", encoding="iso8859-1", errors="strict")
@@ -1262,6 +1281,7 @@
             tar = tarfile.open(tmpname, format=self.format, encoding="ascii",
                     errors=handler)
             self.assertEqual(tar.getnames()[0], name)
+            tar.close()
 
         self.assertRaises(UnicodeError, tarfile.open, tmpname,
                 encoding="ascii", errors="strict")
@@ -1274,6 +1294,7 @@
         tar = tarfile.open(tmpname, format=self.format, encoding="iso8859-1",
                 errors="utf-8")
         self.assertEqual(tar.getnames()[0], "&#65533;&#65533;&#65533;/" + u"&#65533;".encode("utf8"))
+        tar.close()
 
 
 class AppendTest(unittest.TestCase):
@@ -1301,6 +1322,7 @@
     def _test(self, names=["bar"], fileobj=None):
         tar = tarfile.open(self.tarname, fileobj=fileobj)
         self.assertEqual(tar.getnames(), names)
+        tar.close()
 
     def test_non_existing(self):
         self._add_testfile()
@@ -1319,7 +1341,9 @@
 
     def test_fileobj(self):
         self._create_testtar()
-        data = open(self.tarname).read()
+        f = open(self.tarname)
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self._add_testfile(fobj)
         fobj.seek(0)
@@ -1345,7 +1369,9 @@
     # Append mode is supposed to fail if the tarfile to append to
     # does not end with a zero block.
     def _test_error(self, data):
-        open(self.tarname, "wb").write(data)
+        f = open(self.tarname, "wb")
+        f.write(data)
+        f.close()
         self.assertRaises(tarfile.ReadError, self._add_testfile)
 
     def test_null(self):
diff --git a/lib-python/2.7/test/test_tempfile.py b/lib-python/2.7/test/test_tempfile.py
--- a/lib-python/2.7/test/test_tempfile.py
+++ b/lib-python/2.7/test/test_tempfile.py
@@ -23,8 +23,8 @@
 
 # TEST_FILES may need to be tweaked for systems depending on the maximum
 # number of files that can be opened at one time (see ulimit -n)
-if sys.platform in ('openbsd3', 'openbsd4'):
-    TEST_FILES = 48
+if sys.platform.startswith("openbsd"):
+    TEST_FILES = 64 # ulimit -n defaults to 128 for normal users
 else:
     TEST_FILES = 100
 
@@ -244,6 +244,7 @@
         dir = tempfile.mkdtemp()
         try:
             self.do_create(dir=dir).write("blat")
+            test_support.gc_collect()
         finally:
             os.rmdir(dir)
 
@@ -528,12 +529,15 @@
         self.do_create(suf="b")
         self.do_create(pre="a", suf="b")
         self.do_create(pre="aa", suf=".txt")
+        test_support.gc_collect()
 
     def test_many(self):
         # mktemp can choose many usable file names (stochastic)
         extant = range(TEST_FILES)
         for i in extant:
             extant[i] = self.do_create(pre="aa")
+        del extant
+        test_support.gc_collect()
 
 ##     def test_warning(self):
 ##         # mktemp issues a warning when used
diff --git a/lib-python/2.7/test/test_thread.py b/lib-python/2.7/test/test_thread.py
--- a/lib-python/2.7/test/test_thread.py
+++ b/lib-python/2.7/test/test_thread.py
@@ -128,6 +128,7 @@
         del task
         while not done:
             time.sleep(0.01)
+            test_support.gc_collect()
         self.assertEqual(thread._count(), orig)
 
 
diff --git a/lib-python/2.7/test/test_threading.py b/lib-python/2.7/test/test_threading.py
--- a/lib-python/2.7/test/test_threading.py
+++ b/lib-python/2.7/test/test_threading.py
@@ -161,6 +161,7 @@
 
     # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently)
     # exposed at the Python level.  This test relies on ctypes to get at it.
+    @test.test_support.cpython_only
     def test_PyThreadState_SetAsyncExc(self):
         try:
             import ctypes
@@ -266,6 +267,7 @@
         finally:
             threading._start_new_thread = _start_new_thread
 
+    @test.test_support.cpython_only
     def test_finalize_runnning_thread(self):
         # Issue 1402: the PyGILState_Ensure / _Release functions may be called
         # very late on python exit: on deallocation of a running thread for
@@ -383,6 +385,7 @@
         finally:
             sys.setcheckinterval(old_interval)
 
+    @test.test_support.cpython_only
     def test_no_refcycle_through_target(self):
         class RunSelfFunction(object):
             def __init__(self, should_raise):
@@ -425,6 +428,9 @@
             def joiningfunc(mainthread):
                 mainthread.join()
                 print 'end of thread'
+                # stdout is fully buffered because not a tty, we have to flush
+                # before exit.
+                sys.stdout.flush()
         \n""" + script
 
         p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE)
diff --git a/lib-python/2.7/test/test_threading_local.py b/lib-python/2.7/test/test_threading_local.py
--- a/lib-python/2.7/test/test_threading_local.py
+++ b/lib-python/2.7/test/test_threading_local.py
@@ -173,8 +173,9 @@
         obj = cls()
         obj.x = 5
         self.assertEqual(obj.__dict__, {'x': 5})
-        with self.assertRaises(AttributeError):
-            obj.__dict__ = {}
+        if test_support.check_impl_detail():
+            with self.assertRaises(AttributeError):
+                obj.__dict__ = {}
         with self.assertRaises(AttributeError):
             del obj.__dict__
 
diff --git a/lib-python/2.7/test/test_traceback.py b/lib-python/2.7/test/test_traceback.py
--- a/lib-python/2.7/test/test_traceback.py
+++ b/lib-python/2.7/test/test_traceback.py
@@ -5,7 +5,8 @@
 import sys
 import unittest
 from imp import reload
-from test.test_support import run_unittest, is_jython, Error
+from test.test_support import run_unittest, Error
+from test.test_support import impl_detail, check_impl_detail
 
 import traceback
 
@@ -49,10 +50,8 @@
         self.assertTrue(err[2].count('\n') == 1) # and no additional newline
         self.assertTrue(err[1].find("+") == err[2].find("^")) # in the right place
 
+    @impl_detail("other implementations may add a caret (why shouldn't they?)")
     def test_nocaret(self):
-        if is_jython:
-            # jython adds a caret in this case (why shouldn't it?)
-            return
         err = self.get_exception_format(self.syntax_error_without_caret,
                                         SyntaxError)
         self.assertTrue(len(err) == 3)
@@ -63,8 +62,11 @@
                                         IndentationError)
         self.assertTrue(len(err) == 4)
         self.assertTrue(err[1].strip() == "print 2")
-        self.assertIn("^", err[2])
-        self.assertTrue(err[1].find("2") == err[2].find("^"))
+        if check_impl_detail():
+            # on CPython, there is a "^" at the end of the line
+            # on PyPy, there is a "^" too, but at the start, more logically
+            self.assertIn("^", err[2])
+            self.assertTrue(err[1].find("2") == err[2].find("^"))
 
     def test_bug737473(self):
         import os, tempfile, time
@@ -74,7 +76,8 @@
         try:
             sys.path.insert(0, testdir)
             testfile = os.path.join(testdir, 'test_bug737473.py')
-            print >> open(testfile, 'w'), """
+            with open(testfile, 'w') as f:
+                print >> f, """
 def test():
     raise ValueError"""
 
@@ -96,7 +99,8 @@
             # three seconds are needed for this test to pass reliably :-(
             time.sleep(4)
 
-            print >> open(testfile, 'w'), """
+            with open(testfile, 'w') as f:
+                print >> f, """
 def test():
     raise NotImplementedError"""
             reload(test_bug737473)
diff --git a/lib-python/2.7/test/test_types.py b/lib-python/2.7/test/test_types.py
--- a/lib-python/2.7/test/test_types.py
+++ b/lib-python/2.7/test/test_types.py
@@ -1,7 +1,8 @@
 # Python test set -- part 6, built-in types
 
 from test.test_support import run_unittest, have_unicode, run_with_locale, \
-                              check_py3k_warnings
+                              check_py3k_warnings, \
+                              impl_detail, check_impl_detail
 import unittest
 import sys
 import locale
@@ -289,9 +290,14 @@
         # array.array() returns an object that does not implement a char buffer,
         # something which int() uses for conversion.
         import array
-        try: int(buffer(array.array('c')))
+        try: int(buffer(array.array('c', '5')))
         except TypeError: pass
-        else: self.fail("char buffer (at C level) not working")
+        else:
+            if check_impl_detail():
+                self.fail("char buffer (at C level) not working")
+            #else:
+            #   it works on PyPy, which does not have the distinction
+            #   between char buffer and binary buffer.  XXX fine enough?
 
     def test_int__format__(self):
         def test(i, format_spec, result):
@@ -741,6 +747,7 @@
         for code in 'xXobns':
             self.assertRaises(ValueError, format, 0, ',' + code)
 
+    @impl_detail("the types' internal size attributes are CPython-only")
     def test_internal_sizes(self):
         self.assertGreater(object.__basicsize__, 0)
         self.assertGreater(tuple.__itemsize__, 0)
diff --git a/lib-python/2.7/test/test_unicode.py b/lib-python/2.7/test/test_unicode.py
--- a/lib-python/2.7/test/test_unicode.py
+++ b/lib-python/2.7/test/test_unicode.py
@@ -448,10 +448,11 @@
                 meth('\xff')
             with self.assertRaises(TypeError) as cm:
                 meth(['f'])
-            exc = str(cm.exception)
-            self.assertIn('unicode', exc)
-            self.assertIn('str', exc)
-            self.assertIn('tuple', exc)
+            if test_support.check_impl_detail():
+                exc = str(cm.exception)
+                self.assertIn('unicode', exc)
+                self.assertIn('str', exc)
+                self.assertIn('tuple', exc)
 
     @test_support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
     def test_format_float(self):
@@ -1062,7 +1063,8 @@
         # to take a 64-bit long, this test should apply to all platforms.
         if sys.maxint > (1 << 32) or struct.calcsize('P') != 4:
             return
-        self.assertRaises(OverflowError, u't\tt\t'.expandtabs, sys.maxint)
+        self.assertRaises((OverflowError, MemoryError),
+                          u't\tt\t'.expandtabs, sys.maxint)
 
     def test__format__(self):
         def test(value, format, expected):
diff --git a/lib-python/2.7/test/test_unicodedata.py b/lib-python/2.7/test/test_unicodedata.py
--- a/lib-python/2.7/test/test_unicodedata.py
+++ b/lib-python/2.7/test/test_unicodedata.py
@@ -233,10 +233,12 @@
         # been loaded in this process.
         popen = subprocess.Popen(args, stderr=subprocess.PIPE)
         popen.wait()
-        self.assertEqual(popen.returncode, 1)
-        error = "SyntaxError: (unicode error) \N escapes not supported " \
-            "(can't load unicodedata module)"
-        self.assertIn(error, popen.stderr.read())
+        self.assertIn(popen.returncode, [0, 1]) # at least it did not segfault
+        if test.test_support.check_impl_detail():
+            self.assertEqual(popen.returncode, 1)
+            error = "SyntaxError: (unicode error) \N escapes not supported " \
+                "(can't load unicodedata module)"
+            self.assertIn(error, popen.stderr.read())
 
     def test_decimal_numeric_consistent(self):
         # Test that decimal and numeric are consistent,
diff --git a/lib-python/2.7/test/test_unpack.py b/lib-python/2.7/test/test_unpack.py
--- a/lib-python/2.7/test/test_unpack.py
+++ b/lib-python/2.7/test/test_unpack.py
@@ -62,14 +62,14 @@
     >>> a, b = t
     Traceback (most recent call last):
       ...
-    ValueError: too many values to unpack
+    ValueError: expected length 2, got 3
 
 Unpacking tuple of wrong size
 
     >>> a, b = l
     Traceback (most recent call last):
       ...
-    ValueError: too many values to unpack
+    ValueError: expected length 2, got 3
 
 Unpacking sequence too short
 
diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py
--- a/lib-python/2.7/test/test_urllib2.py
+++ b/lib-python/2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/2.7/test/test_warnings.py b/lib-python/2.7/test/test_warnings.py
--- a/lib-python/2.7/test/test_warnings.py
+++ b/lib-python/2.7/test/test_warnings.py
@@ -355,7 +355,8 @@
     # test_support.import_fresh_module utility function
     def test_accelerated(self):
         self.assertFalse(original_warnings is self.module)
-        self.assertFalse(hasattr(self.module.warn, 'func_code'))
+        self.assertFalse(hasattr(self.module.warn, 'func_code') and
+                         hasattr(self.module.warn.func_code, 'co_filename'))
 
 class PyWarnTests(BaseTest, WarnTests):
     module = py_warnings
@@ -364,7 +365,8 @@
     # test_support.import_fresh_module utility function
     def test_pure_python(self):
         self.assertFalse(original_warnings is self.module)
-        self.assertTrue(hasattr(self.module.warn, 'func_code'))
+        self.assertTrue(hasattr(self.module.warn, 'func_code') and
+                        hasattr(self.module.warn.func_code, 'co_filename'))
 
 
 class WCmdLineTests(unittest.TestCase):
diff --git a/lib-python/2.7/test/test_weakref.py b/lib-python/2.7/test/test_weakref.py
--- a/lib-python/2.7/test/test_weakref.py
+++ b/lib-python/2.7/test/test_weakref.py
@@ -1,4 +1,3 @@
-import gc
 import sys
 import unittest
 import UserList
@@ -6,6 +5,7 @@
 import operator
 
 from test import test_support
+from test.test_support import gc_collect
 
 # Used in ReferencesTestCase.test_ref_created_during_del() .
 ref_from_del = None
@@ -70,6 +70,7 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del o
+        gc_collect()
         self.assertTrue(ref1() is None,
                      "expected reference to be invalidated")
         self.assertTrue(ref2() is None,
@@ -101,13 +102,16 @@
         ref1 = weakref.proxy(o, self.callback)
         ref2 = weakref.proxy(o, self.callback)
         del o
+        gc_collect()
 
         def check(proxy):
             proxy.bar
 
         self.assertRaises(weakref.ReferenceError, check, ref1)
         self.assertRaises(weakref.ReferenceError, check, ref2)
-        self.assertRaises(weakref.ReferenceError, bool, weakref.proxy(C()))
+        ref3 = weakref.proxy(C())
+        gc_collect()
+        self.assertRaises(weakref.ReferenceError, bool, ref3)
         self.assertTrue(self.cbcalled == 2)
 
     def check_basic_ref(self, factory):
@@ -124,6 +128,7 @@
         o = factory()
         ref = weakref.ref(o, self.callback)
         del o
+        gc_collect()
         self.assertTrue(self.cbcalled == 1,
                      "callback did not properly set 'cbcalled'")
         self.assertTrue(ref() is None,
@@ -148,6 +153,7 @@
         self.assertTrue(weakref.getweakrefcount(o) == 2,
                      "wrong weak ref count for object")
         del proxy
+        gc_collect()
         self.assertTrue(weakref.getweakrefcount(o) == 1,
                      "wrong weak ref count for object after deleting proxy")
 
@@ -325,6 +331,7 @@
                      "got wrong number of weak reference objects")
 
         del ref1, ref2, proxy1, proxy2
+        gc_collect()
         self.assertTrue(weakref.getweakrefcount(o) == 0,
                      "weak reference objects not unlinked from"
                      " referent when discarded.")
@@ -338,6 +345,7 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del ref1
+        gc_collect()
         self.assertTrue(weakref.getweakrefs(o) == [ref2],
                      "list of refs does not match")
 
@@ -345,10 +353,12 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del ref2
+        gc_collect()
         self.assertTrue(weakref.getweakrefs(o) == [ref1],
                      "list of refs does not match")
 
         del ref1
+        gc_collect()
         self.assertTrue(weakref.getweakrefs(o) == [],
                      "list of refs not cleared")
 
@@ -400,13 +410,11 @@
         # when the second attempt to remove the instance from the "list
         # of all objects" occurs.
 
-        import gc
-
         class C(object):
             pass
 
         c = C()
-        wr = weakref.ref(c, lambda ignore: gc.collect())
+        wr = weakref.ref(c, lambda ignore: gc_collect())
         del c
 
         # There endeth the first part.  It gets worse.
@@ -414,7 +422,7 @@
 
         c1 = C()
         c1.i = C()
-        wr = weakref.ref(c1.i, lambda ignore: gc.collect())
+        wr = weakref.ref(c1.i, lambda ignore: gc_collect())
 
         c2 = C()
         c2.c1 = c1
@@ -430,8 +438,6 @@
         del c2
 
     def test_callback_in_cycle_1(self):
-        import gc
-
         class J(object):
             pass
 
@@ -467,11 +473,9 @@
         # search II.__mro__, but that's NULL.   The result was a segfault in
         # a release build, and an assert failure in a debug build.
         del I, J, II
-        gc.collect()
+        gc_collect()
 
     def test_callback_in_cycle_2(self):
-        import gc
-
         # This is just like test_callback_in_cycle_1, except that II is an
         # old-style class.  The symptom is different then:  an instance of an
         # old-style class looks in its own __dict__ first.  'J' happens to
@@ -496,11 +500,9 @@
         I.wr = weakref.ref(J, I.acallback)
 
         del I, J, II
-        gc.collect()
+        gc_collect()
 
     def test_callback_in_cycle_3(self):
-        import gc
-
         # This one broke the first patch that fixed the last two.  In this
         # case, the objects reachable from the callback aren't also reachable
         # from the object (c1) *triggering* the callback:  you can get to
@@ -520,11 +522,9 @@
         c2.wr = weakref.ref(c1, c2.cb)
 
         del c1, c2
-        gc.collect()
+        gc_collect()
 
     def test_callback_in_cycle_4(self):
-        import gc
-
         # Like test_callback_in_cycle_3, except c2 and c1 have different
         # classes.  c2's class (C) isn't reachable from c1 then, so protecting
         # objects reachable from the dying object (c1) isn't enough to stop
@@ -548,11 +548,9 @@
         c2.wr = weakref.ref(c1, c2.cb)
 
         del c1, c2, C, D
-        gc.collect()
+        gc_collect()
 
     def test_callback_in_cycle_resurrection(self):
-        import gc
-
         # Do something nasty in a weakref callback:  resurrect objects
         # from dead cycles.  For this to be attempted, the weakref and
         # its callback must also be part of the cyclic trash (else the
@@ -583,7 +581,7 @@
         del c1, c2, C   # make them all trash
         self.assertEqual(alist, [])  # del isn't enough to reclaim anything
 
-        gc.collect()
+        gc_collect()
         # c1.wr and c2.wr were part of the cyclic trash, so should have
         # been cleared without their callbacks executing.  OTOH, the weakref
         # to C is bound to a function local (wr), and wasn't trash, so that
@@ -593,12 +591,10 @@
         self.assertEqual(wr(), None)
 
         del alist[:]
-        gc.collect()
+        gc_collect()
         self.assertEqual(alist, [])
 
     def test_callbacks_on_callback(self):
-        import gc
-
         # Set up weakref callbacks *on* weakref callbacks.
         alist = []
         def safe_callback(ignore):
@@ -626,12 +622,12 @@
 
         del callback, c, d, C
         self.assertEqual(alist, [])  # del isn't enough to clean up cycles
-        gc.collect()
+        gc_collect()
         self.assertEqual(alist, ["safe_callback called"])
         self.assertEqual(external_wr(), None)
 
         del alist[:]
-        gc.collect()
+        gc_collect()
         self.assertEqual(alist, [])
 
     def test_gc_during_ref_creation(self):
@@ -641,9 +637,11 @@
         self.check_gc_during_creation(weakref.proxy)
 
     def check_gc_during_creation(self, makeref):
-        thresholds = gc.get_threshold()
-        gc.set_threshold(1, 1, 1)
-        gc.collect()
+        if test_support.check_impl_detail():
+            import gc
+            thresholds = gc.get_threshold()
+            gc.set_threshold(1, 1, 1)
+        gc_collect()
         class A:
             pass
 
@@ -663,7 +661,8 @@
             weakref.ref(referenced, callback)
 
         finally:
-            gc.set_threshold(*thresholds)
+            if test_support.check_impl_detail():
+                gc.set_threshold(*thresholds)
 
     def test_ref_created_during_del(self):
         # Bug #1377858
@@ -683,7 +682,7 @@
         r = weakref.ref(Exception)
         self.assertRaises(TypeError, r.__init__, 0, 0, 0, 0, 0)
         # No exception should be raised here
-        gc.collect()
+        gc_collect()
 
     def test_classes(self):
         # Check that both old-style classes and new-style classes
@@ -696,12 +695,12 @@
         weakref.ref(int)
         a = weakref.ref(A, l.append)
         A = None
-        gc.collect()
+        gc_collect()
         self.assertEqual(a(), None)
         self.assertEqual(l, [a])
         b = weakref.ref(B, l.append)
         B = None
-        gc.collect()
+        gc_collect()
         self.assertEqual(b(), None)
         self.assertEqual(l, [a, b])
 
@@ -722,6 +721,7 @@
         self.assertTrue(mr.called)
         self.assertEqual(mr.value, 24)
         del o
+        gc_collect()
         self.assertTrue(mr() is None)
         self.assertTrue(mr.called)
 
@@ -738,9 +738,11 @@
         self.assertEqual(weakref.getweakrefcount(o), 3)
         refs = weakref.getweakrefs(o)
         self.assertEqual(len(refs), 3)
-        self.assertTrue(r2 is refs[0])
-        self.assertIn(r1, refs[1:])
-        self.assertIn(r3, refs[1:])
+        assert set(refs) == set((r1, r2, r3))
+        if test_support.check_impl_detail():
+            self.assertTrue(r2 is refs[0])
+            self.assertIn(r1, refs[1:])
+            self.assertIn(r3, refs[1:])
 
     def test_subclass_refs_dont_conflate_callbacks(self):
         class MyRef(weakref.ref):
@@ -839,15 +841,18 @@
         del items1, items2
         self.assertTrue(len(dict) == self.COUNT)
         del objects[0]
+        gc_collect()
         self.assertTrue(len(dict) == (self.COUNT - 1),
                      "deleting object did not cause dictionary update")
         del objects, o
+        gc_collect()
         self.assertTrue(len(dict) == 0,
                      "deleting the values did not clear the dictionary")
         # regression on SF bug #447152:
         dict = weakref.WeakValueDictionary()
         self.assertRaises(KeyError, dict.__getitem__, 1)
         dict[2] = C()
+        gc_collect()
         self.assertRaises(KeyError, dict.__getitem__, 2)
 
     def test_weak_keys(self):
@@ -868,9 +873,11 @@
         del items1, items2
         self.assertTrue(len(dict) == self.COUNT)
         del objects[0]
+        gc_collect()
         self.assertTrue(len(dict) == (self.COUNT - 1),
                      "deleting object did not cause dictionary update")
         del objects, o
+        gc_collect()
         self.assertTrue(len(dict) == 0,
                      "deleting the keys did not clear the dictionary")
         o = Object(42)
@@ -986,13 +993,13 @@
         self.assertTrue(len(weakdict) == 2)
         k, v = weakdict.popitem()
         self.assertTrue(len(weakdict) == 1)
-        if k is key1:
+        if k == key1:
             self.assertTrue(v is value1)
         else:
             self.assertTrue(v is value2)
         k, v = weakdict.popitem()
         self.assertTrue(len(weakdict) == 0)
-        if k is key1:
+        if k == key1:
             self.assertTrue(v is value1)
         else:
             self.assertTrue(v is value2)
@@ -1137,6 +1144,7 @@
         for o in objs:
             count += 1
             del d[o]
+        gc_collect()
         self.assertEqual(len(d), 0)
         self.assertEqual(count, 2)
 
@@ -1177,6 +1185,7 @@
 >>> o is o2
 True
 >>> del o, o2
+>>> gc_collect()
 >>> print r()
 None
 
@@ -1229,6 +1238,7 @@
 >>> id2obj(a_id) is a
 True
 >>> del a
+>>> gc_collect()
 >>> try:
 ...     id2obj(a_id)
 ... except KeyError:
diff --git a/lib-python/2.7/test/test_weakset.py b/lib-python/2.7/test/test_weakset.py
--- a/lib-python/2.7/test/test_weakset.py
+++ b/lib-python/2.7/test/test_weakset.py
@@ -57,6 +57,7 @@
         self.assertEqual(len(self.s), len(self.d))
         self.assertEqual(len(self.fs), 1)
         del self.obj
+        test_support.gc_collect()
         self.assertEqual(len(self.fs), 0)
 
     def test_contains(self):
@@ -66,6 +67,7 @@
         self.assertNotIn(1, self.s)
         self.assertIn(self.obj, self.fs)
         del self.obj
+        test_support.gc_collect()
         self.assertNotIn(SomeClass('F'), self.fs)
 
     def test_union(self):
@@ -204,6 +206,7 @@
         self.assertEqual(self.s, dup)
         self.assertRaises(TypeError, self.s.add, [])
         self.fs.add(Foo())
+        test_support.gc_collect()
         self.assertTrue(len(self.fs) == 1)
         self.fs.add(self.obj)
         self.assertTrue(len(self.fs) == 1)
@@ -330,10 +333,11 @@
         next(it)             # Trigger internal iteration
         # Destroy an item
         del items[-1]
-        gc.collect()    # just in case
+        test_support.gc_collect()
         # We have removed either the first consumed items, or another one
         self.assertIn(len(list(it)), [len(items), len(items) - 1])
         del it
+        test_support.gc_collect()
         # The removal has been committed
         self.assertEqual(len(s), len(items))
 
diff --git a/lib-python/2.7/test/test_xml_etree.py b/lib-python/2.7/test/test_xml_etree.py
--- a/lib-python/2.7/test/test_xml_etree.py
+++ b/lib-python/2.7/test/test_xml_etree.py
@@ -1633,10 +1633,10 @@
 
     Check reference leak.
     >>> xmltoolkit63()
-    >>> count = sys.getrefcount(None)
+    >>> count = sys.getrefcount(None)  #doctest: +SKIP
     >>> for i in range(1000):
     ...     xmltoolkit63()
-    >>> sys.getrefcount(None) - count
+    >>> sys.getrefcount(None) - count  #doctest: +SKIP
     0
 
     """
diff --git a/lib-python/2.7/test/test_xmlrpc.py b/lib-python/2.7/test/test_xmlrpc.py
--- a/lib-python/2.7/test/test_xmlrpc.py
+++ b/lib-python/2.7/test/test_xmlrpc.py
@@ -308,7 +308,7 @@
         global ADDR, PORT, URL
         ADDR, PORT = serv.socket.getsockname()
         #connect to IP address directly.  This avoids socket.create_connection()
-        #trying to connect to "localhost" using all address families, which
+        #trying to connect to to "localhost" using all address families, which
         #causes slowdown e.g. on vista which supports AF_INET6.  The server listens
         #on AF_INET only.
         URL = "http://%s:%d"%(ADDR, PORT)
@@ -367,7 +367,7 @@
         global ADDR, PORT, URL
         ADDR, PORT = serv.socket.getsockname()
         #connect to IP address directly.  This avoids socket.create_connection()
-        #trying to connect to "localhost" using all address families, which
+        #trying to connect to to "localhost" using all address families, which
         #causes slowdown e.g. on vista which supports AF_INET6.  The server listens
         #on AF_INET only.
         URL = "http://%s:%d"%(ADDR, PORT)
@@ -435,6 +435,7 @@
 
     def tearDown(self):
         # wait on the server thread to terminate
+        test_support.gc_collect() # to close the active connections
         self.evt.wait(10)
 
         # disable traceback reporting
@@ -472,9 +473,6 @@
                 # protocol error; provide additional information in test output
                 self.fail("%s\n%s" % (e, getattr(e, "headers", "")))
 
-    def test_unicode_host(self):
-        server = xmlrpclib.ServerProxy(u"http://%s:%d/RPC2"%(ADDR, PORT))
-        self.assertEqual(server.add("a", u"\xe9"), u"a\xe9")
 
     # [ch] The test 404 is causing lots of false alarms.
     def XXXtest_404(self):
@@ -589,12 +587,6 @@
         # This avoids waiting for the socket timeout.
         self.test_simple1()
 
-    def test_partial_post(self):
-        # Check that a partial POST doesn't make the server loop: issue #14001.
-        conn = httplib.HTTPConnection(ADDR, PORT)
-        conn.request('POST', '/RPC2 HTTP/1.0\r\nContent-Length: 100\r\n\r\nbye')
-        conn.close()
-
 class MultiPathServerTestCase(BaseServerTestCase):
     threadFunc = staticmethod(http_multi_server)
     request_count = 2
diff --git a/lib-python/2.7/test/test_zlib.py b/lib-python/2.7/test/test_zlib.py
--- a/lib-python/2.7/test/test_zlib.py
+++ b/lib-python/2.7/test/test_zlib.py
@@ -1,6 +1,7 @@
 import unittest
 from test.test_support import TESTFN, run_unittest, import_module, unlink, requires
 import binascii
+import os
 import random
 from test.test_support import precisionbigmemtest, _1G, _4G
 import sys
@@ -99,14 +100,7 @@
 
 class BaseCompressTestCase(object):
     def check_big_compress_buffer(self, size, compress_func):
-        _1M = 1024 * 1024
-        fmt = "%%0%dx" % (2 * _1M)
-        # Generate 10MB worth of random, and expand it by repeating it.
-        # The assumption is that zlib's memory is not big enough to exploit
-        # such spread out redundancy.
-        data = ''.join([binascii.a2b_hex(fmt % random.getrandbits(8 * _1M))
-                        for i in range(10)])
-        data = data * (size // len(data) + 1)
+        data = os.urandom(size)
         try:
             compress_func(data)
         finally:
diff --git a/lib-python/2.7/trace.py b/lib-python/2.7/trace.py
--- a/lib-python/2.7/trace.py
+++ b/lib-python/2.7/trace.py
@@ -559,6 +559,10 @@
             if len(funcs) == 1:
                 dicts = [d for d in gc.get_referrers(funcs[0])
                              if isinstance(d, dict)]
+                if len(dicts) == 0:
+                    # PyPy may store functions directly on the class
+                    # (more exactly: the container is not a Python object)
+                    dicts = funcs
                 if len(dicts) == 1:
                     classes = [c for c in gc.get_referrers(dicts[0])
                                    if hasattr(c, "__bases__")]
diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py
--- a/lib-python/2.7/urllib2.py
+++ b/lib-python/2.7/urllib2.py
@@ -1171,6 +1171,7 @@
             except TypeError: #buffering kw not supported
                 r = h.getresponse()
         except socket.error, err: # XXX what error?
+            h.close()
             raise URLError(err)
 
         # Pick apart the HTTPResponse object to get the addinfourl
diff --git a/lib-python/2.7/uuid.py b/lib-python/2.7/uuid.py
--- a/lib-python/2.7/uuid.py
+++ b/lib-python/2.7/uuid.py
@@ -406,8 +406,12 @@
             continue
         if hasattr(lib, 'uuid_generate_random'):
             _uuid_generate_random = lib.uuid_generate_random
+            _uuid_generate_random.argtypes = [ctypes.c_char * 16]
+            _uuid_generate_random.restype = None
         if hasattr(lib, 'uuid_generate_time'):
             _uuid_generate_time = lib.uuid_generate_time
+            _uuid_generate_time.argtypes = [ctypes.c_char * 16]
+            _uuid_generate_time.restype = None
 
     # The uuid_generate_* functions are broken on MacOS X 10.5, as noted
     # in issue #8621 the function generates the same sequence of values
@@ -436,6 +440,9 @@
         lib = None
     _UuidCreate = getattr(lib, 'UuidCreateSequential',
                           getattr(lib, 'UuidCreate', None))
+    if _UuidCreate is not None:
+        _UuidCreate.argtypes = [ctypes.c_char * 16]
+        _UuidCreate.restype = ctypes.c_int
 except:
     pass
 
diff --git a/lib-python/3.2/_markupbase.py b/lib-python/3.2/_markupbase.py
--- a/lib-python/3.2/_markupbase.py
+++ b/lib-python/3.2/_markupbase.py
@@ -107,6 +107,10 @@
                 if decltype == "doctype":
                     self.handle_decl(data)
                 else:
+                    # According to the HTML5 specs sections "8.2.4.44 Bogus
+                    # comment state" and "8.2.4.45 Markup declaration open
+                    # state", a comment token should be emitted.
+                    # Calling unknown_decl provides more flexibility though.
                     self.unknown_decl(data)
                 return j + 1
             if c in "\"'":
diff --git a/lib-python/3.2/_pyio.py b/lib-python/3.2/_pyio.py
--- a/lib-python/3.2/_pyio.py
+++ b/lib-python/3.2/_pyio.py
@@ -6,6 +6,7 @@
 import abc
 import codecs
 import warnings
+import errno
 # Import _thread instead of threading to reduce startup cost
 try:
     from _thread import allocate_lock as Lock
@@ -720,8 +721,11 @@
 
     def close(self):
         if self.raw is not None and not self.closed:
-            self.flush()
-            self.raw.close()
+            try:
+                # may raise BlockingIOError or BrokenPipeError etc
+                self.flush()
+            finally:
+                self.raw.close()
 
     def detach(self):
         if self.raw is None:
@@ -1080,13 +1084,9 @@
             # XXX we can implement some more tricks to try and avoid
             # partial writes
             if len(self._write_buf) > self.buffer_size:
-                # We're full, so let's pre-flush the buffer
-                try:
-                    self._flush_unlocked()
-                except BlockingIOError as e:
-                    # We can't accept anything else.
-                    # XXX Why not just let the exception pass through?
-                    raise BlockingIOError(e.errno, e.strerror, 0)
+                # We're full, so let's pre-flush the buffer.  (This may
+                # raise BlockingIOError with characters_written == 0.)
+                self._flush_unlocked()
             before = len(self._write_buf)
             self._write_buf.extend(b)
             written = len(self._write_buf) - before
@@ -1117,24 +1117,23 @@
     def _flush_unlocked(self):
         if self.closed:
             raise ValueError("flush of closed file")
-        written = 0
-        try:
-            while self._write_buf:
-                try:
-                    n = self.raw.write(self._write_buf)
-                except IOError as e:
-                    if e.errno != EINTR:
-                        raise
-                    continue
-                if n > len(self._write_buf) or n < 0:
-                    raise IOError("write() returned incorrect number of bytes")
-                del self._write_buf[:n]
-                written += n
-        except BlockingIOError as e:
-            n = e.characters_written
+        while self._write_buf:
+            try:
+                n = self.raw.write(self._write_buf)
+            except BlockingIOError:
+                raise RuntimeError("self.raw should implement RawIOBase: it "
+                                   "should not raise BlockingIOError")
+            except IOError as e:
+                if e.errno != EINTR:
+                    raise
+                continue
+            if n is None:
+                raise BlockingIOError(
+                    errno.EAGAIN,
+                    "write could not complete without blocking", 0)
+            if n > len(self._write_buf) or n < 0:
+                raise IOError("write() returned incorrect number of bytes")
             del self._write_buf[:n]
-            written += n
-            raise BlockingIOError(e.errno, e.strerror, written)
 
     def tell(self):
         return _BufferedIOMixin.tell(self) + len(self._write_buf)
@@ -1460,7 +1459,7 @@
     enabled.  With this enabled, on input, the lines endings '\n', '\r',
     or '\r\n' are translated to '\n' before being returned to the
     caller. Conversely, on output, '\n' is translated to the system
-    default line seperator, os.linesep. If newline is any other of its
+    default line separator, os.linesep. If newline is any other of its
     legal values, that newline becomes the newline when the file is read
     and it is returned untranslated. On output, '\n' is converted to the
     newline.
diff --git a/lib-python/3.2/aifc.py b/lib-python/3.2/aifc.py
--- a/lib-python/3.2/aifc.py
+++ b/lib-python/3.2/aifc.py
@@ -162,6 +162,12 @@
     except struct.error:
         raise EOFError
 
+def _read_ushort(file):
+    try:
+        return struct.unpack('>H', file.read(2))[0]
+    except struct.error:
+        raise EOFError
+
 def _read_string(file):
     length = ord(file.read(1))
     if length == 0:
@@ -194,13 +200,19 @@
 def _write_short(f, x):
     f.write(struct.pack('>h', x))
 
+def _write_ushort(f, x):
+    f.write(struct.pack('>H', x))
+
 def _write_long(f, x):
+    f.write(struct.pack('>l', x))
+
+def _write_ulong(f, x):
     f.write(struct.pack('>L', x))
 
 def _write_string(f, s):
     if len(s) > 255:
         raise ValueError("string exceeds maximum pstring length")
-    f.write(struct.pack('b', len(s)))
+    f.write(struct.pack('B', len(s)))
     f.write(s)
     if len(s) & 1 == 0:
         f.write(b'\x00')
@@ -218,7 +230,7 @@
         lomant = 0
     else:
         fmant, expon = math.frexp(x)
-        if expon > 16384 or fmant >= 1:     # Infinity or NaN
+        if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN
             expon = sign|0x7FFF
             himant = 0
             lomant = 0
@@ -234,9 +246,9 @@
             fmant = math.ldexp(fmant - fsmant, 32)
             fsmant = math.floor(fmant)
             lomant = int(fsmant)
-    _write_short(f, expon)
-    _write_long(f, himant)
-    _write_long(f, lomant)
+    _write_ushort(f, expon)
+    _write_ulong(f, himant)
+    _write_ulong(f, lomant)
 
 from chunk import Chunk
 
@@ -539,8 +551,7 @@
         self._aifc = 1      # AIFF-C is default
 
     def __del__(self):
-        if self._file:
-            self.close()
+        self.close()
 
     #
     # User visible methods.
@@ -643,8 +654,8 @@
             raise Error('marker ID must be > 0')
         if pos < 0:
             raise Error('marker position must be >= 0')
-        if not isinstance(name, str):
-            raise Error('marker name must be a string')
+        if not isinstance(name, bytes):
+            raise Error('marker name must be bytes')
         for i in range(len(self._markers)):
             if id == self._markers[i][0]:
                 self._markers[i] = id, pos, name
@@ -681,19 +692,21 @@
             self._patchheader()
 
     def close(self):
-        self._ensure_header_written(0)
-        if self._datawritten & 1:
-            # quick pad to even size
-            self._file.write(b'\x00')
-            self._datawritten = self._datawritten + 1
-        self._writemarkers()
-        if self._nframeswritten != self._nframes or \
-              self._datalength != self._datawritten or \
-              self._marklength:
-            self._patchheader()
-        # Prevent ref cycles
-        self._convert = None
-        self._file.close()
+        if self._file:
+            self._ensure_header_written(0)
+            if self._datawritten & 1:
+                # quick pad to even size
+                self._file.write(b'\x00')
+                self._datawritten = self._datawritten + 1
+            self._writemarkers()
+            if self._nframeswritten != self._nframes or \
+                  self._datalength != self._datawritten or \
+                  self._marklength:
+                self._patchheader()
+            # Prevent ref cycles
+            self._convert = None
+            self._file.close()
+            self._file = None
 
     #
     # Internal methods.
@@ -716,18 +729,12 @@
 
     def _ensure_header_written(self, datasize):
         if not self._nframeswritten:
-            if self._comptype in (b'ULAW', b'ALAW'):
+            if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
                 if not self._sampwidth:
                     self._sampwidth = 2
                 if self._sampwidth != 2:
                     raise Error('sample width must be 2 when compressing '
-                                'with ulaw/ULAW or alaw/ALAW')
-            if self._comptype == b'G722':
-                if not self._sampwidth:
-                    self._sampwidth = 2
-                if self._sampwidth != 2:
-                    raise Error('sample width must be 2 when compressing '
-                                'with G7.22 (ADPCM)')
+                                'with ulaw/ULAW, alaw/ALAW or G7.22 (ADPCM)')
             if not self._nchannels:
                 raise Error('# channels not specified')
             if not self._sampwidth:
@@ -743,8 +750,6 @@
             self._convert = self._lin2ulaw
         elif self._comptype in (b'alaw', b'ALAW'):
             self._convert = self._lin2alaw
-        else:
-            raise Error('unsupported compression type')
 
     def _write_header(self, initlength):
         if self._aifc and self._comptype != b'NONE':
@@ -769,15 +774,15 @@
         if self._aifc:
             self._file.write(b'AIFC')
             self._file.write(b'FVER')
-            _write_long(self._file, 4)
-            _write_long(self._file, self._version)
+            _write_ulong(self._file, 4)
+            _write_ulong(self._file, self._version)
         else:
             self._file.write(b'AIFF')
         self._file.write(b'COMM')
-        _write_long(self._file, commlength)
+        _write_ulong(self._file, commlength)
         _write_short(self._file, self._nchannels)
         self._nframes_pos = self._file.tell()
-        _write_long(self._file, self._nframes)
+        _write_ulong(self._file, self._nframes)
         _write_short(self._file, self._sampwidth * 8)
         _write_float(self._file, self._framerate)
         if self._aifc:
@@ -785,9 +790,9 @@
             _write_string(self._file, self._compname)
         self._file.write(b'SSND')
         self._ssnd_length_pos = self._file.tell()
-        _write_long(self._file, self._datalength + 8)
-        _write_long(self._file, 0)
-        _write_long(self._file, 0)
+        _write_ulong(self._file, self._datalength + 8)
+        _write_ulong(self._file, 0)
+        _write_ulong(self._file, 0)
 
     def _write_form_length(self, datalength):
         if self._aifc:
@@ -798,8 +803,8 @@
         else:
             commlength = 18
             verslength = 0
-        _write_long(self._file, 4 + verslength + self._marklength + \
-                    8 + commlength + 16 + datalength)
+        _write_ulong(self._file, 4 + verslength + self._marklength + \
+                     8 + commlength + 16 + datalength)
         return commlength
 
     def _patchheader(self):
@@ -817,9 +822,9 @@
         self._file.seek(self._form_length_pos, 0)
         dummy = self._write_form_length(datalength)
         self._file.seek(self._nframes_pos, 0)
-        _write_long(self._file, self._nframeswritten)
+        _write_ulong(self._file, self._nframeswritten)
         self._file.seek(self._ssnd_length_pos, 0)
-        _write_long(self._file, datalength + 8)
+        _write_ulong(self._file, datalength + 8)
         self._file.seek(curpos, 0)
         self._nframes = self._nframeswritten
         self._datalength = datalength
@@ -834,13 +839,13 @@
             length = length + len(name) + 1 + 6
             if len(name) & 1 == 0:
                 length = length + 1
-        _write_long(self._file, length)
+        _write_ulong(self._file, length)
         self._marklength = length + 8
         _write_short(self._file, len(self._markers))
         for marker in self._markers:
             id, pos, name = marker
             _write_short(self._file, id)
-            _write_long(self._file, pos)
+            _write_ulong(self._file, pos)
             _write_string(self._file, name)
 
 def open(f, mode=None):
diff --git a/lib-python/3.2/argparse.py b/lib-python/3.2/argparse.py
--- a/lib-python/3.2/argparse.py
+++ b/lib-python/3.2/argparse.py
@@ -92,10 +92,6 @@
 from gettext import gettext as _, ngettext
 
 
-def _callable(obj):
-    return hasattr(obj, '__call__') or hasattr(obj, '__bases__')
-
-
 SUPPRESS = '==SUPPRESS=='
 
 OPTIONAL = '?'
@@ -1286,13 +1282,13 @@
 
         # create the action object, and add it to the parser
         action_class = self._pop_action_class(kwargs)
-        if not _callable(action_class):
+        if not callable(action_class):
             raise ValueError('unknown action "%s"' % (action_class,))
         action = action_class(**kwargs)
 
         # raise an error if the action type is not callable
         type_func = self._registry_get('type', action.type, action.type)
-        if not _callable(type_func):
+        if not callable(type_func):
             raise ValueError('%r is not callable' % (type_func,))
 
         # raise an error if the metavar does not match the type
@@ -2240,7 +2236,7 @@
 
     def _get_value(self, action, arg_string):
         type_func = self._registry_get('type', action.type, action.type)
-        if not _callable(type_func):
+        if not callable(type_func):
             msg = _('%r is not callable')
             raise ArgumentError(action, msg % type_func)
 
diff --git a/lib-python/3.2/base64.py b/lib-python/3.2/base64.py
old mode 100644
new mode 100755
diff --git a/lib-python/3.2/cProfile.py b/lib-python/3.2/cProfile.py
old mode 100644
new mode 100755
diff --git a/lib-python/3.2/cgi.py b/lib-python/3.2/cgi.py
old mode 100644
new mode 100755
--- a/lib-python/3.2/cgi.py
+++ b/lib-python/3.2/cgi.py
@@ -291,7 +291,7 @@
     while s[:1] == ';':
         s = s[1:]
         end = s.find(';')
-        while end > 0 and s.count('"', 0, end) % 2:
+        while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2:
             end = s.find(';', end + 1)
         if end < 0:
             end = len(s)
diff --git a/lib-python/3.2/cmd.py b/lib-python/3.2/cmd.py
--- a/lib-python/3.2/cmd.py
+++ b/lib-python/3.2/cmd.py
@@ -205,6 +205,8 @@
         if cmd is None:
             return self.default(line)
         self.lastcmd = line
+        if line == 'EOF' :
+            self.lastcmd = ''
         if cmd == '':
             return self.default(line)
         else:
diff --git a/lib-python/3.2/collections.py b/lib-python/3.2/collections.py
--- a/lib-python/3.2/collections.py
+++ b/lib-python/3.2/collections.py
@@ -33,7 +33,7 @@
     # The circular doubly linked list starts and ends with a sentinel element.
     # The sentinel element never gets deleted (this simplifies the algorithm).
     # The sentinel is in self.__hardroot with a weakref proxy in self.__root.
-    # The prev/next links are weakref proxies (to prevent circular references).
+    # The prev links are weakref proxies (to prevent circular references).
     # Individual links are kept alive by the hard reference in self.__map.
     # Those hard references disappear when a key is deleted from an OrderedDict.
 
@@ -583,8 +583,12 @@
     def __repr__(self):
         if not self:
             return '%s()' % self.__class__.__name__
-        items = ', '.join(map('%r: %r'.__mod__, self.most_common()))
-        return '%s({%s})' % (self.__class__.__name__, items)
+        try:
+            items = ', '.join(map('%r: %r'.__mod__, self.most_common()))
+            return '%s({%s})' % (self.__class__.__name__, items)
+        except TypeError:
+            # handle case where values are not orderable
+            return '{0}({1!r})'.format(self.__class__.__name__, dict(self))
 
     # Multiset-style mathematical operations discussed in:
     #       Knuth TAOCP Volume II section 4.6.3 exercise 19
diff --git a/lib-python/3.2/compileall.py b/lib-python/3.2/compileall.py
--- a/lib-python/3.2/compileall.py
+++ b/lib-python/3.2/compileall.py
@@ -142,7 +142,7 @@
 
     Arguments (all optional):
 
-    skip_curdir: if true, skip current directory (default true)
+    skip_curdir: if true, skip current directory (default True)
     maxlevels:   max recursion level (default 0)
     force: as for compile_dir() (default False)
     quiet: as for compile_dir() (default False)
@@ -177,17 +177,17 @@
                         help='use legacy (pre-PEP3147) compiled file locations')
     parser.add_argument('-d', metavar='DESTDIR',  dest='ddir', default=None,
                         help=('directory to prepend to file paths for use in '
-                              'compile time tracebacks and in runtime '
+                              'compile-time tracebacks and in runtime '
                               'tracebacks in cases where the source file is '
                               'unavailable'))
     parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
-                        help=('skip files matching the regular expression. '
-                              'The regexp is searched for in the full path '
-                              'to each file considered for compilation.'))
+                        help=('skip files matching the regular expression; '
+                              'the regexp is searched for in the full path '
+                              'of each file considered for compilation'))
     parser.add_argument('-i', metavar='FILE', dest='flist',
                         help=('add all the files and directories listed in '
-                              'FILE to the list considered for compilation. '
-                              'If "-", names are read from stdin.'))
+                              'FILE to the list considered for compilation; '
+                              'if "-", names are read from stdin'))
     parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
                         help=('zero or more file and directory names '
                               'to compile; if no arguments given, defaults '
diff --git a/lib-python/3.2/concurrent/futures/process.py b/lib-python/3.2/concurrent/futures/process.py
--- a/lib-python/3.2/concurrent/futures/process.py
+++ b/lib-python/3.2/concurrent/futures/process.py
@@ -213,9 +213,7 @@
                 work_item.future.set_exception(result_item.exception)
             else:
                 work_item.future.set_result(result_item.result)
-            continue
-        # If we come here, we either got a timeout or were explicitly woken up.
-        # In either case, check whether we should start shutting down.
+        # Check whether we should start shutting down.
         executor = executor_reference()
         # No more work items can be added if:
         #   - The interpreter is shutting down OR
@@ -234,9 +232,6 @@
                     p.join()
                 call_queue.close()
                 return
-            else:
-                # Start shutting down by telling a process it can exit.
-                shutdown_one_process()
         del executor
 
 _system_limits_checked = False
diff --git a/lib-python/3.2/configparser.py b/lib-python/3.2/configparser.py
--- a/lib-python/3.2/configparser.py
+++ b/lib-python/3.2/configparser.py
@@ -381,7 +381,7 @@
 
     would resolve the "%(dir)s" to the value of dir.  All reference
     expansions are done late, on demand. If a user needs to use a bare % in
-    a configuration file, she can escape it by writing %%. Other other % usage
+    a configuration file, she can escape it by writing %%. Other % usage
     is considered a user error and raises `InterpolationSyntaxError'."""
 
     _KEYCRE = re.compile(r"%\(([^)]+)\)s")
diff --git a/lib-python/3.2/copyreg.py b/lib-python/3.2/copyreg.py
--- a/lib-python/3.2/copyreg.py
+++ b/lib-python/3.2/copyreg.py
@@ -10,7 +10,7 @@
 dispatch_table = {}
 
 def pickle(ob_type, pickle_function, constructor_ob=None):
-    if not hasattr(pickle_function, '__call__'):
+    if not callable(pickle_function):
         raise TypeError("reduction functions must be callable")
     dispatch_table[ob_type] = pickle_function
 
@@ -20,7 +20,7 @@
         constructor(constructor_ob)
 
 def constructor(object):
-    if not hasattr(object, '__call__'):
+    if not callable(object):
         raise TypeError("constructors must be callable")
 
 # Example: provide pickling support for complex numbers.
diff --git a/lib-python/3.2/ctypes/__init__.py b/lib-python/3.2/ctypes/__init__.py
--- a/lib-python/3.2/ctypes/__init__.py
+++ b/lib-python/3.2/ctypes/__init__.py
@@ -265,7 +265,21 @@
 class c_wchar(_SimpleCData):
     _type_ = "u"
 
-POINTER(c_wchar).from_param = c_wchar_p.from_param #_SimpleCData.c_wchar_p_from_param
+def _reset_cache():
+    _pointer_type_cache.clear()
+    _c_functype_cache.clear()
+    if _os.name in ("nt", "ce"):
+        _win_functype_cache.clear()
+    # _SimpleCData.c_wchar_p_from_param
+    POINTER(c_wchar).from_param = c_wchar_p.from_param
+    # _SimpleCData.c_char_p_from_param
+    POINTER(c_char).from_param = c_char_p.from_param
+    _pointer_type_cache[None] = c_void_p
+    # XXX for whatever reasons, creating the first instance of a callback
+    # function is needed for the unittests on Win64 to succeed.  This MAY
+    # be a compiler bug, since the problem occurs only when _ctypes is
+    # compiled with the MS SDK compiler.  Or an uninitialized variable?
+    CFUNCTYPE(c_int)(lambda: None)
 
 def create_unicode_buffer(init, size=None):
     """create_unicode_buffer(aString) -> character array
@@ -285,7 +299,6 @@
         return buf
     raise TypeError(init)
 
-POINTER(c_char).from_param = c_char_p.from_param #_SimpleCData.c_char_p_from_param
 
 # XXX Deprecated
 def SetPointerType(pointer, cls):
@@ -445,8 +458,6 @@
             descr = FormatError(code).strip()
         return WindowsError(code, descr)
 
-_pointer_type_cache[None] = c_void_p
-
 if sizeof(c_uint) == sizeof(c_void_p):
     c_size_t = c_uint
     c_ssize_t = c_int
@@ -529,8 +540,4 @@
     elif sizeof(kind) == 8: c_uint64 = kind
 del(kind)
 
-# XXX for whatever reasons, creating the first instance of a callback
-# function is needed for the unittests on Win64 to succeed.  This MAY
-# be a compiler bug, since the problem occurs only when _ctypes is
-# compiled with the MS SDK compiler.  Or an uninitialized variable?
-CFUNCTYPE(c_int)(lambda: None)
+_reset_cache()
diff --git a/lib-python/3.2/ctypes/_endian.py b/lib-python/3.2/ctypes/_endian.py
--- a/lib-python/3.2/ctypes/_endian.py
+++ b/lib-python/3.2/ctypes/_endian.py
@@ -1,7 +1,7 @@
 import sys
 from ctypes import *
 
-_array_type = type(c_int * 3)
+_array_type = type(Array)
 
 def _other_endian(typ):
     """Return the type with the 'other' byte order.  Simple types like
diff --git a/lib-python/3.2/ctypes/macholib/fetch_macholib b/lib-python/3.2/ctypes/macholib/fetch_macholib
old mode 100644
new mode 100755
diff --git a/lib-python/3.2/ctypes/test/test_arrays.py b/lib-python/3.2/ctypes/test/test_arrays.py
--- a/lib-python/3.2/ctypes/test/test_arrays.py
+++ b/lib-python/3.2/ctypes/test/test_arrays.py
@@ -127,5 +127,57 @@
         t2 = my_int * 1
         self.assertTrue(t1 is t2)
 
+    def test_subclass(self):
+        class T(Array):
+            _type_ = c_int
+            _length_ = 13
+        class U(T):
+            pass
+        class V(U):
+            pass
+        class W(V):
+            pass
+        class X(T):
+            _type_ = c_short
+        class Y(T):
+            _length_ = 187
+
+        for c in [T, U, V, W]:
+            self.assertEqual(c._type_, c_int)
+            self.assertEqual(c._length_, 13)
+            self.assertEqual(c()._type_, c_int)
+            self.assertEqual(c()._length_, 13)
+
+        self.assertEqual(X._type_, c_short)
+        self.assertEqual(X._length_, 13)
+        self.assertEqual(X()._type_, c_short)
+        self.assertEqual(X()._length_, 13)
+
+        self.assertEqual(Y._type_, c_int)
+        self.assertEqual(Y._length_, 187)
+        self.assertEqual(Y()._type_, c_int)
+        self.assertEqual(Y()._length_, 187)
+
+    def test_bad_subclass(self):
+        import sys
+
+        with self.assertRaises(AttributeError):
+            class T(Array):
+                pass
+        with self.assertRaises(AttributeError):
+            class T(Array):
+                _type_ = c_int
+        with self.assertRaises(AttributeError):
+            class T(Array):
+                _length_ = 13
+        with self.assertRaises(OverflowError):
+            class T(Array):
+                _type_ = c_int
+                _length_ = sys.maxsize * 2
+        with self.assertRaises(AttributeError):
+            class T(Array):
+                _type_ = c_int
+                _length_ = 1.87
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/lib-python/3.2/ctypes/test/test_as_parameter.py b/lib-python/3.2/ctypes/test/test_as_parameter.py
--- a/lib-python/3.2/ctypes/test/test_as_parameter.py
+++ b/lib-python/3.2/ctypes/test/test_as_parameter.py
@@ -74,6 +74,7 @@
     def test_callbacks(self):
         f = dll._testfunc_callback_i_if
         f.restype = c_int
+        f.argtypes = None
 
         MyCallback = CFUNCTYPE(c_int, c_int)
 
diff --git a/lib-python/3.2/ctypes/test/test_callbacks.py b/lib-python/3.2/ctypes/test/test_callbacks.py
--- a/lib-python/3.2/ctypes/test/test_callbacks.py
+++ b/lib-python/3.2/ctypes/test/test_callbacks.py
@@ -134,6 +134,14 @@
                 if isinstance(x, X)]
         self.assertEqual(len(live), 0)
 
+    def test_issue12483(self):
+        import gc
+        class Nasty:
+            def __del__(self):
+                gc.collect()
+        CFUNCTYPE(None)(lambda x=Nasty(): None)
+        
+
 try:
     WINFUNCTYPE
 except NameError:
diff --git a/lib-python/3.2/ctypes/test/test_functions.py b/lib-python/3.2/ctypes/test/test_functions.py
--- a/lib-python/3.2/ctypes/test/test_functions.py
+++ b/lib-python/3.2/ctypes/test/test_functions.py
@@ -250,6 +250,7 @@
     def test_callbacks(self):
         f = dll._testfunc_callback_i_if
         f.restype = c_int
+        f.argtypes = None
 
         MyCallback = CFUNCTYPE(c_int, c_int)
 
diff --git a/lib-python/3.2/ctypes/test/test_structures.py b/lib-python/3.2/ctypes/test/test_structures.py
--- a/lib-python/3.2/ctypes/test/test_structures.py
+++ b/lib-python/3.2/ctypes/test/test_structures.py
@@ -239,6 +239,14 @@
             pass
         self.assertRaises(TypeError, setattr, POINT, "_fields_", [("x", 1), ("y", 2)])
 
+    def test_invalid_name(self):
+        # field name must be string
+        def declare_with_name(name):
+            class S(Structure):
+                _fields_ = [(name, c_int)]
+
+        self.assertRaises(TypeError, declare_with_name, b"x")
+
     def test_intarray_fields(self):
         class SomeInts(Structure):
             _fields_ = [("a", c_int * 4)]
@@ -318,6 +326,18 @@
         else:
             self.assertEqual(msg, "(Phone) TypeError: too many initializers")
 
+    def test_huge_field_name(self):
+        # issue12881: segfault with large structure field names
+        def create_class(length):
+            class S(Structure):
+                _fields_ = [('x' * length, c_int)]
+
+        for length in [10 ** i for i in range(0, 8)]:
+            try:
+                create_class(length)
+            except MemoryError:
+                # MemoryErrors are OK, we just don't want to segfault
+                pass
 
     def get_except(self, func, *args):
         try:
diff --git a/lib-python/3.2/ctypes/util.py b/lib-python/3.2/ctypes/util.py
--- a/lib-python/3.2/ctypes/util.py
+++ b/lib-python/3.2/ctypes/util.py
@@ -171,22 +171,6 @@
 
     else:
 
-        def _findLib_ldconfig(name):
-            # XXX assuming GLIBC's ldconfig (with option -p)
-            expr = r'/[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
-            with contextlib.closing(os.popen('/sbin/ldconfig -p 2>/dev/null')) as f:
-                data = f.read()
-            res = re.search(expr, data)
-            if not res:
-                # Hm, this works only for libs needed by the python executable.
-                cmd = 'ldd %s 2>/dev/null' % sys.executable
-                with contextlib.closing(os.popen(cmd)) as f:
-                    data = f.read()
-                res = re.search(expr, data)
-                if not res:
-                    return None
-            return res.group(0)
-
         def _findSoname_ldconfig(name):
             import struct
             if struct.calcsize('l') == 4:
@@ -203,8 +187,7 @@
             abi_type = mach_map.get(machine, 'libc6')
 
             # XXX assuming GLIBC's ldconfig (with option -p)
-            expr = r'(\S+)\s+\((%s(?:, OS ABI:[^\)]*)?)\)[^/]*(/[^\(\)\s]*lib%s\.[^\(\)\s]*)' \
-                   % (abi_type, re.escape(name))
+            expr = r'\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type)
             with contextlib.closing(os.popen('LC_ALL=C LANG=C /sbin/ldconfig -p 2>/dev/null')) as f:
                 data = f.read()
             res = re.search(expr, data)
diff --git a/lib-python/3.2/curses/__init__.py b/lib-python/3.2/curses/__init__.py
--- a/lib-python/3.2/curses/__init__.py
+++ b/lib-python/3.2/curses/__init__.py
@@ -54,4 +54,4 @@
 try:
     has_key
 except NameError:
-    from has_key import has_key
+    from .has_key import has_key
diff --git a/lib-python/3.2/datetime.py b/lib-python/3.2/datetime.py
--- a/lib-python/3.2/datetime.py
+++ b/lib-python/3.2/datetime.py
@@ -2057,7 +2057,7 @@
 
 Because we know z.d said z was in daylight time (else [5] would have held and
 we would have stopped then), and we know z.d != z'.d (else [8] would have held
-and we we have stopped then), and there are only 2 possible values dst() can
+and we have stopped then), and there are only 2 possible values dst() can
 return in Eastern, it follows that z'.d must be 0 (which it is in the example,
 but the reasoning doesn't depend on the example -- it depends on there being
 two possible dst() outcomes, one zero and the other non-zero).  Therefore
diff --git a/lib-python/3.2/dbm/__init__.py b/lib-python/3.2/dbm/__init__.py
--- a/lib-python/3.2/dbm/__init__.py
+++ b/lib-python/3.2/dbm/__init__.py
@@ -166,7 +166,7 @@
         return ""
 
     # Check for GNU dbm
-    if magic == 0x13579ace:
+    if magic in (0x13579ace, 0x13579acd, 0x13579acf):
         return "dbm.gnu"
 
     # Later versions of Berkeley db hash file have a 12-byte pad in
diff --git a/lib-python/3.2/distutils/__init__.py b/lib-python/3.2/distutils/__init__.py
--- a/lib-python/3.2/distutils/__init__.py
+++ b/lib-python/3.2/distutils/__init__.py
@@ -13,5 +13,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "3.2.2"
+__version__ = "3.2.3"
 #--end constants--
diff --git a/lib-python/3.2/distutils/command/bdist_rpm.py b/lib-python/3.2/distutils/command/bdist_rpm.py
--- a/lib-python/3.2/distutils/command/bdist_rpm.py
+++ b/lib-python/3.2/distutils/command/bdist_rpm.py
@@ -365,16 +365,28 @@
         self.spawn(rpm_cmd)
 
         if not self.dry_run:
+            if self.distribution.has_ext_modules():
+                pyversion = get_python_version()
+            else:
+                pyversion = 'any'
+
             if not self.binary_only:
                 srpm = os.path.join(rpm_dir['SRPMS'], source_rpm)
                 assert(os.path.exists(srpm))
                 self.move_file(srpm, self.dist_dir)
+                filename = os.path.join(self.dist_dir, source_rpm)
+                self.distribution.dist_files.append(
+                    ('bdist_rpm', pyversion, filename))
 
             if not self.source_only:
                 for rpm in binary_rpms:
                     rpm = os.path.join(rpm_dir['RPMS'], rpm)
                     if os.path.exists(rpm):
                         self.move_file(rpm, self.dist_dir)
+                        filename = os.path.join(self.dist_dir,
+                                                os.path.basename(rpm))
+                        self.distribution.dist_files.append(
+                            ('bdist_rpm', pyversion, filename))
 
     def _dist_path(self, path):
         return os.path.join(self.dist_dir, os.path.basename(path))
diff --git a/lib-python/3.2/distutils/command/build_ext.py b/lib-python/3.2/distutils/command/build_ext.py
--- a/lib-python/3.2/distutils/command/build_ext.py
+++ b/lib-python/3.2/distutils/command/build_ext.py
@@ -165,8 +165,7 @@
         if plat_py_include != py_include:
             self.include_dirs.append(plat_py_include)
 
-        if isinstance(self.libraries, str):
-            self.libraries = [self.libraries]
+        self.ensure_string_list('libraries')
 
         # Life is easier if we're not forever checking for None, so
         # simplify these options to empty lists if unset
diff --git a/lib-python/3.2/distutils/command/build_py.py b/lib-python/3.2/distutils/command/build_py.py
--- a/lib-python/3.2/distutils/command/build_py.py
+++ b/lib-python/3.2/distutils/command/build_py.py
@@ -2,7 +2,8 @@
 
 Implements the Distutils 'build_py' command."""
 
-import sys, os
+import os
+import imp
 import sys
 from glob import glob
 
@@ -311,9 +312,11 @@
             outputs.append(filename)
             if include_bytecode:
                 if self.compile:
-                    outputs.append(filename + "c")
+                    outputs.append(imp.cache_from_source(filename,
+                                                         debug_override=True))
                 if self.optimize > 0:
-                    outputs.append(filename + "o")
+                    outputs.append(imp.cache_from_source(filename,
+                                                         debug_override=False))
 
         outputs += [
             os.path.join(build_dir, filename)
diff --git a/lib-python/3.2/distutils/command/install_egg_info.py b/lib-python/3.2/distutils/command/install_egg_info.py
--- a/lib-python/3.2/distutils/command/install_egg_info.py
+++ b/lib-python/3.2/distutils/command/install_egg_info.py
@@ -40,9 +40,8 @@
                          "Creating "+self.install_dir)
         log.info("Writing %s", target)
         if not self.dry_run:
-            f = open(target, 'w')
-            self.distribution.metadata.write_pkg_file(f)
-            f.close()
+            with open(target, 'w', encoding='UTF-8') as f:
+                self.distribution.metadata.write_pkg_file(f)
 
     def get_outputs(self):
         return self.outputs
diff --git a/lib-python/3.2/distutils/command/install_lib.py b/lib-python/3.2/distutils/command/install_lib.py
--- a/lib-python/3.2/distutils/command/install_lib.py
+++ b/lib-python/3.2/distutils/command/install_lib.py
@@ -4,6 +4,7 @@
 (install all Python modules)."""
 
 import os
+import imp
 import sys
 
 from distutils.core import Command
@@ -164,9 +165,11 @@
             if ext != PYTHON_SOURCE_EXTENSION:
                 continue
             if self.compile:
-                bytecode_files.append(py_file + "c")
+                bytecode_files.append(imp.cache_from_source(
+                    py_file, debug_override=True))
             if self.optimize > 0:
-                bytecode_files.append(py_file + "o")
+                bytecode_files.append(imp.cache_from_source(
+                    py_file, debug_override=False))
 
         return bytecode_files
 
diff --git a/lib-python/3.2/distutils/command/sdist.py b/lib-python/3.2/distutils/command/sdist.py
--- a/lib-python/3.2/distutils/command/sdist.py
+++ b/lib-python/3.2/distutils/command/sdist.py
@@ -306,7 +306,10 @@
 
                 try:
                     self.filelist.process_template_line(line)
-                except DistutilsTemplateError as msg:
+                # the call above can raise a DistutilsTemplateError for
+                # malformed lines, or a ValueError from the lower-level
+                # convert_path function
+                except (DistutilsTemplateError, ValueError) as msg:
                     self.warn("%s, line %d: %s" % (template.filename,
                                                    template.current_line,
                                                    msg))
diff --git a/lib-python/3.2/distutils/dist.py b/lib-python/3.2/distutils/dist.py
--- a/lib-python/3.2/distutils/dist.py
+++ b/lib-python/3.2/distutils/dist.py
@@ -537,7 +537,7 @@
             for (help_option, short, desc, func) in cmd_class.help_options:
                 if hasattr(opts, parser.get_attr_name(help_option)):
                     help_option_found=1
-                    if hasattr(func, '__call__'):
+                    if callable(func):
                         func()
                     else:
                         raise DistutilsClassError(
@@ -1010,17 +1010,16 @@
     def write_pkg_info(self, base_dir):
         """Write the PKG-INFO file into the release tree.
         """
-        pkg_info = open(os.path.join(base_dir, 'PKG-INFO'), 'w')
-        try:
+        with open(os.path.join(base_dir, 'PKG-INFO'), 'w',
+                  encoding='UTF-8') as pkg_info:
             self.write_pkg_file(pkg_info)
-        finally:
-            pkg_info.close()
 
     def write_pkg_file(self, file):
         """Write the PKG-INFO format data to a file object.
         """
         version = '1.0'
-        if self.provides or self.requires or self.obsoletes:
+        if (self.provides or self.requires or self.obsoletes or
+            self.classifiers or self.download_url):
             version = '1.1'
 
         file.write('Metadata-Version: %s\n' % version)
diff --git a/lib-python/3.2/distutils/filelist.py b/lib-python/3.2/distutils/filelist.py
--- a/lib-python/3.2/distutils/filelist.py
+++ b/lib-python/3.2/distutils/filelist.py
@@ -201,6 +201,7 @@
 
         Return True if files are found, False otherwise.
         """
+        # XXX docstring lying about what the special chars are?
         files_found = False
         pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
         self.debug_print("include_pattern: applying regex r'%s'" %
@@ -284,11 +285,14 @@
     # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
     # and by extension they shouldn't match such "special characters" under
     # any OS.  So change all non-escaped dots in the RE to match any
-    # character except the special characters.
-    # XXX currently the "special characters" are just slash -- i.e. this is
-    # Unix-only.
-    pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', r'\1[^/]', pattern_re)
-
+    # character except the special characters (currently: just os.sep).
+    sep = os.sep
+    if os.sep == '\\':
+        # we're using a regex to manipulate a regex, so we need
+        # to escape the backslash twice
+        sep = r'\\\\'
+    escaped = r'\1[^%s]' % sep
+    pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
     return pattern_re
 
 
@@ -312,8 +316,11 @@
     if prefix is not None:
         # ditch end of pattern character
         empty_pattern = glob_to_re('')
-        prefix_re = (glob_to_re(prefix))[:-len(empty_pattern)]
-        pattern_re = "^" + os.path.join(prefix_re, ".*" + pattern_re)
+        prefix_re = glob_to_re(prefix)[:-len(empty_pattern)]
+        sep = os.sep
+        if os.sep == '\\':
+            sep = r'\\'
+        pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re))
     else:                               # no prefix -- respect anchor flag
         if anchor:
             pattern_re = "^" + pattern_re
diff --git a/lib-python/3.2/distutils/msvc9compiler.py b/lib-python/3.2/distutils/msvc9compiler.py
--- a/lib-python/3.2/distutils/msvc9compiler.py
+++ b/lib-python/3.2/distutils/msvc9compiler.py
@@ -627,15 +627,7 @@
                     self.library_filename(dll_name))
                 ld_args.append ('/IMPLIB:' + implib_file)
 
-            # Embedded manifests are recommended - see MSDN article titled
-            # "How to: Embed a Manifest Inside a C/C++ Application"
-            # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx)
-            # Ask the linker to generate the manifest in the temp dir, so
-            # we can embed it later.
-            temp_manifest = os.path.join(
-                    build_temp,
-                    os.path.basename(output_filename) + ".manifest")
-            ld_args.append('/MANIFESTFILE:' + temp_manifest)
+            self.manifest_setup_ldargs(output_filename, build_temp, ld_args)
 
             if extra_preargs:
                 ld_args[:0] = extra_preargs
@@ -653,21 +645,54 @@
             # will still consider the DLL up-to-date, but it will not have a
             # manifest.  Maybe we should link to a temp file?  OTOH, that
             # implies a build environment error that shouldn't go undetected.
-            if target_desc == CCompiler.EXECUTABLE:
-                mfid = 1
-            else:
-                mfid = 2
-                # Remove references to the Visual C runtime
-                self._remove_visual_c_ref(temp_manifest)
-            out_arg = '-outputresource:%s;%s' % (output_filename, mfid)
-            try:
-                self.spawn(['mt.exe', '-nologo', '-manifest',
-                            temp_manifest, out_arg])
-            except DistutilsExecError as msg:
-                raise LinkError(msg)
+            mfinfo = self.manifest_get_embed_info(target_desc, ld_args)
+            if mfinfo is not None:
+                mffilename, mfid = mfinfo
+                out_arg = '-outputresource:%s;%s' % (output_filename, mfid)
+                try:
+                    self.spawn(['mt.exe', '-nologo', '-manifest',
+                                mffilename, out_arg])
+                except DistutilsExecError as msg:
+                    raise LinkError(msg)
         else:
             log.debug("skipping %s (up-to-date)", output_filename)
 
+    def manifest_setup_ldargs(self, output_filename, build_temp, ld_args):
+        # If we need a manifest at all, an embedded manifest is recommended.
+        # See MSDN article titled
+        # "How to: Embed a Manifest Inside a C/C++ Application"
+        # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx)
+        # Ask the linker to generate the manifest in the temp dir, so
+        # we can check it, and possibly embed it, later.
+        temp_manifest = os.path.join(
+                build_temp,
+                os.path.basename(output_filename) + ".manifest")
+        ld_args.append('/MANIFESTFILE:' + temp_manifest)
+
+    def manifest_get_embed_info(self, target_desc, ld_args):
+        # If a manifest should be embedded, return a tuple of
+        # (manifest_filename, resource_id).  Returns None if no manifest
+        # should be embedded.  See http://bugs.python.org/issue7833 for why
+        # we want to avoid any manifest for extension modules if we can)
+        for arg in ld_args:
+            if arg.startswith("/MANIFESTFILE:"):
+                temp_manifest = arg.split(":", 1)[1]
+                break
+        else:
+            # no /MANIFESTFILE so nothing to do.
+            return None
+        if target_desc == CCompiler.EXECUTABLE:
+            # by default, executables always get the manifest with the
+            # CRT referenced.
+            mfid = 1
+        else:
+            # Extension modules try and avoid any manifest if possible.
+            mfid = 2
+            temp_manifest = self._remove_visual_c_ref(temp_manifest)
+        if temp_manifest is None:
+            return None
+        return temp_manifest, mfid
+
     def _remove_visual_c_ref(self, manifest_file):
         try:
             # Remove references to the Visual C runtime, so they will
@@ -676,6 +701,8 @@
             # runtimes are not in WinSxS folder, but in Python's own
             # folder), the runtimes do not need to be in every folder
             # with .pyd's.
+            # Returns either the filename of the modified manifest or
+            # None if no manifest should be embedded.
             manifest_f = open(manifest_file)
             try:
                 manifest_buf = manifest_f.read()
@@ -688,9 +715,18 @@
             manifest_buf = re.sub(pattern, "", manifest_buf)
             pattern = "<dependentAssembly>\s*</dependentAssembly>"
             manifest_buf = re.sub(pattern, "", manifest_buf)
+            # Now see if any other assemblies are referenced - if not, we
+            # don't want a manifest embedded.
+            pattern = re.compile(
+                r"""<assemblyIdentity.*?name=(?:"|')(.+?)(?:"|')"""
+                r""".*?(?:/>|</assemblyIdentity>)""", re.DOTALL)
+            if re.search(pattern, manifest_buf) is None:
+                return None
+
             manifest_f = open(manifest_file, 'w')
             try:
                 manifest_f.write(manifest_buf)
+                return manifest_file
             finally:
                 manifest_f.close()
         except IOError:
diff --git a/lib-python/3.2/distutils/sysconfig.py b/lib-python/3.2/distutils/sysconfig.py
--- a/lib-python/3.2/distutils/sysconfig.py
+++ b/lib-python/3.2/distutils/sysconfig.py
@@ -146,6 +146,7 @@
             "I don't know where Python installs its library "
             "on platform '%s'" % os.name)
 
+_USE_CLANG = None
 
 def customize_compiler(compiler):
     """Do any platform-specific customization of a CCompiler instance.
@@ -158,8 +159,38 @@
             get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
                             'CCSHARED', 'LDSHARED', 'SO', 'AR', 'ARFLAGS')
 
+        newcc = None
         if 'CC' in os.environ:
-            cc = os.environ['CC']
+            newcc = os.environ['CC']
+        elif sys.platform == 'darwin' and cc == 'gcc-4.2':
+            # Issue #13590:
+            #       Since Apple removed gcc-4.2 in Xcode 4.2, we can no
+            #       longer assume it is available for extension module builds.
+            #       If Python was built with gcc-4.2, check first to see if
+            #       it is available on this system; if not, try to use clang
+            #       instead unless the caller explicitly set CC.
+            global _USE_CLANG
+            if _USE_CLANG is None:
+                from distutils import log
+                from subprocess import Popen, PIPE
+                p = Popen("! type gcc-4.2 && type clang && exit 2",
+                                shell=True, stdout=PIPE, stderr=PIPE)
+                p.wait()
+                if p.returncode == 2:
+                    _USE_CLANG = True
+                    log.warn("gcc-4.2 not found, using clang instead")
+                else:
+                    _USE_CLANG = False
+            if _USE_CLANG:
+                newcc = 'clang'
+        if newcc:
+            # On OS X, if CC is overridden, use that as the default
+            #       command for LDSHARED as well
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
         if 'CXX' in os.environ:
             cxx = os.environ['CXX']
         if 'LDSHARED' in os.environ:
@@ -218,7 +249,7 @@
     """Return full pathname of installed Makefile from the Python build."""
     if python_build:
         return os.path.join(os.path.dirname(sys.executable), "Makefile")
-    lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
+    lib_dir = get_python_lib(plat_specific=0, standard_lib=1)
     config_file = 'config-{}{}'.format(get_python_version(), build_flags)
     return os.path.join(lib_dir, config_file, 'Makefile')
 
diff --git a/lib-python/3.2/distutils/tests/support.py b/lib-python/3.2/distutils/tests/support.py
--- a/lib-python/3.2/distutils/tests/support.py
+++ b/lib-python/3.2/distutils/tests/support.py
@@ -141,9 +141,9 @@
 
     Example use:
 
-            def test_compile(self):
-                copy_xxmodule_c(self.tmpdir)
-                self.assertIn('xxmodule.c', os.listdir(self.tmpdir)
+        def test_compile(self):
+            copy_xxmodule_c(self.tmpdir)
+            self.assertIn('xxmodule.c', os.listdir(self.tmpdir))
 
     If the source file can be found, it will be copied to *directory*.  If not,
     the test will be skipped.  Errors during copy are not caught.
@@ -175,10 +175,9 @@
 def fixup_build_ext(cmd):
     """Function needed to make build_ext tests pass.
 
-    When Python was build with --enable-shared on Unix, -L. is not good
-    enough to find the libpython<blah>.so.  This is because regrtest runs
-    it under a tempdir, not in the top level where the .so lives.  By the
-    time we've gotten here, Python's already been chdir'd to the tempdir.
+    When Python was built with --enable-shared on Unix, -L. is not enough to
+    find libpython<blah>.so, because regrtest runs in a tempdir, not in the
+    source directory where the .so lives.
 
     When Python was built with in debug mode on Windows, build_ext commands
     need their debug attribute set, and it is not done automatically for
@@ -189,6 +188,9 @@
         cmd = build_ext(dist)
         support.fixup_build_ext(cmd)
         cmd.ensure_finalized()
+
+    Unlike most other Unix platforms, Mac OS X embeds absolute paths
+    to shared libraries into executables, so the fixup is not needed there.
     """
     if os.name == 'nt':
         cmd.debug = sys.executable.endswith('_d.exe')
@@ -200,5 +202,8 @@
         if runshared is None:
             cmd.library_dirs = ['.']
         else:
-            name, equals, value = runshared.partition('=')
-            cmd.library_dirs = value.split(os.pathsep)
+            if sys.platform == 'darwin':
+                cmd.library_dirs = []
+            else:
+                name, equals, value = runshared.partition('=')
+                cmd.library_dirs = value.split(os.pathsep)
diff --git a/lib-python/3.2/distutils/tests/test_bdist_dumb.py b/lib-python/3.2/distutils/tests/test_bdist_dumb.py
--- a/lib-python/3.2/distutils/tests/test_bdist_dumb.py
+++ b/lib-python/3.2/distutils/tests/test_bdist_dumb.py
@@ -1,8 +1,10 @@
 """Tests for distutils.command.bdist_dumb."""
 
+import os
+import imp
+import sys
+import zipfile
 import unittest
-import sys
-import os
 from test.support import run_unittest
 
 from distutils.core import Distribution
@@ -72,15 +74,24 @@
 
         # see what we have
         dist_created = os.listdir(os.path.join(pkg_dir, 'dist'))
-        base = "%s.%s" % (dist.get_fullname(), cmd.plat_name)
+        base = "%s.%s.zip" % (dist.get_fullname(), cmd.plat_name)
         if os.name == 'os2':
             base = base.replace(':', '-')
 
-        wanted = ['%s.zip' % base]
-        self.assertEqual(dist_created, wanted)
+        self.assertEqual(dist_created, [base])
 
         # now let's check what we have in the zip file
-        # XXX to be done
+        fp = zipfile.ZipFile(os.path.join('dist', base))
+        try:
+            contents = fp.namelist()
+        finally:
+            fp.close()
+
+        contents = sorted(os.path.basename(fn) for fn in contents)
+        wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2],
+                  'foo.%s.pyc' % imp.get_tag(),
+                  'foo.py']
+        self.assertEqual(contents, sorted(wanted))
 
 def test_suite():
     return unittest.makeSuite(BuildDumbTestCase)
diff --git a/lib-python/3.2/distutils/tests/test_bdist_rpm.py b/lib-python/3.2/distutils/tests/test_bdist_rpm.py
--- a/lib-python/3.2/distutils/tests/test_bdist_rpm.py
+++ b/lib-python/3.2/distutils/tests/test_bdist_rpm.py
@@ -78,6 +78,10 @@
         dist_created = os.listdir(os.path.join(pkg_dir, 'dist'))
         self.assertTrue('foo-0.1-1.noarch.rpm' in dist_created)
 
+        # bug #2945: upload ignores bdist_rpm files
+        self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files)
+        self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files)
+
     def test_no_optimize_flag(self):
 
         # XXX I am unable yet to make this test work without
@@ -117,6 +121,11 @@
 
         dist_created = os.listdir(os.path.join(pkg_dir, 'dist'))
         self.assertTrue('foo-0.1-1.noarch.rpm' in dist_created)
+
+        # bug #2945: upload ignores bdist_rpm files
+        self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files)
+        self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files)
+
         os.remove(os.path.join(pkg_dir, 'dist', 'foo-0.1-1.noarch.rpm'))
 
 def test_suite():
diff --git a/lib-python/3.2/distutils/tests/test_build_ext.py b/lib-python/3.2/distutils/tests/test_build_ext.py
--- a/lib-python/3.2/distutils/tests/test_build_ext.py
+++ b/lib-python/3.2/distutils/tests/test_build_ext.py
@@ -178,21 +178,22 @@
         # make sure cmd.libraries is turned into a list
         # if it's a string
         cmd = build_ext(dist)
-        cmd.libraries = 'my_lib'
+        cmd.libraries = 'my_lib, other_lib lastlib'
         cmd.finalize_options()
-        self.assertEqual(cmd.libraries, ['my_lib'])
+        self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib'])
 
         # make sure cmd.library_dirs is turned into a list
         # if it's a string
         cmd = build_ext(dist)
-        cmd.library_dirs = 'my_lib_dir'
+        cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep
         cmd.finalize_options()
-        self.assertTrue('my_lib_dir' in cmd.library_dirs)
+        self.assertIn('my_lib_dir', cmd.library_dirs)
+        self.assertIn('other_lib_dir', cmd.library_dirs)
 
         # make sure rpath is turned into a list
-        # if it's a list of os.pathsep's paths
+        # if it's a string
         cmd = build_ext(dist)
-        cmd.rpath = os.pathsep.join(['one', 'two'])
+        cmd.rpath = 'one%stwo' % os.pathsep
         cmd.finalize_options()
         self.assertEqual(cmd.rpath, ['one', 'two'])
 
diff --git a/lib-python/3.2/distutils/tests/test_build_py.py b/lib-python/3.2/distutils/tests/test_build_py.py
--- a/lib-python/3.2/distutils/tests/test_build_py.py
+++ b/lib-python/3.2/distutils/tests/test_build_py.py
@@ -2,7 +2,7 @@
 
 import os
 import sys
-import io
+import imp
 import unittest
 
 from distutils.command.build_py import build_py
@@ -53,23 +53,20 @@
         # This makes sure the list of outputs includes byte-compiled
         # files for Python modules but not for package data files
         # (there shouldn't *be* byte-code files for those!).
-        #
         self.assertEqual(len(cmd.get_outputs()), 3)
         pkgdest = os.path.join(destination, "pkg")
         files = os.listdir(pkgdest)
+        pycache_dir = os.path.join(pkgdest, "__pycache__")
         self.assertIn("__init__.py", files)
         self.assertIn("README.txt", files)
-        # XXX even with -O, distutils writes pyc, not pyo; bug?
         if sys.dont_write_bytecode:
-            self.assertNotIn("__init__.pyc", files)
+            self.assertFalse(os.path.exists(pycache_dir))
         else:
-            self.assertIn("__init__.pyc", files)
+            pyc_files = os.listdir(pycache_dir)
+            self.assertIn("__init__.%s.pyc" % imp.get_tag(), pyc_files)
 
     def test_empty_package_dir(self):
-        # See SF 1668596/1720897.
-        cwd = os.getcwd()
-
-        # create the distribution files.
+        # See bugs #1668596/#1720897
         sources = self.mkdtemp()
         open(os.path.join(sources, "__init__.py"), "w").close()
 
@@ -78,30 +75,55 @@
         open(os.path.join(testdir, "testfile"), "w").close()
 
         os.chdir(sources)
-        old_stdout = sys.stdout
-        sys.stdout = io.StringIO()
+        dist = Distribution({"packages": ["pkg"],
+                             "package_dir": {"pkg": ""},
+                             "package_data": {"pkg": ["doc/*"]}})
+        # script_name need not exist, it just need to be initialized
+        dist.script_name = os.path.join(sources, "setup.py")
+        dist.script_args = ["build"]
+        dist.parse_command_line()
 
         try:
-            dist = Distribution({"packages": ["pkg"],
-                                 "package_dir": {"pkg": ""},
-                                 "package_data": {"pkg": ["doc/*"]}})
-            # script_name need not exist, it just need to be initialized
-            dist.script_name = os.path.join(sources, "setup.py")
-            dist.script_args = ["build"]
-            dist.parse_command_line()
+            dist.run_commands()
+        except DistutilsFileError:
+            self.fail("failed package_data test when package_dir is ''")
 
-            try:
-                dist.run_commands()
-            except DistutilsFileError:
-                self.fail("failed package_data test when package_dir is ''")
-        finally:
-            # Restore state.
-            os.chdir(cwd)
-            sys.stdout = old_stdout
+    @unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
+    def test_byte_compile(self):
+        project_dir, dist = self.create_dist(py_modules=['boiledeggs'])
+        os.chdir(project_dir)
+        self.write_file('boiledeggs.py', 'import antigravity')
+        cmd = build_py(dist)
+        cmd.compile = 1
+        cmd.build_lib = 'here'
+        cmd.finalize_options()
+        cmd.run()
+
+        found = os.listdir(cmd.build_lib)
+        self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py'])
+        found = os.listdir(os.path.join(cmd.build_lib, '__pycache__'))
+        self.assertEqual(found, ['boiledeggs.%s.pyc' % imp.get_tag()])
+
+    @unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
+    def test_byte_compile_optimized(self):
+        project_dir, dist = self.create_dist(py_modules=['boiledeggs'])
+        os.chdir(project_dir)
+        self.write_file('boiledeggs.py', 'import antigravity')
+        cmd = build_py(dist)
+        cmd.compile = 0
+        cmd.optimize = 1
+        cmd.build_lib = 'here'
+        cmd.finalize_options()
+        cmd.run()
+
+        found = os.listdir(cmd.build_lib)
+        self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py'])
+        found = os.listdir(os.path.join(cmd.build_lib, '__pycache__'))
+        self.assertEqual(sorted(found), ['boiledeggs.%s.pyo' % imp.get_tag()])
 
     def test_dont_write_bytecode(self):
         # makes sure byte_compile is not used
-        pkg_dir, dist = self.create_dist()
+        dist = self.create_dist()[1]
         cmd = build_py(dist)
         cmd.compile = 1
         cmd.optimize = 1
@@ -115,6 +137,7 @@
 
         self.assertIn('byte-compiling is disabled', self.logs[0][1])
 
+
 def test_suite():
     return unittest.makeSuite(BuildPyTestCase)
 
diff --git a/lib-python/3.2/distutils/tests/test_check.py b/lib-python/3.2/distutils/tests/test_check.py
--- a/lib-python/3.2/distutils/tests/test_check.py
+++ b/lib-python/3.2/distutils/tests/test_check.py
@@ -46,6 +46,15 @@
         cmd = self._run(metadata, strict=1)
         self.assertEqual(cmd._warnings, 0)
 
+        # now a test with non-ASCII characters
+        metadata = {'url': 'xxx', 'author': '\u00c9ric',
+                    'author_email': 'xxx', 'name': 'xxx',
+                    'version': 'xxx',
+                    'description': 'Something about esszet \u00df',
+                    'long_description': 'More things about esszet \u00df'}
+        cmd = self._run(metadata)
+        self.assertEqual(cmd._warnings, 0)
+
     def test_check_document(self):
         if not HAS_DOCUTILS: # won't test without docutils
             return
@@ -80,8 +89,8 @@
         self.assertRaises(DistutilsSetupError, self._run, metadata,
                           **{'strict': 1, 'restructuredtext': 1})
 
-        # and non-broken rest
-        metadata['long_description'] = 'title\n=====\n\ntest'
+        # and non-broken rest, including a non-ASCII character to test #12114
+        metadata['long_description'] = 'title\n=====\n\ntest \u00df'
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
diff --git a/lib-python/3.2/distutils/tests/test_config_cmd.py b/lib-python/3.2/distutils/tests/test_config_cmd.py
--- a/lib-python/3.2/distutils/tests/test_config_cmd.py
+++ b/lib-python/3.2/distutils/tests/test_config_cmd.py
@@ -44,10 +44,10 @@
         cmd = config(dist)
 
         # simple pattern searches
-        match = cmd.search_cpp(pattern='xxx', body='// xxx')
+        match = cmd.search_cpp(pattern='xxx', body='/* xxx */')
         self.assertEqual(match, 0)
 
-        match = cmd.search_cpp(pattern='_configtest', body='// xxx')
+        match = cmd.search_cpp(pattern='_configtest', body='/* xxx */')
         self.assertEqual(match, 1)
 
     def test_finalize_options(self):
diff --git a/lib-python/3.2/distutils/tests/test_dist.py b/lib-python/3.2/distutils/tests/test_dist.py
--- a/lib-python/3.2/distutils/tests/test_dist.py
+++ b/lib-python/3.2/distutils/tests/test_dist.py
@@ -74,7 +74,7 @@
         self.assertEqual(d.get_command_packages(),
                          ["distutils.command", "foo.bar", "distutils.tests"])
         cmd = d.get_command_obj("test_dist")
-        self.assertTrue(isinstance(cmd, test_dist))
+        self.assertIsInstance(cmd, test_dist)
         self.assertEqual(cmd.sample_option, "sometext")
 
     def test_command_packages_configfile(self):
@@ -106,28 +106,23 @@
     def test_empty_options(self):
         # an empty options dictionary should not stay in the
         # list of attributes
-        klass = Distribution
 
         # catching warnings
         warns = []
+
         def _warn(msg):
             warns.append(msg)
 
-        old_warn = warnings.warn
+        self.addCleanup(setattr, warnings, 'warn', warnings.warn)
         warnings.warn = _warn
-        try:
-            dist = klass(attrs={'author': 'xxx',
-                                'name': 'xxx',
-                                'version': 'xxx',
-                                'url': 'xxxx',
-                                'options': {}})
-        finally:
-            warnings.warn = old_warn
+        dist = Distribution(attrs={'author': 'xxx', 'name': 'xxx',
+                                   'version': 'xxx', 'url': 'xxxx',
+                                   'options': {}})
 
         self.assertEqual(len(warns), 0)
+        self.assertNotIn('options', dir(dist))
 
     def test_finalize_options(self):
-
         attrs = {'keywords': 'one,two',
                  'platforms': 'one,two'}
 
@@ -150,7 +145,6 @@
         cmds = dist.get_command_packages()
         self.assertEqual(cmds, ['distutils.command', 'one', 'two'])
 
-
     def test_announce(self):
         # make sure the level is known
         dist = Distribution()
@@ -158,6 +152,7 @@
         kwargs = {'level': 'ok2'}
         self.assertRaises(ValueError, dist.announce, args, kwargs)
 
+
 class MetadataTestCase(support.TempdirManager, support.EnvironGuard,
                        unittest.TestCase):
 
@@ -170,15 +165,20 @@
         sys.argv[:] = self.argv[1]
         super(MetadataTestCase, self).tearDown()
 
+    def format_metadata(self, dist):
+        sio = io.StringIO()
+        dist.metadata.write_pkg_file(sio)
+        return sio.getvalue()
+
     def test_simple_metadata(self):
         attrs = {"name": "package",
                  "version": "1.0"}
         dist = Distribution(attrs)
         meta = self.format_metadata(dist)
-        self.assertTrue("Metadata-Version: 1.0" in meta)
-        self.assertTrue("provides:" not in meta.lower())
-        self.assertTrue("requires:" not in meta.lower())
-        self.assertTrue("obsoletes:" not in meta.lower())
+        self.assertIn("Metadata-Version: 1.0", meta)
+        self.assertNotIn("provides:", meta.lower())
+        self.assertNotIn("requires:", meta.lower())
+        self.assertNotIn("obsoletes:", meta.lower())
 
     def test_provides(self):
         attrs = {"name": "package",
@@ -190,9 +190,9 @@
         self.assertEqual(dist.get_provides(),
                          ["package", "package.sub"])
         meta = self.format_metadata(dist)
-        self.assertTrue("Metadata-Version: 1.1" in meta)
-        self.assertTrue("requires:" not in meta.lower())
-        self.assertTrue("obsoletes:" not in meta.lower())
+        self.assertIn("Metadata-Version: 1.1", meta)
+        self.assertNotIn("requires:", meta.lower())
+        self.assertNotIn("obsoletes:", meta.lower())
 
     def test_provides_illegal(self):
         self.assertRaises(ValueError, Distribution,
@@ -210,11 +210,11 @@
         self.assertEqual(dist.get_requires(),
                          ["other", "another (==1.0)"])
         meta = self.format_metadata(dist)
-        self.assertTrue("Metadata-Version: 1.1" in meta)
-        self.assertTrue("provides:" not in meta.lower())
-        self.assertTrue("Requires: other" in meta)
-        self.assertTrue("Requires: another (==1.0)" in meta)
-        self.assertTrue("obsoletes:" not in meta.lower())
+        self.assertIn("Metadata-Version: 1.1", meta)
+        self.assertNotIn("provides:", meta.lower())
+        self.assertIn("Requires: other", meta)
+        self.assertIn("Requires: another (==1.0)", meta)
+        self.assertNotIn("obsoletes:", meta.lower())
 
     def test_requires_illegal(self):
         self.assertRaises(ValueError, Distribution,
@@ -232,11 +232,11 @@
         self.assertEqual(dist.get_obsoletes(),
                          ["other", "another (<1.0)"])
         meta = self.format_metadata(dist)
-        self.assertTrue("Metadata-Version: 1.1" in meta)
-        self.assertTrue("provides:" not in meta.lower())
-        self.assertTrue("requires:" not in meta.lower())
-        self.assertTrue("Obsoletes: other" in meta)
-        self.assertTrue("Obsoletes: another (<1.0)" in meta)
+        self.assertIn("Metadata-Version: 1.1", meta)
+        self.assertNotIn("provides:", meta.lower())
+        self.assertNotIn("requires:", meta.lower())
+        self.assertIn("Obsoletes: other", meta)
+        self.assertIn("Obsoletes: another (<1.0)", meta)
 
     def test_obsoletes_illegal(self):
         self.assertRaises(ValueError, Distribution,
@@ -244,10 +244,34 @@
                            "version": "1.0",
                            "obsoletes": ["my.pkg (splat)"]})
 
-    def format_metadata(self, dist):
-        sio = io.StringIO()
-        dist.metadata.write_pkg_file(sio)
-        return sio.getvalue()
+    def test_classifier(self):
+        attrs = {'name': 'Boa', 'version': '3.0',
+                 'classifiers': ['Programming Language :: Python :: 3']}
+        dist = Distribution(attrs)
+        meta = self.format_metadata(dist)
+        self.assertIn('Metadata-Version: 1.1', meta)
+
+    def test_download_url(self):
+        attrs = {'name': 'Boa', 'version': '3.0',
+                 'download_url': 'http://example.org/boa'}
+        dist = Distribution(attrs)
+        meta = self.format_metadata(dist)
+        self.assertIn('Metadata-Version: 1.1', meta)
+
+    def test_long_description(self):
+        long_desc = textwrap.dedent("""\
+        example::
+              We start here
+            and continue here
+          and end here.""")
+        attrs = {"name": "package",
+                 "version": "1.0",
+                 "long_description": long_desc}
+
+        dist = Distribution(attrs)
+        meta = self.format_metadata(dist)
+        meta = meta.replace('\n' + 8 * ' ', '\n')
+        self.assertIn(long_desc, meta)
 
     def test_custom_pydistutils(self):
         # fixes #2166
@@ -272,14 +296,14 @@
             if sys.platform in ('linux', 'darwin'):
                 os.environ['HOME'] = temp_dir
                 files = dist.find_config_files()
-                self.assertTrue(user_filename in files)
+                self.assertIn(user_filename, files)
 
             # win32-style
             if sys.platform == 'win32':
                 # home drive should be found
                 os.environ['HOME'] = temp_dir
                 files = dist.find_config_files()
-                self.assertTrue(user_filename in files,
+                self.assertIn(user_filename, files,
                              '%r not found in %r' % (user_filename, files))
         finally:
             os.remove(user_filename)
@@ -301,22 +325,8 @@
 
         output = [line for line in s.getvalue().split('\n')
                   if line.strip() != '']
-        self.assertTrue(len(output) > 0)
+        self.assertTrue(output)
 
-    def test_long_description(self):
-        long_desc = textwrap.dedent("""\
-        example::
-              We start here
-            and continue here
-          and end here.""")
-        attrs = {"name": "package",
-                 "version": "1.0",
-                 "long_description": long_desc}
-
-        dist = Distribution(attrs)
-        meta = self.format_metadata(dist)
-        meta = meta.replace('\n' + 8 * ' ', '\n')
-        self.assertTrue(long_desc in meta)
 
 def test_suite():
     suite = unittest.TestSuite()
diff --git a/lib-python/3.2/distutils/tests/test_filelist.py b/lib-python/3.2/distutils/tests/test_filelist.py
--- a/lib-python/3.2/distutils/tests/test_filelist.py
+++ b/lib-python/3.2/distutils/tests/test_filelist.py
@@ -1,40 +1,297 @@
 """Tests for distutils.filelist."""
+import os
+import re
 import unittest
+from distutils import debug
+from distutils.log import WARN
+from distutils.errors import DistutilsTemplateError
+from distutils.filelist import glob_to_re, translate_pattern, FileList
 
-from distutils.filelist import glob_to_re, FileList
 from test.support import captured_stdout, run_unittest
-from distutils import debug
+from distutils.tests import support
 
-class FileListTestCase(unittest.TestCase):
+MANIFEST_IN = """\
+include ok
+include xo
+exclude xo
+include foo.tmp
+include buildout.cfg
+global-include *.x
+global-include *.txt
+global-exclude *.tmp
+recursive-include f *.oo
+recursive-exclude global *.x
+graft dir
+prune dir3
+"""
+
+
+def make_local_path(s):
+    """Converts '/' in a string to os.sep"""
+    return s.replace('/', os.sep)
+
+
+class FileListTestCase(support.LoggingSilencer,
+                       unittest.TestCase):
+
+    def assertNoWarnings(self):
+        self.assertEqual(self.get_logs(WARN), [])
+        self.clear_logs()
+
+    def assertWarnings(self):
+        self.assertGreater(len(self.get_logs(WARN)), 0)
+        self.clear_logs()
 
     def test_glob_to_re(self):
-        # simple cases
-        self.assertEqual(glob_to_re('foo*'), 'foo[^/]*\\Z(?ms)')
-        self.assertEqual(glob_to_re('foo?'), 'foo[^/]\\Z(?ms)')
-        self.assertEqual(glob_to_re('foo??'), 'foo[^/][^/]\\Z(?ms)')
+        sep = os.sep
+        if os.sep == '\\':
+            sep = re.escape(os.sep)
 
-        # special cases
-        self.assertEqual(glob_to_re(r'foo\\*'), r'foo\\\\[^/]*\Z(?ms)')
-        self.assertEqual(glob_to_re(r'foo\\\*'), r'foo\\\\\\[^/]*\Z(?ms)')
-        self.assertEqual(glob_to_re('foo????'), r'foo[^/][^/][^/][^/]\Z(?ms)')
-        self.assertEqual(glob_to_re(r'foo\\??'), r'foo\\\\[^/][^/]\Z(?ms)')
+        for glob, regex in (
+            # simple cases
+            ('foo*', r'foo[^%(sep)s]*\Z(?ms)'),
+            ('foo?', r'foo[^%(sep)s]\Z(?ms)'),
+            ('foo??', r'foo[^%(sep)s][^%(sep)s]\Z(?ms)'),
+            # special cases
+            (r'foo\\*', r'foo\\\\[^%(sep)s]*\Z(?ms)'),
+            (r'foo\\\*', r'foo\\\\\\[^%(sep)s]*\Z(?ms)'),
+            ('foo????', r'foo[^%(sep)s][^%(sep)s][^%(sep)s][^%(sep)s]\Z(?ms)'),
+            (r'foo\\??', r'foo\\\\[^%(sep)s][^%(sep)s]\Z(?ms)')):
+            regex = regex % {'sep': sep}
+            self.assertEqual(glob_to_re(glob), regex)
+
+    def test_process_template_line(self):
+        # testing  all MANIFEST.in template patterns
+        file_list = FileList()
+        l = make_local_path
+
+        # simulated file list
+        file_list.allfiles = ['foo.tmp', 'ok', 'xo', 'four.txt',
+                              'buildout.cfg',
+                              # filelist does not filter out VCS directories,
+                              # it's sdist that does
+                              l('.hg/last-message.txt'),
+                              l('global/one.txt'),
+                              l('global/two.txt'),
+                              l('global/files.x'),
+                              l('global/here.tmp'),
+                              l('f/o/f.oo'),
+                              l('dir/graft-one'),
+                              l('dir/dir2/graft2'),
+                              l('dir3/ok'),
+                              l('dir3/sub/ok.txt'),
+                             ]
+
+        for line in MANIFEST_IN.split('\n'):
+            if line.strip() == '':
+                continue
+            file_list.process_template_line(line)
+
+        wanted = ['ok',
+                  'buildout.cfg',
+                  'four.txt',
+                  l('.hg/last-message.txt'),
+                  l('global/one.txt'),
+                  l('global/two.txt'),
+                  l('f/o/f.oo'),
+                  l('dir/graft-one'),
+                  l('dir/dir2/graft2'),
+                 ]
+
+        self.assertEqual(file_list.files, wanted)
 
     def test_debug_print(self):
         file_list = FileList()
         with captured_stdout() as stdout:
             file_list.debug_print('xxx')
-        stdout.seek(0)
-        self.assertEqual(stdout.read(), '')
+        self.assertEqual(stdout.getvalue(), '')
 
         debug.DEBUG = True
         try:
             with captured_stdout() as stdout:
                 file_list.debug_print('xxx')
-            stdout.seek(0)
-            self.assertEqual(stdout.read(), 'xxx\n')
+            self.assertEqual(stdout.getvalue(), 'xxx\n')
         finally:
             debug.DEBUG = False
 
+    def test_set_allfiles(self):
+        file_list = FileList()
+        files = ['a', 'b', 'c']
+        file_list.set_allfiles(files)
+        self.assertEqual(file_list.allfiles, files)
+
+    def test_remove_duplicates(self):
+        file_list = FileList()
+        file_list.files = ['a', 'b', 'a', 'g', 'c', 'g']
+        # files must be sorted beforehand (sdist does it)
+        file_list.sort()
+        file_list.remove_duplicates()
+        self.assertEqual(file_list.files, ['a', 'b', 'c', 'g'])
+
+    def test_translate_pattern(self):
+        # not regex
+        self.assertTrue(hasattr(
+            translate_pattern('a', anchor=True, is_regex=False),
+            'search'))
+
+        # is a regex
+        regex = re.compile('a')
+        self.assertEqual(
+            translate_pattern(regex, anchor=True, is_regex=True),
+            regex)
+
+        # plain string flagged as regex
+        self.assertTrue(hasattr(
+            translate_pattern('a', anchor=True, is_regex=True),
+            'search'))
+
+        # glob support
+        self.assertTrue(translate_pattern(
+            '*.py', anchor=True, is_regex=False).search('filelist.py'))
+
+    def test_exclude_pattern(self):
+        # return False if no match
+        file_list = FileList()
+        self.assertFalse(file_list.exclude_pattern('*.py'))
+
+        # return True if files match
+        file_list = FileList()
+        file_list.files = ['a.py', 'b.py']
+        self.assertTrue(file_list.exclude_pattern('*.py'))
+
+        # test excludes
+        file_list = FileList()
+        file_list.files = ['a.py', 'a.txt']
+        file_list.exclude_pattern('*.py')
+        self.assertEqual(file_list.files, ['a.txt'])
+
+    def test_include_pattern(self):
+        # return False if no match
+        file_list = FileList()
+        file_list.set_allfiles([])
+        self.assertFalse(file_list.include_pattern('*.py'))
+
+        # return True if files match
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', 'b.txt'])
+        self.assertTrue(file_list.include_pattern('*.py'))
+
+        # test * matches all files
+        file_list = FileList()
+        self.assertIsNone(file_list.allfiles)
+        file_list.set_allfiles(['a.py', 'b.txt'])
+        file_list.include_pattern('*')
+        self.assertEqual(file_list.allfiles, ['a.py', 'b.txt'])
+
+    def test_process_template(self):
+        l = make_local_path
+        # invalid lines
+        file_list = FileList()
+        for action in ('include', 'exclude', 'global-include',
+                       'global-exclude', 'recursive-include',
+                       'recursive-exclude', 'graft', 'prune', 'blarg'):
+            self.assertRaises(DistutilsTemplateError,
+                              file_list.process_template_line, action)
+
+        # include
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', 'b.txt', l('d/c.py')])
+
+        file_list.process_template_line('include *.py')
+        self.assertEqual(file_list.files, ['a.py'])
+        self.assertNoWarnings()
+
+        file_list.process_template_line('include *.rb')
+        self.assertEqual(file_list.files, ['a.py'])
+        self.assertWarnings()
+
+        # exclude
+        file_list = FileList()
+        file_list.files = ['a.py', 'b.txt', l('d/c.py')]
+
+        file_list.process_template_line('exclude *.py')
+        self.assertEqual(file_list.files, ['b.txt', l('d/c.py')])
+        self.assertNoWarnings()
+
+        file_list.process_template_line('exclude *.rb')
+        self.assertEqual(file_list.files, ['b.txt', l('d/c.py')])
+        self.assertWarnings()
+
+        # global-include
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', 'b.txt', l('d/c.py')])
+
+        file_list.process_template_line('global-include *.py')
+        self.assertEqual(file_list.files, ['a.py', l('d/c.py')])
+        self.assertNoWarnings()
+
+        file_list.process_template_line('global-include *.rb')
+        self.assertEqual(file_list.files, ['a.py', l('d/c.py')])
+        self.assertWarnings()
+
+        # global-exclude
+        file_list = FileList()
+        file_list.files = ['a.py', 'b.txt', l('d/c.py')]
+
+        file_list.process_template_line('global-exclude *.py')
+        self.assertEqual(file_list.files, ['b.txt'])
+        self.assertNoWarnings()
+
+        file_list.process_template_line('global-exclude *.rb')
+        self.assertEqual(file_list.files, ['b.txt'])
+        self.assertWarnings()
+
+        # recursive-include
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', l('d/b.py'), l('d/c.txt'),
+                                l('d/d/e.py')])
+
+        file_list.process_template_line('recursive-include d *.py')
+        self.assertEqual(file_list.files, [l('d/b.py'), l('d/d/e.py')])
+        self.assertNoWarnings()
+
+        file_list.process_template_line('recursive-include e *.py')
+        self.assertEqual(file_list.files, [l('d/b.py'), l('d/d/e.py')])
+        self.assertWarnings()
+
+        # recursive-exclude
+        file_list = FileList()
+        file_list.files = ['a.py', l('d/b.py'), l('d/c.txt'), l('d/d/e.py')]
+
+        file_list.process_template_line('recursive-exclude d *.py')
+        self.assertEqual(file_list.files, ['a.py', l('d/c.txt')])
+        self.assertNoWarnings()
+
+        file_list.process_template_line('recursive-exclude e *.py')
+        self.assertEqual(file_list.files, ['a.py', l('d/c.txt')])
+        self.assertWarnings()
+
+        # graft
+        file_list = FileList()
+        file_list.set_allfiles(['a.py', l('d/b.py'), l('d/d/e.py'),
+                                l('f/f.py')])
+
+        file_list.process_template_line('graft d')
+        self.assertEqual(file_list.files, [l('d/b.py'), l('d/d/e.py')])
+        self.assertNoWarnings()
+
+        file_list.process_template_line('graft e')
+        self.assertEqual(file_list.files, [l('d/b.py'), l('d/d/e.py')])
+        self.assertWarnings()
+
+        # prune
+        file_list = FileList()
+        file_list.files = ['a.py', l('d/b.py'), l('d/d/e.py'), l('f/f.py')]
+
+        file_list.process_template_line('prune d')
+        self.assertEqual(file_list.files, ['a.py', l('f/f.py')])
+        self.assertNoWarnings()
+
+        file_list.process_template_line('prune e')
+        self.assertEqual(file_list.files, ['a.py', l('f/f.py')])
+        self.assertWarnings()
+
+
 def test_suite():
     return unittest.makeSuite(FileListTestCase)
 
diff --git a/lib-python/3.2/distutils/tests/test_install.py b/lib-python/3.2/distutils/tests/test_install.py
--- a/lib-python/3.2/distutils/tests/test_install.py
+++ b/lib-python/3.2/distutils/tests/test_install.py
@@ -1,6 +1,7 @@
 """Tests for distutils.command.install."""
 
 import os
+import imp
 import sys
 import unittest
 import site
@@ -20,9 +21,8 @@
 
 
 def _make_ext_name(modname):
-    if os.name == 'nt':
-        if sys.executable.endswith('_d.exe'):
-            modname += '_d'
+    if os.name == 'nt' and sys.executable.endswith('_d.exe'):
+        modname += '_d'
     return modname + sysconfig.get_config_var('SO')
 
 
@@ -68,10 +68,7 @@
         check_path(cmd.install_data, destination)
 
     def test_user_site(self):
-        # site.USER_SITE was introduced in 2.6
-        if sys.version < '2.6':
-            return
-
+        # test install with --user
         # preparing the environment for the test
         self.old_user_base = site.USER_BASE
         self.old_user_site = site.USER_SITE
@@ -88,19 +85,17 @@
         self.old_expand = os.path.expanduser
         os.path.expanduser = _expanduser
 
-        try:
-            # this is the actual test
-            self._test_user_site()
-        finally:
+        def cleanup():
             site.USER_BASE = self.old_user_base
             site.USER_SITE = self.old_user_site
             install_module.USER_BASE = self.old_user_base
             install_module.USER_SITE = self.old_user_site
             os.path.expanduser = self.old_expand
 
-    def _test_user_site(self):
+        self.addCleanup(cleanup)
+
         for key in ('nt_user', 'unix_user', 'os2_home'):
-            self.assertTrue(key in INSTALL_SCHEMES)
+            self.assertIn(key, INSTALL_SCHEMES)
 
         dist = Distribution({'name': 'xx'})
         cmd = install(dist)
@@ -108,14 +103,14 @@
         # making sure the user option is there
         options = [name for name, short, lable in
                    cmd.user_options]
-        self.assertTrue('user' in options)
+        self.assertIn('user', options)
 
         # setting a value
         cmd.user = 1
 
         # user base and site shouldn't be created yet
-        self.assertTrue(not os.path.exists(self.user_base))
-        self.assertTrue(not os.path.exists(self.user_site))
+        self.assertFalse(os.path.exists(self.user_base))
+        self.assertFalse(os.path.exists(self.user_site))
 
         # let's run finalize
         cmd.ensure_finalized()
@@ -124,8 +119,8 @@
         self.assertTrue(os.path.exists(self.user_base))
         self.assertTrue(os.path.exists(self.user_site))
 
-        self.assertTrue('userbase' in cmd.config_vars)
-        self.assertTrue('usersite' in cmd.config_vars)
+        self.assertIn('userbase', cmd.config_vars)
+        self.assertIn('usersite', cmd.config_vars)
 
     def test_handle_extra_path(self):
         dist = Distribution({'name': 'xx', 'extra_path': 'path,dirs'})
@@ -178,15 +173,16 @@
 
     def test_record(self):
         install_dir = self.mkdtemp()
-        project_dir, dist = self.create_dist(scripts=['hello'])
-        self.addCleanup(os.chdir, os.getcwd())
+        project_dir, dist = self.create_dist(py_modules=['hello'],
+                                             scripts=['sayhi'])
         os.chdir(project_dir)
-        self.write_file('hello', "print('o hai')")
+        self.write_file('hello.py', "def main(): print('o hai')")
+        self.write_file('sayhi', 'from hello import main; main()')
 
         cmd = install(dist)
         dist.command_obj['install'] = cmd
         cmd.root = install_dir
-        cmd.record = os.path.join(project_dir, 'RECORD')
+        cmd.record = os.path.join(project_dir, 'filelist')
         cmd.ensure_finalized()
         cmd.run()
 
@@ -197,7 +193,7 @@
             f.close()
 
         found = [os.path.basename(line) for line in content.splitlines()]
-        expected = ['hello',
+        expected = ['hello.py', 'hello.%s.pyc' % imp.get_tag(), 'sayhi',
                     'UNKNOWN-0.0.0-py%s.%s.egg-info' % sys.version_info[:2]]
         self.assertEqual(found, expected)
 
@@ -205,7 +201,6 @@
         install_dir = self.mkdtemp()
         project_dir, dist = self.create_dist(ext_modules=[
             Extension('xx', ['xxmodule.c'])])
-        self.addCleanup(os.chdir, os.getcwd())
         os.chdir(project_dir)
         support.copy_xxmodule_c(project_dir)
 
@@ -217,7 +212,7 @@
         dist.command_obj['install'] = cmd
         dist.command_obj['build_ext'] = buildextcmd
         cmd.root = install_dir
-        cmd.record = os.path.join(project_dir, 'RECORD')
+        cmd.record = os.path.join(project_dir, 'filelist')
         cmd.ensure_finalized()
         cmd.run()
 
@@ -243,6 +238,7 @@
             install_module.DEBUG = False
         self.assertTrue(len(self.logs) > old_logs_len)
 
+
 def test_suite():
     return unittest.makeSuite(InstallTestCase)
 
diff --git a/lib-python/3.2/distutils/tests/test_install_lib.py b/lib-python/3.2/distutils/tests/test_install_lib.py
--- a/lib-python/3.2/distutils/tests/test_install_lib.py
+++ b/lib-python/3.2/distutils/tests/test_install_lib.py
@@ -1,6 +1,7 @@
 """Tests for distutils.command.install_data."""
 import sys
 import os
+import imp
 import unittest
 
 from distutils.command.install_lib import install_lib
@@ -9,13 +10,14 @@
 from distutils.errors import DistutilsOptionError
 from test.support import run_unittest
 
+
 class InstallLibTestCase(support.TempdirManager,
                          support.LoggingSilencer,
                          support.EnvironGuard,
                          unittest.TestCase):
 
     def test_finalize_options(self):
-        pkg_dir, dist = self.create_dist()
+        dist = self.create_dist()[1]
         cmd = install_lib(dist)
 
         cmd.finalize_options()
@@ -32,56 +34,64 @@
         cmd.finalize_options()
         self.assertEqual(cmd.optimize, 2)
 
-    @unittest.skipUnless(not sys.dont_write_bytecode,
-                         'byte-compile not supported')
+    @unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
     def test_byte_compile(self):
-        pkg_dir, dist = self.create_dist()
+        project_dir, dist = self.create_dist()
+        os.chdir(project_dir)
         cmd = install_lib(dist)
         cmd.compile = cmd.optimize = 1
 
-        f = os.path.join(pkg_dir, 'foo.py')
+        f = os.path.join(project_dir, 'foo.py')

From noreply at buildbot.pypy.org  Wed Apr 18 13:18:17 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 18 Apr 2012 13:18:17 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification/py3k: whops,
 actually inline the 3.2 stdlib changes
Message-ID: <20120418111817.6F9AC820B3@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification/py3k
Changeset: r54492:88d5dfc492a9
Date: 2012-04-18 13:17 +0200
http://bitbucket.org/pypy/pypy/changeset/88d5dfc492a9/

Log:	whops, actually inline the 3.2 stdlib changes

diff --git a/lib-python/3.2/opcode.py b/lib-python/3.2/opcode.py
--- a/lib-python/3.2/opcode.py
+++ b/lib-python/3.2/opcode.py
@@ -177,4 +177,10 @@
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 
+# pypy modification, experimental bytecode
+def_op('LOOKUP_METHOD', 201)          # Index in name list
+hasname.append(201)
+def_op('CALL_METHOD', 202)            # #args not including 'self'
+def_op('BUILD_LIST_FROM_ARG', 203)
+
 del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/3.2/sysconfig.py b/lib-python/3.2/sysconfig.py
--- a/lib-python/3.2/sysconfig.py
+++ b/lib-python/3.2/sysconfig.py
@@ -42,6 +42,16 @@
         'scripts': '{base}/bin',
         'data'   : '{base}',
         },
+    'pypy': {
+        'stdlib': '{base}/lib-python',
+        'platstdlib': '{base}/lib-python',
+        'purelib': '{base}/lib-python',
+        'platlib': '{base}/lib-python',
+        'include': '{base}/include',
+        'platinclude': '{base}/include',
+        'scripts': '{base}/bin',
+        'data'   : '{base}',
+        },
     'nt': {
         'stdlib': '{base}/Lib',
         'platstdlib': '{base}/Lib',
@@ -174,7 +184,9 @@
     return res
 
 def _get_default_scheme():
-    if os.name == 'posix':
+    if '__pypy__' in sys.builtin_module_names:
+        return 'pypy'
+    elif os.name == 'posix':
         # the default scheme for posix is posix_prefix
         return 'posix_prefix'
     return os.name
@@ -198,160 +210,9 @@
     return env_base if env_base else joinuser("~", ".local")
 
 
-def _parse_makefile(filename, vars=None):
-    """Parse a Makefile-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    import re
-    # Regexes needed for parsing Makefile (and similar syntaxes,
-    # like old-style Setup files).
-    _variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
-    _findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
-    _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
-
-    if vars is None:
-        vars = {}
-    done = {}
-    notdone = {}
-
-    with open(filename, errors="surrogateescape") as f:
-        lines = f.readlines()
-
-    for line in lines:
-        if line.startswith('#') or line.strip() == '':
-            continue
-        m = _variable_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            v = v.strip()
-            # `$$' is a literal `$' in make
-            tmpv = v.replace('$$', '')
-
-            if "$" in tmpv:
-                notdone[n] = v
-            else:
-                try:
-                    v = int(v)
-                except ValueError:
-                    # insert literal `$'
-                    done[n] = v.replace('$$', '$')
-                else:
-                    done[n] = v
-
-    # do variable interpolation here
-    variables = list(notdone.keys())
-
-    # Variables with a 'PY_' prefix in the makefile. These need to
-    # be made available without that prefix through sysconfig.
-    # Special care is needed to ensure that variable expansion works, even
-    # if the expansion uses the name without a prefix.
-    renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS')
-
-    while len(variables) > 0:
-        for name in tuple(variables):
-            value = notdone[name]
-            m = _findvar1_rx.search(value) or _findvar2_rx.search(value)
-            if m is not None:
-                n = m.group(1)
-                found = True
-                if n in done:
-                    item = str(done[n])
-                elif n in notdone:
-                    # get it on a subsequent round
-                    found = False
-                elif n in os.environ:
-                    # do it like make: fall back to environment
-                    item = os.environ[n]
-
-                elif n in renamed_variables:
-                    if name.startswith('PY_') and name[3:] in renamed_variables:
-                        item = ""
-
-                    elif 'PY_' + n in notdone:
-                        found = False
-
-                    else:
-                        item = str(done['PY_' + n])
-
-                else:
-                    done[n] = item = ""
-
-                if found:
-                    after = value[m.end():]
-                    value = value[:m.start()] + item + after
-                    if "$" in after:
-                        notdone[name] = value
-                    else:
-                        try:
-                            value = int(value)
-                        except ValueError:
-                            done[name] = value.strip()
-                        else:
-                            done[name] = value
-                        variables.remove(name)
-
-                        if name.startswith('PY_') \
-                        and name[3:] in renamed_variables:
-
-                            name = name[3:]
-                            if name not in done:
-                                done[name] = value
-
-
-            else:
-                # bogus variable reference (e.g. "prefix=$/opt/python");
-                # just drop it since we can't deal
-                done[name] = value
-                variables.remove(name)
-
-    # strip spurious spaces
-    for k, v in done.items():
-        if isinstance(v, str):
-            done[k] = v.strip()
-
-    # save the results in the global dictionary
-    vars.update(done)
-    return vars
-
-
-def get_makefile_filename():
-    """Return the path of the Makefile."""
-    if _PYTHON_BUILD:
-        return os.path.join(_PROJECT_BASE, "Makefile")
-    return os.path.join(get_path('stdlib'),
-                        'config-{}{}'.format(_PY_VERSION_SHORT, sys.abiflags),
-                        'Makefile')
-
-
 def _init_posix(vars):
     """Initialize the module as appropriate for POSIX systems."""
-    # load the installed Makefile:
-    makefile = get_makefile_filename()
-    try:
-        _parse_makefile(makefile, vars)
-    except IOError as e:
-        msg = "invalid Python installation: unable to open %s" % makefile
-        if hasattr(e, "strerror"):
-            msg = msg + " (%s)" % e.strerror
-        raise IOError(msg)
-    # load the installed pyconfig.h:
-    config_h = get_config_h_filename()
-    try:
-        with open(config_h) as f:
-            parse_config_h(f, vars)
-    except IOError as e:
-        msg = "invalid Python installation: unable to open %s" % config_h
-        if hasattr(e, "strerror"):
-            msg = msg + " (%s)" % e.strerror
-        raise IOError(msg)
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if _PYTHON_BUILD:
-        vars['LDSHARED'] = vars['BLDSHARED']
+    return
 
 def _init_non_posix(vars):
     """Initialize the module as appropriate for NT"""
@@ -639,11 +500,6 @@
         if release[0] >= "5":           # SunOS 5 == Solaris 2
             osname = "solaris"
             release = "%d.%s" % (int(release[0]) - 3, release[2:])
-            # We can't use "platform.architecture()[0]" because a
-            # bootstrap problem. We use a dict to get an error
-            # if some suspicious happens.
-            bitness = {2147483647:"32bit", 9223372036854775807:"64bit"}
-            machine += ".%s" % bitness[sys.maxsize]
         # fall through to standard osname-release-machine representation
     elif osname[:4] == "irix":              # could be "irix64"!
         return "%s-%s" % (osname, release)
diff --git a/lib-python/3.2/test/test_sysconfig.py b/lib-python/3.2/test/test_sysconfig.py
--- a/lib-python/3.2/test/test_sysconfig.py
+++ b/lib-python/3.2/test/test_sysconfig.py
@@ -1,5 +1,9 @@
-"""Tests for sysconfig."""
+"""Tests for 'site'.
 
+Tests assume the initial paths in sys.path once the interpreter has begun
+executing have not been removed.
+
+"""
 import unittest
 import sys
 import os
@@ -7,7 +11,7 @@
 import shutil
 from copy import copy, deepcopy
 
-from test.support import (run_unittest, TESTFN, unlink,
+from test.support import (run_unittest, TESTFN, unlink, get_attribute,
                           captured_stdout, skip_unless_symlink)
 
 import sysconfig
@@ -231,7 +235,7 @@
 
     def test_get_scheme_names(self):
         wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
-                  'posix_home', 'posix_prefix', 'posix_user')
+                  'posix_home', 'posix_prefix', 'posix_user', 'pypy')
         self.assertEqual(get_scheme_names(), wanted)
 
     @skip_unless_symlink
@@ -257,21 +261,14 @@
             unlink(link)
 
     def test_user_similar(self):
-        # Issue #8759: make sure the posix scheme for the users
+        # Issue 8759 : make sure the posix scheme for the users
         # is similar to the global posix_prefix one
         base = get_config_var('base')
         user = get_config_var('userbase')
-        # the global scheme mirrors the distinction between prefix and
-        # exec-prefix but not the user scheme, so we have to adapt the paths
-        # before comparing (issue #9100)
-        adapt = sys.prefix != sys.exec_prefix
         for name in ('stdlib', 'platstdlib', 'purelib', 'platlib'):
             global_path = get_path(name, 'posix_prefix')
-            if adapt:
-                global_path = global_path.replace(sys.exec_prefix, sys.prefix)
-                base = base.replace(sys.exec_prefix, sys.prefix)
             user_path = get_path(name, 'posix_user')
-            self.assertEqual(user_path, global_path.replace(base, user, 1))
+            self.assertEqual(user_path, global_path.replace(base, user))
 
     def test_main(self):
         # just making sure _main() runs and returns things in the stdout
diff --git a/lib-python/modified-3.2/opcode.py b/lib-python/modified-3.2/opcode.py
deleted file mode 100644
--- a/lib-python/modified-3.2/opcode.py
+++ /dev/null
@@ -1,186 +0,0 @@
-
-"""
-opcode module - potentially shared between dis and other modules which
-operate on bytecodes (e.g. peephole optimizers).
-"""
-
-__all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs",
-           "haslocal", "hascompare", "hasfree", "opname", "opmap",
-           "HAVE_ARGUMENT", "EXTENDED_ARG"]
-
-cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is',
-        'is not', 'exception match', 'BAD')
-
-hasconst = []
-hasname = []
-hasjrel = []
-hasjabs = []
-haslocal = []
-hascompare = []
-hasfree = []
-
-opmap = {}
-opname = [''] * 256
-for op in range(256): opname[op] = '<%r>' % (op,)
-del op
-
-def def_op(name, op):
-    opname[op] = name
-    opmap[name] = op
-
-def name_op(name, op):
-    def_op(name, op)
-    hasname.append(op)
-
-def jrel_op(name, op):
-    def_op(name, op)
-    hasjrel.append(op)
-
-def jabs_op(name, op):
-    def_op(name, op)
-    hasjabs.append(op)
-
-# Instruction opcodes for compiled code
-# Blank lines correspond to available opcodes
-
-def_op('STOP_CODE', 0)
-def_op('POP_TOP', 1)
-def_op('ROT_TWO', 2)
-def_op('ROT_THREE', 3)
-def_op('DUP_TOP', 4)
-def_op('DUP_TOP_TWO', 5)
-
-def_op('NOP', 9)
-def_op('UNARY_POSITIVE', 10)
-def_op('UNARY_NEGATIVE', 11)
-def_op('UNARY_NOT', 12)
-
-def_op('UNARY_INVERT', 15)
-
-def_op('BINARY_POWER', 19)
-def_op('BINARY_MULTIPLY', 20)
-
-def_op('BINARY_MODULO', 22)
-def_op('BINARY_ADD', 23)
-def_op('BINARY_SUBTRACT', 24)
-def_op('BINARY_SUBSCR', 25)
-def_op('BINARY_FLOOR_DIVIDE', 26)
-def_op('BINARY_TRUE_DIVIDE', 27)
-def_op('INPLACE_FLOOR_DIVIDE', 28)
-def_op('INPLACE_TRUE_DIVIDE', 29)
-
-def_op('STORE_MAP', 54)
-def_op('INPLACE_ADD', 55)
-def_op('INPLACE_SUBTRACT', 56)
-def_op('INPLACE_MULTIPLY', 57)
-
-def_op('INPLACE_MODULO', 59)
-def_op('STORE_SUBSCR', 60)
-def_op('DELETE_SUBSCR', 61)
-def_op('BINARY_LSHIFT', 62)
-def_op('BINARY_RSHIFT', 63)
-def_op('BINARY_AND', 64)
-def_op('BINARY_XOR', 65)
-def_op('BINARY_OR', 66)
-def_op('INPLACE_POWER', 67)
-def_op('GET_ITER', 68)
-def_op('STORE_LOCALS', 69)
-
-def_op('PRINT_EXPR', 70)
-def_op('LOAD_BUILD_CLASS', 71)
-
-def_op('INPLACE_LSHIFT', 75)
-def_op('INPLACE_RSHIFT', 76)
-def_op('INPLACE_AND', 77)
-def_op('INPLACE_XOR', 78)
-def_op('INPLACE_OR', 79)
-def_op('BREAK_LOOP', 80)
-def_op('WITH_CLEANUP', 81)
-
-def_op('RETURN_VALUE', 83)
-def_op('IMPORT_STAR', 84)
-
-def_op('YIELD_VALUE', 86)
-def_op('POP_BLOCK', 87)
-def_op('END_FINALLY', 88)
-def_op('POP_EXCEPT', 89)
-
-HAVE_ARGUMENT = 90              # Opcodes from here have an argument:
-
-name_op('STORE_NAME', 90)       # Index in name list
-name_op('DELETE_NAME', 91)      # ""
-def_op('UNPACK_SEQUENCE', 92)   # Number of tuple items
-jrel_op('FOR_ITER', 93)
-def_op('UNPACK_EX', 94)
-name_op('STORE_ATTR', 95)       # Index in name list
-name_op('DELETE_ATTR', 96)      # ""
-name_op('STORE_GLOBAL', 97)     # ""
-name_op('DELETE_GLOBAL', 98)    # ""
-def_op('LOAD_CONST', 100)       # Index in const list
-hasconst.append(100)
-name_op('LOAD_NAME', 101)       # Index in name list
-def_op('BUILD_TUPLE', 102)      # Number of tuple items
-def_op('BUILD_LIST', 103)       # Number of list items
-def_op('BUILD_SET', 104)        # Number of set items
-def_op('BUILD_MAP', 105)        # Number of dict entries (upto 255)
-name_op('LOAD_ATTR', 106)       # Index in name list
-def_op('COMPARE_OP', 107)       # Comparison operator
-hascompare.append(107)
-name_op('IMPORT_NAME', 108)     # Index in name list
-name_op('IMPORT_FROM', 109)     # Index in name list
-
-jrel_op('JUMP_FORWARD', 110)    # Number of bytes to skip
-jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code
-jabs_op('JUMP_IF_TRUE_OR_POP', 112)  # ""
-jabs_op('JUMP_ABSOLUTE', 113)        # ""
-jabs_op('POP_JUMP_IF_FALSE', 114)    # ""
-jabs_op('POP_JUMP_IF_TRUE', 115)     # ""
-
-name_op('LOAD_GLOBAL', 116)     # Index in name list
-
-jabs_op('CONTINUE_LOOP', 119)   # Target address
-jrel_op('SETUP_LOOP', 120)      # Distance to target address
-jrel_op('SETUP_EXCEPT', 121)    # ""
-jrel_op('SETUP_FINALLY', 122)   # ""
-
-def_op('LOAD_FAST', 124)        # Local variable number
-haslocal.append(124)
-def_op('STORE_FAST', 125)       # Local variable number
-haslocal.append(125)
-def_op('DELETE_FAST', 126)      # Local variable number
-haslocal.append(126)
-
-def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
-def_op('CALL_FUNCTION', 131)    # #args + (#kwargs << 8)
-def_op('MAKE_FUNCTION', 132)    # Number of args with default values
-def_op('BUILD_SLICE', 133)      # Number of items
-def_op('MAKE_CLOSURE', 134)
-def_op('LOAD_CLOSURE', 135)
-hasfree.append(135)
-def_op('LOAD_DEREF', 136)
-hasfree.append(136)
-def_op('STORE_DEREF', 137)
-hasfree.append(137)
-def_op('DELETE_DEREF', 138)
-hasfree.append(138)
-
-def_op('CALL_FUNCTION_VAR', 140)     # #args + (#kwargs << 8)
-def_op('CALL_FUNCTION_KW', 141)      # #args + (#kwargs << 8)
-def_op('CALL_FUNCTION_VAR_KW', 142)  # #args + (#kwargs << 8)
-
-jrel_op('SETUP_WITH', 143)
-
-def_op('LIST_APPEND', 145)
-def_op('SET_ADD', 146)
-def_op('MAP_ADD', 147)
-
-def_op('EXTENDED_ARG', 144)
-EXTENDED_ARG = 144
-
-# pypy modification, experimental bytecode
-def_op('LOOKUP_METHOD', 201)          # Index in name list
-hasname.append(201)
-def_op('CALL_METHOD', 202)            # #args not including 'self'
-def_op('BUILD_LIST_FROM_ARG', 203)
-
-del def_op, name_op, jrel_op, jabs_op
diff --git a/lib-python/modified-3.2/sysconfig.py b/lib-python/modified-3.2/sysconfig.py
deleted file mode 100644
--- a/lib-python/modified-3.2/sysconfig.py
+++ /dev/null
@@ -1,625 +0,0 @@
-"""Provide access to Python's configuration information.
-
-"""
-import sys
-import os
-from os.path import pardir, realpath
-
-__all__ = [
-    'get_config_h_filename',
-    'get_config_var',
-    'get_config_vars',
-    'get_makefile_filename',
-    'get_path',
-    'get_path_names',
-    'get_paths',
-    'get_platform',
-    'get_python_version',
-    'get_scheme_names',
-    'parse_config_h',
-    ]
-
-_INSTALL_SCHEMES = {
-    'posix_prefix': {
-        'stdlib': '{base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
-        'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
-        'include':
-            '{base}/include/python{py_version_short}{abiflags}',
-        'platinclude':
-            '{platbase}/include/python{py_version_short}{abiflags}',
-        'scripts': '{base}/bin',
-        'data': '{base}',
-        },
-    'posix_home': {
-        'stdlib': '{base}/lib/python',
-        'platstdlib': '{base}/lib/python',
-        'purelib': '{base}/lib/python',
-        'platlib': '{base}/lib/python',
-        'include': '{base}/include/python',
-        'platinclude': '{base}/include/python',
-        'scripts': '{base}/bin',
-        'data'   : '{base}',
-        },
-    'pypy': {
-        'stdlib': '{base}/lib-python',
-        'platstdlib': '{base}/lib-python',
-        'purelib': '{base}/lib-python',
-        'platlib': '{base}/lib-python',
-        'include': '{base}/include',
-        'platinclude': '{base}/include',
-        'scripts': '{base}/bin',
-        'data'   : '{base}',
-        },
-    'nt': {
-        'stdlib': '{base}/Lib',
-        'platstdlib': '{base}/Lib',
-        'purelib': '{base}/Lib/site-packages',
-        'platlib': '{base}/Lib/site-packages',
-        'include': '{base}/Include',
-        'platinclude': '{base}/Include',
-        'scripts': '{base}/Scripts',
-        'data'   : '{base}',
-        },
-    'os2': {
-        'stdlib': '{base}/Lib',
-        'platstdlib': '{base}/Lib',
-        'purelib': '{base}/Lib/site-packages',
-        'platlib': '{base}/Lib/site-packages',
-        'include': '{base}/Include',
-        'platinclude': '{base}/Include',
-        'scripts': '{base}/Scripts',
-        'data'   : '{base}',
-        },
-    'os2_home': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
-        'scripts': '{userbase}/bin',
-        'data'   : '{userbase}',
-        },
-    'nt_user': {
-        'stdlib': '{userbase}/Python{py_version_nodot}',
-        'platstdlib': '{userbase}/Python{py_version_nodot}',
-        'purelib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'platlib': '{userbase}/Python{py_version_nodot}/site-packages',
-        'include': '{userbase}/Python{py_version_nodot}/Include',
-        'scripts': '{userbase}/Scripts',
-        'data'   : '{userbase}',
-        },
-    'posix_user': {
-        'stdlib': '{userbase}/lib/python{py_version_short}',
-        'platstdlib': '{userbase}/lib/python{py_version_short}',
-        'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
-        'include': '{userbase}/include/python{py_version_short}',
-        'scripts': '{userbase}/bin',
-        'data'   : '{userbase}',
-        },
-    'osx_framework_user': {
-        'stdlib': '{userbase}/lib/python',
-        'platstdlib': '{userbase}/lib/python',
-        'purelib': '{userbase}/lib/python/site-packages',
-        'platlib': '{userbase}/lib/python/site-packages',
-        'include': '{userbase}/include',
-        'scripts': '{userbase}/bin',
-        'data'   : '{userbase}',
-        },
-    }
-
-_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include',
-                'scripts', 'data')
-_PY_VERSION = sys.version.split()[0]
-_PY_VERSION_SHORT = sys.version[:3]
-_PY_VERSION_SHORT_NO_DOT = _PY_VERSION[0] + _PY_VERSION[2]
-_PREFIX = os.path.normpath(sys.prefix)
-_EXEC_PREFIX = os.path.normpath(sys.exec_prefix)
-_CONFIG_VARS = None
-_USER_BASE = None
-
-def _safe_realpath(path):
-    try:
-        return realpath(path)
-    except OSError:
-        return path
-
-if sys.executable:
-    _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable))
-else:
-    # sys.executable can be empty if argv[0] has been changed and Python is
-    # unable to retrieve the real program name
-    _PROJECT_BASE = _safe_realpath(os.getcwd())
-
-if os.name == "nt" and "pcbuild" in _PROJECT_BASE[-8:].lower():
-    _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir))
-# PC/VS7.1
-if os.name == "nt" and "\\pc\\v" in _PROJECT_BASE[-10:].lower():
-    _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir))
-# PC/AMD64
-if os.name == "nt" and "\\pcbuild\\amd64" in _PROJECT_BASE[-14:].lower():
-    _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir))
-
-def is_python_build():
-    for fn in ("Setup.dist", "Setup.local"):
-        if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)):
-            return True
-    return False
-
-_PYTHON_BUILD = is_python_build()
-
-if _PYTHON_BUILD:
-    for scheme in ('posix_prefix', 'posix_home'):
-        _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include'
-        _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.'
-
-def _subst_vars(s, local_vars):
-    try:
-        return s.format(**local_vars)
-    except KeyError:
-        try:
-            return s.format(**os.environ)
-        except KeyError as var:
-            raise AttributeError('{%s}' % var)
-
-def _extend_dict(target_dict, other_dict):
-    target_keys = target_dict.keys()
-    for key, value in other_dict.items():
-        if key in target_keys:
-            continue
-        target_dict[key] = value
-
-def _expand_vars(scheme, vars):
-    res = {}
-    if vars is None:
-        vars = {}
-    _extend_dict(vars, get_config_vars())
-
-    for key, value in _INSTALL_SCHEMES[scheme].items():
-        if os.name in ('posix', 'nt'):
-            value = os.path.expanduser(value)
-        res[key] = os.path.normpath(_subst_vars(value, vars))
-    return res
-
-def _get_default_scheme():
-    if '__pypy__' in sys.builtin_module_names:
-        return 'pypy'
-    elif os.name == 'posix':
-        # the default scheme for posix is posix_prefix
-        return 'posix_prefix'
-    return os.name
-
-def _getuserbase():
-    env_base = os.environ.get("PYTHONUSERBASE", None)
-    def joinuser(*args):
-        return os.path.expanduser(os.path.join(*args))
-
-    # what about 'os2emx', 'riscos' ?
-    if os.name == "nt":
-        base = os.environ.get("APPDATA") or "~"
-        return env_base if env_base else joinuser(base, "Python")
-
-    if sys.platform == "darwin":
-        framework = get_config_var("PYTHONFRAMEWORK")
-        if framework:
-            return env_base if env_base else joinuser("~", "Library", framework, "%d.%d"%(
-                sys.version_info[:2]))
-
-    return env_base if env_base else joinuser("~", ".local")
-
-
-def _init_posix(vars):
-    """Initialize the module as appropriate for POSIX systems."""
-    return
-
-def _init_non_posix(vars):
-    """Initialize the module as appropriate for NT"""
-    # set basic install directories
-    vars['LIBDEST'] = get_path('stdlib')
-    vars['BINLIBDEST'] = get_path('platstdlib')
-    vars['INCLUDEPY'] = get_path('include')
-    vars['SO'] = '.pyd'
-    vars['EXE'] = '.exe'
-    vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT
-    vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable))
-
-#
-# public APIs
-#
-
-
-def parse_config_h(fp, vars=None):
-    """Parse a config.h-style file.
-
-    A dictionary containing name/value pairs is returned.  If an
-    optional dictionary is passed in as the second argument, it is
-    used instead of a new dictionary.
-    """
-    import re
-    if vars is None:
-        vars = {}
-    define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
-    undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
-
-    while True:
-        line = fp.readline()
-        if not line:
-            break
-        m = define_rx.match(line)
-        if m:
-            n, v = m.group(1, 2)
-            try: v = int(v)
-            except ValueError: pass
-            vars[n] = v
-        else:
-            m = undef_rx.match(line)
-            if m:
-                vars[m.group(1)] = 0
-    return vars
-
-def get_config_h_filename():
-    """Return the path of pyconfig.h."""
-    if _PYTHON_BUILD:
-        if os.name == "nt":
-            inc_dir = os.path.join(_PROJECT_BASE, "PC")
-        else:
-            inc_dir = _PROJECT_BASE
-    else:
-        inc_dir = get_path('platinclude')
-    return os.path.join(inc_dir, 'pyconfig.h')
-
-def get_scheme_names():
-    """Return a tuple containing the schemes names."""
-    schemes = list(_INSTALL_SCHEMES.keys())
-    schemes.sort()
-    return tuple(schemes)
-
-def get_path_names():
-    """Return a tuple containing the paths names."""
-    return _SCHEME_KEYS
-
-def get_paths(scheme=_get_default_scheme(), vars=None, expand=True):
-    """Return a mapping containing an install scheme.
-
-    ``scheme`` is the install scheme name. If not provided, it will
-    return the default scheme for the current platform.
-    """
-    if expand:
-        return _expand_vars(scheme, vars)
-    else:
-        return _INSTALL_SCHEMES[scheme]
-
-def get_path(name, scheme=_get_default_scheme(), vars=None, expand=True):
-    """Return a path corresponding to the scheme.
-
-    ``scheme`` is the install scheme name.
-    """
-    return get_paths(scheme, vars, expand)[name]
-
-def get_config_vars(*args):
-    """With no arguments, return a dictionary of all configuration
-    variables relevant for the current platform.
-
-    On Unix, this means every variable defined in Python's installed Makefile;
-    On Windows and Mac OS it's a much smaller set.
-
-    With arguments, return a list of values that result from looking up
-    each argument in the configuration variable dictionary.
-    """
-    import re
-    global _CONFIG_VARS
-    if _CONFIG_VARS is None:
-        _CONFIG_VARS = {}
-        # Normalized versions of prefix and exec_prefix are handy to have;
-        # in fact, these are the standard versions used most places in the
-        # Distutils.
-        _CONFIG_VARS['prefix'] = _PREFIX
-        _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX
-        _CONFIG_VARS['py_version'] = _PY_VERSION
-        _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT
-        _CONFIG_VARS['py_version_nodot'] = _PY_VERSION[0] + _PY_VERSION[2]
-        _CONFIG_VARS['base'] = _PREFIX
-        _CONFIG_VARS['platbase'] = _EXEC_PREFIX
-        _CONFIG_VARS['projectbase'] = _PROJECT_BASE
-        try:
-            _CONFIG_VARS['abiflags'] = sys.abiflags
-        except AttributeError:
-            # sys.abiflags may not be defined on all platforms.
-            _CONFIG_VARS['abiflags'] = ''
-
-        if os.name in ('nt', 'os2'):
-            _init_non_posix(_CONFIG_VARS)
-        if os.name == 'posix':
-            _init_posix(_CONFIG_VARS)
-        # Setting 'userbase' is done below the call to the
-        # init function to enable using 'get_config_var' in
-        # the init-function.
-        _CONFIG_VARS['userbase'] = _getuserbase()
-
-        if 'srcdir' not in _CONFIG_VARS:
-            _CONFIG_VARS['srcdir'] = _PROJECT_BASE
-        else:
-            _CONFIG_VARS['srcdir'] = _safe_realpath(_CONFIG_VARS['srcdir'])
-
-
-        # Convert srcdir into an absolute path if it appears necessary.
-        # Normally it is relative to the build directory.  However, during
-        # testing, for example, we might be running a non-installed python
-        # from a different directory.
-        if _PYTHON_BUILD and os.name == "posix":
-            base = _PROJECT_BASE
-            try:
-                cwd = os.getcwd()
-            except OSError:
-                cwd = None
-            if (not os.path.isabs(_CONFIG_VARS['srcdir']) and
-                base != cwd):
-                # srcdir is relative and we are not in the same directory
-                # as the executable. Assume executable is in the build
-                # directory and make srcdir absolute.
-                srcdir = os.path.join(base, _CONFIG_VARS['srcdir'])
-                _CONFIG_VARS['srcdir'] = os.path.normpath(srcdir)
-
-        if sys.platform == 'darwin':
-            kernel_version = os.uname()[2] # Kernel version (8.4.3)
-            major_version = int(kernel_version.split('.')[0])
-
-            if major_version < 8:
-                # On Mac OS X before 10.4, check if -arch and -isysroot
-                # are in CFLAGS or LDFLAGS and remove them if they are.
-                # This is needed when building extensions on a 10.3 system
-                # using a universal build of python.
-                for key in ('LDFLAGS', 'BASECFLAGS',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-                    flags = _CONFIG_VARS[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
-                    _CONFIG_VARS[key] = flags
-            else:
-                # Allow the user to override the architecture flags using
-                # an environment variable.
-                # NOTE: This name was introduced by Apple in OSX 10.5 and
-                # is used by several scripting languages distributed with
-                # that OS release.
-                if 'ARCHFLAGS' in os.environ:
-                    arch = os.environ['ARCHFLAGS']
-                    for key in ('LDFLAGS', 'BASECFLAGS',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                        flags = _CONFIG_VARS[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _CONFIG_VARS[key] = flags
-
-                # If we're on OSX 10.5 or later and the user tries to
-                # compiles an extension using an SDK that is not present
-                # on the current machine it is better to not use an SDK
-                # than to fail.
-                #
-                # The major usecase for this is users using a Python.org
-                # binary installer  on OSX 10.6: that installer uses
-                # the 10.4u SDK, but that SDK is not installed by default
-                # when you install Xcode.
-                #
-                CFLAGS = _CONFIG_VARS.get('CFLAGS', '')
-                m = re.search('-isysroot\s+(\S+)', CFLAGS)
-                if m is not None:
-                    sdk = m.group(1)
-                    if not os.path.exists(sdk):
-                        for key in ('LDFLAGS', 'BASECFLAGS',
-                             # a number of derived variables. These need to be
-                             # patched up as well.
-                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                            flags = _CONFIG_VARS[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
-                            _CONFIG_VARS[key] = flags
-
-    if args:
-        vals = []
-        for name in args:
-            vals.append(_CONFIG_VARS.get(name))
-        return vals
-    else:
-        return _CONFIG_VARS
-
-def get_config_var(name):
-    """Return the value of a single variable using the dictionary returned by
-    'get_config_vars()'.
-
-    Equivalent to get_config_vars().get(name)
-    """
-    return get_config_vars().get(name)
-
-def get_platform():
-    """Return a string that identifies the current platform.
-
-    This is used mainly to distinguish platform-specific build directories and
-    platform-specific built distributions.  Typically includes the OS name
-    and version and the architecture (as supplied by 'os.uname()'),
-    although the exact information included depends on the OS; eg. for IRIX
-    the architecture isn't particularly important (IRIX only runs on SGI
-    hardware), but for Linux the kernel version isn't particularly
-    important.
-
-    Examples of returned values:
-       linux-i586
-       linux-alpha (?)
-       solaris-2.6-sun4u
-       irix-5.3
-       irix64-6.2
-
-    Windows will return one of:
-       win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc)
-       win-ia64 (64bit Windows on Itanium)
-       win32 (all others - specifically, sys.platform is returned)
-
-    For other non-POSIX platforms, currently just returns 'sys.platform'.
-    """
-    import re
-    if os.name == 'nt':
-        # sniff sys.version for architecture.
-        prefix = " bit ("
-        i = sys.version.find(prefix)
-        if i == -1:
-            return sys.platform
-        j = sys.version.find(")", i)
-        look = sys.version[i+len(prefix):j].lower()
-        if look == 'amd64':
-            return 'win-amd64'
-        if look == 'itanium':
-            return 'win-ia64'
-        return sys.platform
-
-    if os.name != "posix" or not hasattr(os, 'uname'):
-        # XXX what about the architecture? NT is Intel or Alpha,
-        # Mac OS is M68k or PPC, etc.
-        return sys.platform
-
-    # Try to distinguish various flavours of Unix
-    osname, host, release, version, machine = os.uname()
-
-    # Convert the OS name to lowercase, remove '/' characters
-    # (to accommodate BSD/OS), and translate spaces (for "Power Macintosh")
-    osname = osname.lower().replace('/', '')
-    machine = machine.replace(' ', '_')
-    machine = machine.replace('/', '-')
-
-    if osname[:5] == "linux":
-        # At least on Linux/Intel, 'machine' is the processor --
-        # i386, etc.
-        # XXX what about Alpha, SPARC, etc?
-        return  "%s-%s" % (osname, machine)
-    elif osname[:5] == "sunos":
-        if release[0] >= "5":           # SunOS 5 == Solaris 2
-            osname = "solaris"
-            release = "%d.%s" % (int(release[0]) - 3, release[2:])
-        # fall through to standard osname-release-machine representation
-    elif osname[:4] == "irix":              # could be "irix64"!
-        return "%s-%s" % (osname, release)
-    elif osname[:3] == "aix":
-        return "%s-%s.%s" % (osname, version, release)
-    elif osname[:6] == "cygwin":
-        osname = "cygwin"
-        rel_re = re.compile (r'[\d.]+')
-        m = rel_re.match(release)
-        if m:
-            release = m.group()
-    elif osname[:6] == "darwin":
-        #
-        # For our purposes, we'll assume that the system version from
-        # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set
-        # to. This makes the compatibility story a bit more sane because the
-        # machine is going to compile and link as if it were
-        # MACOSX_DEPLOYMENT_TARGET.
-        #
-        cfgvars = get_config_vars()
-        macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
-
-        if 1:
-            # Always calculate the release of the running machine,
-            # needed to determine if we can build fat binaries or not.
-
-            macrelease = macver
-            # Get the system version. Reading this plist is a documented
-            # way to get the system version (see the documentation for
-            # the Gestalt Manager)
-            try:
-                f = open('/System/Library/CoreServices/SystemVersion.plist')
-            except IOError:
-                # We're on a plain darwin box, fall back to the default
-                # behaviour.
-                pass
-            else:
-                try:
-                    m = re.search(
-                            r'<key>ProductUserVisibleVersion</key>\s*' +
-                            r'<string>(.*?)</string>', f.read())
-                    if m is not None:
-                        macrelease = '.'.join(m.group(1).split('.')[:2])
-                    # else: fall back to the default behaviour
-                finally:
-                    f.close()
-
-        if not macver:
-            macver = macrelease
-
-        if macver:
-            release = macver
-            osname = "macosx"
-
-            if (macrelease + '.') >= '10.4.' and \
-                    '-arch' in get_config_vars().get('CFLAGS', '').strip():
-                # The universal build will build fat binaries, but not on
-                # systems before 10.4
-                #
-                # Try to detect 4-way universal builds, those have machine-type
-                # 'universal' instead of 'fat'.
-
-                machine = 'fat'
-                cflags = get_config_vars().get('CFLAGS')
-
-                archs = re.findall('-arch\s+(\S+)', cflags)
-                archs = tuple(sorted(set(archs)))
-
-                if len(archs) == 1:
-                    machine = archs[0]
-                elif archs == ('i386', 'ppc'):
-                    machine = 'fat'
-                elif archs == ('i386', 'x86_64'):
-                    machine = 'intel'
-                elif archs == ('i386', 'ppc', 'x86_64'):
-                    machine = 'fat3'
-                elif archs == ('ppc64', 'x86_64'):
-                    machine = 'fat64'
-                elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'):
-                    machine = 'universal'
-                else:
-                    raise ValueError(
-                       "Don't know machine value for archs=%r"%(archs,))
-
-            elif machine == 'i386':
-                # On OSX the machine type returned by uname is always the
-                # 32-bit variant, even if the executable architecture is
-                # the 64-bit variant
-                if sys.maxsize >= 2**32:
-                    machine = 'x86_64'
-
-            elif machine in ('PowerPC', 'Power_Macintosh'):
-                # Pick a sane name for the PPC architecture.
-                # See 'i386' case
-                if sys.maxsize >= 2**32:
-                    machine = 'ppc64'
-                else:
-                    machine = 'ppc'
-
-    return "%s-%s-%s" % (osname, release, machine)
-
-
-def get_python_version():
-    return _PY_VERSION_SHORT
-
-def _print_dict(title, data):
-    for index, (key, value) in enumerate(sorted(data.items())):
-        if index == 0:
-            print('{0}: '.format(title))
-        print('\t{0} = "{1}"'.format(key, value))
-
-def _main():
-    """Display all information sysconfig detains."""
-    print('Platform: "{0}"'.format(get_platform()))
-    print('Python version: "{0}"'.format(get_python_version()))
-    print('Current installation scheme: "{0}"'.format(_get_default_scheme()))
-    print('')
-    _print_dict('Paths', get_paths())
-    print('')
-    _print_dict('Variables', get_config_vars())
-
-if __name__ == '__main__':
-    _main()
diff --git a/lib-python/modified-3.2/test/test_sysconfig.py b/lib-python/modified-3.2/test/test_sysconfig.py
deleted file mode 100644
--- a/lib-python/modified-3.2/test/test_sysconfig.py
+++ /dev/null
@@ -1,362 +0,0 @@
-"""Tests for 'site'.
-
-Tests assume the initial paths in sys.path once the interpreter has begun
-executing have not been removed.
-
-"""
-import unittest
-import sys
-import os
-import subprocess
-import shutil
-from copy import copy, deepcopy
-
-from test.support import (run_unittest, TESTFN, unlink, get_attribute,
-                          captured_stdout, skip_unless_symlink)
-
-import sysconfig
-from sysconfig import (get_paths, get_platform, get_config_vars,
-                       get_path, get_path_names, _INSTALL_SCHEMES,
-                       _get_default_scheme, _expand_vars,
-                       get_scheme_names, get_config_var, _main)
-
-class TestSysConfig(unittest.TestCase):
-
-    def setUp(self):
-        """Make a copy of sys.path"""
-        super(TestSysConfig, self).setUp()
-        self.sys_path = sys.path[:]
-        # patching os.uname
-        if hasattr(os, 'uname'):
-            self.uname = os.uname
-            self._uname = os.uname()
-        else:
-            self.uname = None
-            self._uname = None
-        os.uname = self._get_uname
-        # saving the environment
-        self.name = os.name
-        self.platform = sys.platform
-        self.version = sys.version
-        self.sep = os.sep
-        self.join = os.path.join
-        self.isabs = os.path.isabs
-        self.splitdrive = os.path.splitdrive
-        self._config_vars = copy(sysconfig._CONFIG_VARS)
-        self.old_environ = deepcopy(os.environ)
-
-    def tearDown(self):
-        """Restore sys.path"""
-        sys.path[:] = self.sys_path
-        self._cleanup_testfn()
-        if self.uname is not None:
-            os.uname = self.uname
-        else:
-            del os.uname
-        os.name = self.name
-        sys.platform = self.platform
-        sys.version = self.version
-        os.sep = self.sep
-        os.path.join = self.join
-        os.path.isabs = self.isabs
-        os.path.splitdrive = self.splitdrive
-        sysconfig._CONFIG_VARS = copy(self._config_vars)
-        for key, value in self.old_environ.items():
-            if os.environ.get(key) != value:
-                os.environ[key] = value
-
-        for key in list(os.environ.keys()):
-            if key not in self.old_environ:
-                del os.environ[key]
-
-        super(TestSysConfig, self).tearDown()
-
-    def _set_uname(self, uname):
-        self._uname = uname
-
-    def _get_uname(self):
-        return self._uname
-
-    def _cleanup_testfn(self):
-        path = TESTFN
-        if os.path.isfile(path):
-            os.remove(path)
-        elif os.path.isdir(path):
-            shutil.rmtree(path)
-
-    def test_get_path_names(self):
-        self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS)
-
-    def test_get_paths(self):
-        scheme = get_paths()
-        default_scheme = _get_default_scheme()
-        wanted = _expand_vars(default_scheme, None)
-        wanted = list(wanted.items())
-        wanted.sort()
-        scheme = list(scheme.items())
-        scheme.sort()
-        self.assertEqual(scheme, wanted)
-
-    def test_get_path(self):
-        # xxx make real tests here
-        for scheme in _INSTALL_SCHEMES:
-            for name in _INSTALL_SCHEMES[scheme]:
-                res = get_path(name, scheme)
-
-    def test_get_config_vars(self):
-        cvars = get_config_vars()
-        self.assertTrue(isinstance(cvars, dict))
-        self.assertTrue(cvars)
-
-    def test_get_platform(self):
-        # windows XP, 32bits
-        os.name = 'nt'
-        sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) '
-                       '[MSC v.1310 32 bit (Intel)]')
-        sys.platform = 'win32'
-        self.assertEqual(get_platform(), 'win32')
-
-        # windows XP, amd64
-        os.name = 'nt'
-        sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) '
-                       '[MSC v.1310 32 bit (Amd64)]')
-        sys.platform = 'win32'
-        self.assertEqual(get_platform(), 'win-amd64')
-
-        # windows XP, itanium
-        os.name = 'nt'
-        sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) '
-                       '[MSC v.1310 32 bit (Itanium)]')
-        sys.platform = 'win32'
-        self.assertEqual(get_platform(), 'win-ia64')
-
-        # macbook
-        os.name = 'posix'
-        sys.version = ('2.5 (r25:51918, Sep 19 2006, 08:49:13) '
-                       '\n[GCC 4.0.1 (Apple Computer, Inc. build 5341)]')
-        sys.platform = 'darwin'
-        self._set_uname(('Darwin', 'macziade', '8.11.1',
-                   ('Darwin Kernel Version 8.11.1: '
-                    'Wed Oct 10 18:23:28 PDT 2007; '
-                    'root:xnu-792.25.20~1/RELEASE_I386'), 'PowerPC'))
-
-
-        get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3'
-
-        get_config_vars()['CFLAGS'] = ('-fno-strict-aliasing -DNDEBUG -g '
-                                       '-fwrapv -O3 -Wall -Wstrict-prototypes')
-
-        maxint = sys.maxsize
-        try:
-            sys.maxsize = 2147483647
-            self.assertEqual(get_platform(), 'macosx-10.3-ppc')
-            sys.maxsize = 9223372036854775807
-            self.assertEqual(get_platform(), 'macosx-10.3-ppc64')
-        finally:
-            sys.maxsize = maxint
-
-
-        self._set_uname(('Darwin', 'macziade', '8.11.1',
-                   ('Darwin Kernel Version 8.11.1: '
-                    'Wed Oct 10 18:23:28 PDT 2007; '
-                    'root:xnu-792.25.20~1/RELEASE_I386'), 'i386'))
-        get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3'
-        get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3'
-
-        get_config_vars()['CFLAGS'] = ('-fno-strict-aliasing -DNDEBUG -g '
-                                       '-fwrapv -O3 -Wall -Wstrict-prototypes')
-        maxint = sys.maxsize
-        try:
-            sys.maxsize = 2147483647
-            self.assertEqual(get_platform(), 'macosx-10.3-i386')
-            sys.maxsize = 9223372036854775807
-            self.assertEqual(get_platform(), 'macosx-10.3-x86_64')
-        finally:
-            sys.maxsize = maxint
-
-        # macbook with fat binaries (fat, universal or fat64)
-        get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.4'
-        get_config_vars()['CFLAGS'] = ('-arch ppc -arch i386 -isysroot '
-                                       '/Developer/SDKs/MacOSX10.4u.sdk  '
-                                       '-fno-strict-aliasing -fno-common '
-                                       '-dynamic -DNDEBUG -g -O3')
-
-        self.assertEqual(get_platform(), 'macosx-10.4-fat')
-
-        get_config_vars()['CFLAGS'] = ('-arch x86_64 -arch i386 -isysroot '
-                                       '/Developer/SDKs/MacOSX10.4u.sdk  '
-                                       '-fno-strict-aliasing -fno-common '
-                                       '-dynamic -DNDEBUG -g -O3')
-
-        self.assertEqual(get_platform(), 'macosx-10.4-intel')
-
-        get_config_vars()['CFLAGS'] = ('-arch x86_64 -arch ppc -arch i386 -isysroot '
-                                       '/Developer/SDKs/MacOSX10.4u.sdk  '
-                                       '-fno-strict-aliasing -fno-common '
-                                       '-dynamic -DNDEBUG -g -O3')
-        self.assertEqual(get_platform(), 'macosx-10.4-fat3')
-
-        get_config_vars()['CFLAGS'] = ('-arch ppc64 -arch x86_64 -arch ppc -arch i386 -isysroot '
-                                       '/Developer/SDKs/MacOSX10.4u.sdk  '
-                                       '-fno-strict-aliasing -fno-common '
-                                       '-dynamic -DNDEBUG -g -O3')
-        self.assertEqual(get_platform(), 'macosx-10.4-universal')
-
-        get_config_vars()['CFLAGS'] = ('-arch x86_64 -arch ppc64 -isysroot '
-                                       '/Developer/SDKs/MacOSX10.4u.sdk  '
-                                       '-fno-strict-aliasing -fno-common '
-                                       '-dynamic -DNDEBUG -g -O3')
-
-        self.assertEqual(get_platform(), 'macosx-10.4-fat64')
-
-        for arch in ('ppc', 'i386', 'x86_64', 'ppc64'):
-            get_config_vars()['CFLAGS'] = ('-arch %s -isysroot '
-                                           '/Developer/SDKs/MacOSX10.4u.sdk  '
-                                           '-fno-strict-aliasing -fno-common '
-                                           '-dynamic -DNDEBUG -g -O3'%(arch,))
-
-            self.assertEqual(get_platform(), 'macosx-10.4-%s'%(arch,))
-
-        # linux debian sarge
-        os.name = 'posix'
-        sys.version = ('2.3.5 (#1, Jul  4 2007, 17:28:59) '
-                       '\n[GCC 4.1.2 20061115 (prerelease) (Debian 4.1.1-21)]')
-        sys.platform = 'linux2'
-        self._set_uname(('Linux', 'aglae', '2.6.21.1dedibox-r7',
-                    '#1 Mon Apr 30 17:25:38 CEST 2007', 'i686'))
-
-        self.assertEqual(get_platform(), 'linux-i686')
-
-        # XXX more platforms to tests here
-
-    def test_get_config_h_filename(self):
-        config_h = sysconfig.get_config_h_filename()
-        self.assertTrue(os.path.isfile(config_h), config_h)
-
-    def test_get_scheme_names(self):
-        wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user',
-                  'posix_home', 'posix_prefix', 'posix_user', 'pypy')
-        self.assertEqual(get_scheme_names(), wanted)
-
-    @skip_unless_symlink
-    def test_symlink(self):
-        # On Windows, the EXE needs to know where pythonXY.dll is at so we have
-        # to add the directory to the path.
-        if sys.platform == "win32":
-            os.environ["Path"] = "{};{}".format(
-                os.path.dirname(sys.executable), os.environ["Path"])
-
-        # Issue 7880
-        def get(python):
-            cmd = [python, '-c',
-                   'import sysconfig; print(sysconfig.get_platform())']
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=os.environ)
-            return p.communicate()
-        real = os.path.realpath(sys.executable)
-        link = os.path.abspath(TESTFN)
-        os.symlink(real, link)
-        try:
-            self.assertEqual(get(real), get(link))
-        finally:
-            unlink(link)
-
-    def test_user_similar(self):
-        # Issue 8759 : make sure the posix scheme for the users
-        # is similar to the global posix_prefix one
-        base = get_config_var('base')
-        user = get_config_var('userbase')
-        for name in ('stdlib', 'platstdlib', 'purelib', 'platlib'):
-            global_path = get_path(name, 'posix_prefix')
-            user_path = get_path(name, 'posix_user')
-            self.assertEqual(user_path, global_path.replace(base, user))
-
-    def test_main(self):
-        # just making sure _main() runs and returns things in the stdout
-        with captured_stdout() as output:
-            _main()
-        self.assertTrue(len(output.getvalue().split('\n')) > 0)
-
-    @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows")
-    def test_ldshared_value(self):
-        ldflags = sysconfig.get_config_var('LDFLAGS')
-        ldshared = sysconfig.get_config_var('LDSHARED')
-
-        self.assertIn(ldflags, ldshared)
-
-
-    @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX")
-    def test_platform_in_subprocess(self):
-        my_platform = sysconfig.get_platform()
-
-        # Test without MACOSX_DEPLOYMENT_TARGET in the environment
-
-        env = os.environ.copy()
-        if 'MACOSX_DEPLOYMENT_TARGET' in env:
-            del env['MACOSX_DEPLOYMENT_TARGET']
-
-        with open('/dev/null', 'w') as devnull_fp:
-            p = subprocess.Popen([
-                    sys.executable, '-c',
-                    'import sysconfig; print(sysconfig.get_platform())',
-                ],
-                stdout=subprocess.PIPE,
-                stderr=devnull_fp,
-                env=env)
-        test_platform = p.communicate()[0].strip()
-        test_platform = test_platform.decode('utf-8')
-        status = p.wait()
-
-        self.assertEqual(status, 0)
-        self.assertEqual(my_platform, test_platform)
-
-
-        # Test with MACOSX_DEPLOYMENT_TARGET in the environment, and
-        # using a value that is unlikely to be the default one.
-        env = os.environ.copy()
-        env['MACOSX_DEPLOYMENT_TARGET'] = '10.1'
-
-        p = subprocess.Popen([
-                sys.executable, '-c',
-                'import sysconfig; print(sysconfig.get_platform())',
-            ],
-            stdout=subprocess.PIPE,
-            stderr=open('/dev/null'),
-            env=env)
-        test_platform = p.communicate()[0].strip()
-        test_platform = test_platform.decode('utf-8')
-        status = p.wait()
-
-        self.assertEqual(status, 0)
-        self.assertEqual(my_platform, test_platform)
-
-
-class MakefileTests(unittest.TestCase):
-    @unittest.skipIf(sys.platform.startswith('win'),
-                     'Test is not Windows compatible')
-    def test_get_makefile_filename(self):
-        makefile = sysconfig.get_makefile_filename()
-        self.assertTrue(os.path.isfile(makefile), makefile)
-
-    def test_parse_makefile(self):
-        self.addCleanup(unlink, TESTFN)
-        with open(TESTFN, "w") as makefile:
-            print("var1=a$(VAR2)", file=makefile)
-            print("VAR2=b$(var3)", file=makefile)
-            print("var3=42", file=makefile)
-            print("var4=$/invalid", file=makefile)
-            print("var5=dollar$$5", file=makefile)
-        vars = sysconfig._parse_makefile(TESTFN)
-        self.assertEqual(vars, {
-            'var1': 'ab42',
-            'VAR2': 'b42',
-            'var3': 42,
-            'var4': '$/invalid',
-            'var5': 'dollar$5',
-        })
-
-
-def test_main():
-    run_unittest(TestSysConfig, MakefileTests)
-
-if __name__ == "__main__":
-    test_main()

From noreply at buildbot.pypy.org  Wed Apr 18 15:03:23 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Apr 2012 15:03:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: (arigo, bivab) remove unused imports
Message-ID: <20120418130323.79E93820B3@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r54493:79034a42a195
Date: 2012-02-16 16:51 +0100
http://bitbucket.org/pypy/pypy/changeset/79034a42a195/

Log:	(arigo, bivab) remove unused imports

diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct

From noreply at buildbot.pypy.org  Wed Apr 18 15:03:24 2012
From: noreply at buildbot.pypy.org (bivab)
Date: Wed, 18 Apr 2012 15:03:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: (cfbolz,
 bivab) mark readdir as macro in the implementation of listdir
Message-ID: <20120418130324.B5ABB82F4F@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: 
Changeset: r54494:9cda37139e0d
Date: 2012-04-18 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/9cda37139e0d/

Log:	(cfbolz, bivab) mark readdir as macro in the implementation of
	listdir

diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1168,8 +1168,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 

From noreply at buildbot.pypy.org  Wed Apr 18 16:01:54 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Apr 2012 16:01:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix typo and skip more tests
Message-ID: <20120418140154.4CD86820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54495:e664176f1c0c
Date: 2012-04-18 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/e664176f1c0c/

Log:	fix typo and skip more tests

diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
--- a/pypy/module/_continuation/test/test_zpickle.py
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -1,9 +1,10 @@
+import py
 from pypy.conftest import gettestobjspace
 
 
 class AppTestCopy:
     def setup_class(cls):
-        py3k_skip("_continuation not supported yet")
+        py.test.py3k_skip("_continuation not supported yet")
         cls.space = gettestobjspace(usemodules=('_continuation',),
                                     CALL_METHOD=True)
         cls.space.config.translation.continuation = True
@@ -107,6 +108,7 @@
     version = 0
 
     def setup_class(cls):
+        py.test.py3k_skip("_continuation not supported yet")
         cls.space = gettestobjspace(usemodules=('_continuation', 'struct'),
                                     CALL_METHOD=True)
         cls.space.config.translation.continuation = True

From noreply at buildbot.pypy.org  Wed Apr 18 16:01:55 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Apr 2012 16:01:55 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify
Message-ID: <20120418140155.8A81082F4F@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54496:22995c54e8ce
Date: 2012-04-18 12:14 +0200
http://bitbucket.org/pypy/pypy/changeset/22995c54e8ce/

Log:	py3k-ify

diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -1014,9 +1014,10 @@
             l = [A(), B(), C()] * 10
             __pypy__.reset_method_cache_counter()
             # 'exec' to make sure that a.f() is compiled with CALL_METHOD
-            exec """for i, a in enumerate(l):
+            d = {'l': l}
+            exec("""for i, a in enumerate(l):
                         assert a.f() == 42 + i % 3
-            """ in locals()
+            """, d)
             cache_counter = __pypy__.mapdict_cache_counter("f")
             if cache_counter == (27, 3):
                 break

From noreply at buildbot.pypy.org  Wed Apr 18 16:01:56 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Apr 2012 16:01:56 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify
Message-ID: <20120418140156.C9F59820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54497:85d52c68ef6d
Date: 2012-04-18 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/85d52c68ef6d/

Log:	py3k-ify

diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -840,10 +840,11 @@
         class C(object):
             def m(*args):
                 return args
-        C.sm = staticmethod(C.m.im_func)
-        C.cm = classmethod(C.m.im_func)
+        C.sm = staticmethod(C.m)
+        C.cm = classmethod(C.m)
 
-        exec """if 1:
+        d = {'C': C}
+        exec("""if 1:
 
             def f():
                 c = C()
@@ -862,7 +863,10 @@
                 res = c.cm(1)
                 assert res == (C, 1)
                 return 42
-        """
+        """, d)
+        f = d['f']
+        g = d['g']
+        h = d['h']
         res = self.check(f, 'm')
         assert res == (1, 0, 0)
         res = self.check(f, 'm')

From noreply at buildbot.pypy.org  Wed Apr 18 16:01:58 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Apr 2012 16:01:58 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify one test, and skip a py2-only one
Message-ID: <20120418140158.13318820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54498:83dcf8843d34
Date: 2012-04-18 12:18 +0200
http://bitbucket.org/pypy/pypy/changeset/83dcf8843d34/

Log:	py3k-ify one test, and skip a py2-only one

diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -815,6 +815,7 @@
         assert res == (0, 1, 0)
 
     def test_old_style_base(self):
+        py.test.skip('py3k no longer has old style classes')
         class B:
             pass
         class C(object):
@@ -896,7 +897,8 @@
             def m(*args):
                 return args
 
-        exec """if 1:
+        d = {'C': C}
+        exec("""if 1:
 
             def f():
                 c = C()
@@ -907,7 +909,8 @@
                 assert res == (c, 1)
                 return 42
 
-        """
+        """, d)
+        f = d['f']
         res = self.check(f, 'm')
         assert res == (1, 1, 1)
         res = self.check(f, 'm')

From noreply at buildbot.pypy.org  Wed Apr 18 16:01:59 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Apr 2012 16:01:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: (antocuni,
 cfbolz around): reverse-engineer what's going on, and document it
Message-ID: <20120418140159.6E4D1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54499:c20aa5de87cf
Date: 2012-04-18 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/c20aa5de87cf/

Log:	(antocuni, cfbolz around): reverse-engineer what's going on, and
	document it

diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):

From noreply at buildbot.pypy.org  Wed Apr 18 16:02:00 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Apr 2012 16:02:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120418140200.DA5BD820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54500:e37e49727337
Date: 2012-04-18 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/e37e49727337/

Log:	merge heads

diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,8 +453,8 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, 1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1168,8 +1168,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct

From noreply at buildbot.pypy.org  Wed Apr 18 18:07:40 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 18 Apr 2012 18:07:40 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: AddressStack.clear()
Message-ID: <20120418160740.AD895820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54501:fe66c773ed6c
Date: 2012-04-18 16:28 +0200
http://bitbucket.org/pypy/pypy/changeset/fe66c773ed6c/

Log:	AddressStack.clear()

diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -117,6 +117,15 @@
                 self.shrink()
             return result
 
+        def clear(self):
+            cur = self.chunk
+            while cur.next:
+                next = cur.next
+                unused_chunks.put(cur)
+                cur = next
+            self.chunk = cur
+            self.used_in_last_chunk = 0
+
         def delete(self):
             cur = self.chunk
             while cur:
diff --git a/pypy/rpython/memory/test/test_support.py b/pypy/rpython/memory/test/test_support.py
--- a/pypy/rpython/memory/test/test_support.py
+++ b/pypy/rpython/memory/test/test_support.py
@@ -94,6 +94,19 @@
             assert a == addrs[i]
         assert not ll.non_empty()
 
+    def test_clear(self):
+        AddressStack = get_address_stack(10)
+        ll = AddressStack()
+        for count in [0, 5, 15, 25]:
+            for i in range(count):
+                ll.append(llmemory.NULL)
+            ll.clear()
+            assert not ll.non_empty()
+            ll.append(llmemory.NULL)
+            assert ll.non_empty()
+            ll.pop()
+            assert not ll.non_empty()
+
 
 class TestAddressDeque:
     def test_big_access(self):

From noreply at buildbot.pypy.org  Wed Apr 18 18:07:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 18 Apr 2012 18:07:41 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Weakrefs again.
Message-ID: <20120418160741.F171A82F4F@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54502:7f73ab377959
Date: 2012-04-18 16:50 +0200
http://bitbucket.org/pypy/pypy/changeset/7f73ab377959/

Log:	Weakrefs again.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -42,8 +42,7 @@
 GCFLAG_WAS_COPIED = first_gcflag << 1     # keep in sync with et.c
 GCFLAG_HAS_SHADOW = first_gcflag << 2
 GCFLAG_FIXED_HASH = first_gcflag << 3
-GCFLAG_WEAKREF    = first_gcflag << 4
-GCFLAG_VISITED    = first_gcflag << 5
+GCFLAG_VISITED    = first_gcflag << 4
 
 
 def always_inline(fn):
@@ -161,15 +160,15 @@
         # Get the memory from the nursery.
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
-        result = self.get_tls().allocate_bump_pointer(totalsize)
+        tls = self.get_tls()
+        result = tls.allocate_bump_pointer(totalsize)
         #
         # Build the object.
         llarena.arena_reserve(result, totalsize)
         obj = result + size_gc_header
-        flags = 0
+        self.init_gc_object(result, typeid, flags=0)
         if contains_weakptr:   # check constant-folded
-            flags |= GCFLAG_WEAKREF
-        self.init_gc_object(result, typeid, flags=flags)
+            tls.fresh_new_weakref(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -43,7 +43,7 @@
         adr1 = adr2 - self.gc.gcheaderbuilder.size_gc_header
         llarena.arena_free(llarena.getfakearenaaddress(adr1))
 
-    def clear(self):
+    def free_and_clear(self):
         obj = self.chained_list
         self.chained_list = NULL
         while obj:
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -38,25 +38,21 @@
         self.nursery_size  = self.gc.nursery_size
         self.nursery_start = self._alloc_nursery(self.nursery_size)
         #
-        # --- the local raw-malloced objects (chained list via hdr.version)
-        #self.rawmalloced_objects = NULL
-        # --- the local "normal" old objects (chained list via hdr.version)
-        #self.old_objects = NULL
-        # --- the local objects with weakrefs (chained list via hdr.version)
-        #self.young_objects_with_weakrefs = NULL
-        #self.old_objects_with_weakrefs = NULL
-        #
         # --- a thread-local allocator for the shared area
         from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator
         self.sharedarea_tls = StmGCThreadLocalAllocator(self.gc.sharedarea)
         # --- the LOCAL objects with GCFLAG_WAS_COPIED
         self.copied_local_objects = self.AddressStack()
+        # --- the LOCAL objects which are weakrefs.  They are also listed
+        #     in the appropriate place, like sharedarea_tls, if needed.
+        self.local_weakrefs = self.AddressStack()
         #
         self._register_with_C_code()
 
     def teardown_thread(self):
         self._cleanup_state()
         self._unregister_with_C_code()
+        self.local_weakrefs.delete()
         self.copied_local_objects.delete()
         self.sharedarea_tls.delete()
         self._free_nursery(self.nursery_start)
@@ -199,9 +195,9 @@
         # Now repeatedly follow objects until 'pending' is empty.
         self.collect_flush_pending()
         #
-        # Walk the list of LOCAL raw-malloced objects, and free them if
-        # necessary.
-        #self.free_local_rawmalloced_objects()
+        # Walk the list of LOCAL weakrefs, and update it if necessary.
+        if self.local_weakrefs.non_empty():
+            self.update_local_weakrefs()
         #
         # Visit all previous OLD objects.  Free the ones that have not been
         # visited above, and reset GCFLAG_VISITED on the others.
@@ -253,6 +249,9 @@
         self.copied_local_objects.append(localobj)
         return localobj
 
+    def fresh_new_weakref(self, obj):
+        self.local_weakrefs.append(obj)
+
     # ------------------------------------------------------------
 
     def _promote_locals_to_globals(self):
@@ -275,12 +274,14 @@
 
         # free the old unused local objects still allocated in the
         # StmGCThreadLocalAllocator
-        self.sharedarea_tls.clear()
+        self.sharedarea_tls.free_and_clear()
         # free the local copies.  Note that commonly, they are leftovers
         # from the previous transaction running in this thread.  The C code
         # has just copied them over the corresponding GLOBAL objects at the
         # very end of that transaction.
         self._free_and_clear_list(self.copied_local_objects)
+        # forget the local weakrefs.
+        self.local_weakrefs.clear()
 
     def _free_and_clear_list(self, lst):
         while lst.non_empty():
@@ -302,6 +303,33 @@
     def _trace_drag_out1(self, root):
         self._trace_drag_out(root, None)
 
+    @always_inline
+    def categorize_object(self, obj, can_be_in_nursery):
+        """Return the current surviving state of the object:
+            0: not marked as surviving, so far
+            1: survives and does not move
+            2: survives, but moves to 'hdr.version'
+        """
+        hdr = self.gc.header(obj)
+        flag_combination = hdr.tid & (GCFLAG_GLOBAL |
+                                      GCFLAG_WAS_COPIED |
+                                      GCFLAG_VISITED)
+        if flag_combination == 0:
+            return 0    # not marked as surviving, so far
+
+        if flag_combination == self.detect_flag_combination:
+            # At a normal time, self.detect_flag_combination is -1
+            # and this case is never seen.  At end of transactions,
+            # detect_flag_combination is GCFLAG_WAS_COPIED|GCFLAG_VISITED.
+            # This case is to force pointers to the LOCAL copy to be
+            # replaced with pointers to the GLOBAL copy.
+            return 2
+
+        if can_be_in_nursery and self.is_in_nursery(obj):
+            return 2
+        else:
+            return 1
+
     def _trace_drag_out(self, root, ignored):
         """Trace callback: 'root' is the address of some pointer.  If that
         pointer points to a YOUNG object, allocate an OLD copy of it and
@@ -317,18 +345,11 @@
         if not self.is_in_nursery(obj):
             # we ignore both GLOBAL objects and objects which have already
             # been VISITED
-            flag_combination = hdr.tid & (GCFLAG_GLOBAL |
-                                          GCFLAG_WAS_COPIED |
-                                          GCFLAG_VISITED)
-            if flag_combination == 0:
+            cat = self.categorize_object(obj, can_be_in_nursery=False)
+            if cat == 0:
                 hdr.tid |= GCFLAG_VISITED
                 self.pending.append(obj)
-            elif flag_combination == self.detect_flag_combination:
-                # At a normal time, self.detect_flag_combination is -1
-                # and this case is never seen.  At end of transactions,
-                # detect_flag_combination is GCFLAG_WAS_COPIED|GCFLAG_VISITED.
-                # Replace references to the local copy with references
-                # to the global copy
+            elif cat == 2:
                 root.address[0] = hdr.version
             return
         #
@@ -437,6 +458,34 @@
             self.trace_and_drag_out_of_nursery(obj)
         self.pending.delete()
 
+    def update_local_weakrefs(self):
+        old = self.local_weakrefs
+        new = self.AddressStack()
+        while old.non_empty():
+            obj = old.pop()
+            hdr = self.gc.header(obj)
+            if hdr.tid & GCFLAG_VISITED == 0:
+                continue # weakref itself dies
+            #
+            if self.is_in_nursery(obj):
+                obj = hdr.version
+                hdr = self.gc.header(obj)
+            offset = self.gc.weakpointer_offset(self.gc.get_type_id(obj))
+            pointing_to = (obj + offset).address[0]
+            cat = self.categorize_object(pointing_to, can_be_in_nursery=True)
+            if cat == 0:
+                # the weakref points to a dying object; no need to rememeber it
+                (obj + offset).address[0] = llmemory.NULL
+            else:
+                # the weakref points to an object that stays alive
+                if cat == 2:
+                    pointing_hdr = self.gc.header(pointing_to)
+                    (obj + offset).address[0] = pointing_hdr.version
+                new.append(obj)    # stays alive
+        #
+        self.local_weakrefs = new
+        old.delete()
+
     def mass_free_old_local(self, previous_sharedarea_tls):
         obj = previous_sharedarea_tls.chained_list
         previous_sharedarea_tls.delete()
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -582,7 +582,7 @@
         twr1.wr = wr1
         self.gc.commit_transaction()
         wr2 = twr1.wr      # twr1 is a root, so not copied yet
-        assert wr2 and wr2 != wr1
+        assert wr2 and wr2._obj0 != wr1._obj0
         assert wr2.wadr == s2_adr   # survives
 
     def test_weakref_to_local_dying(self):
@@ -596,7 +596,7 @@
         twr1.wr = wr1
         self.gc.commit_transaction()
         wr2 = twr1.wr      # twr1 is a root, so not copied yet
-        assert wr2 and wr2 != wr1
+        assert wr2 and wr2._obj0 != wr1._obj0
         assert wr2.wadr == llmemory.NULL   # dies
 
     def test_weakref_to_local_surviving(self):
@@ -615,8 +615,8 @@
         t2.a = 4242
         self.gc.commit_transaction()
         wr2 = twr1.wr      # twr1 is a root, so not copied yet
-        assert wr2 and wr2 != wr1
-        assert wr2.wadr and wr2.wadr != t2_adr       # survives
+        assert wr2 and wr2._obj0 != wr1._obj0
+        assert wr2.wadr and wr2.wadr.ptr._obj0 != t2_adr.ptr._obj0   # survives
         s2 = llmemory.cast_adr_to_ptr(wr2.wadr, lltype.Ptr(S))
         assert s2.a == 4242
         assert s2 == tr1.s1   # tr1 is a root, so not copied yet

From noreply at buildbot.pypy.org  Wed Apr 18 18:07:43 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 18 Apr 2012 18:07:43 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Comments and asserts.
Message-ID: <20120418160743.3E72846E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54503:f0cbf218c8e3
Date: 2012-04-18 17:24 +0200
http://bitbucket.org/pypy/pypy/changeset/f0cbf218c8e3/

Log:	Comments and asserts.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -203,6 +203,13 @@
         # visited above, and reset GCFLAG_VISITED on the others.
         self.mass_free_old_local(previous_sharedarea_tls)
         #
+        # Note that the last step guarantees the invariant that between
+        # collections, all the objects linked within 'self.sharedarea_tls'
+        # don't have GCFLAG_VISITED.  As the newly allocated nursery
+        # objects don't have it either, at the start of the next
+        # collection, the only LOCAL objects that have it are the ones
+        # in 'copied_local_objects'.
+        #
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
         ll_assert(bool(self.nursery_free), "nursery_free is NULL")
@@ -464,24 +471,27 @@
         while old.non_empty():
             obj = old.pop()
             hdr = self.gc.header(obj)
+            ll_assert(hdr.tid & (GCFLAG_GLOBAL|GCFLAG_WAS_COPIED) == 0,
+                      "local weakref: bad flags")
             if hdr.tid & GCFLAG_VISITED == 0:
                 continue # weakref itself dies
             #
             if self.is_in_nursery(obj):
                 obj = hdr.version
-                hdr = self.gc.header(obj)
+                #hdr = self.gc.header(obj) --- not needed any more
             offset = self.gc.weakpointer_offset(self.gc.get_type_id(obj))
             pointing_to = (obj + offset).address[0]
+            ll_assert(bool(pointing_to), "weakref to NULL in local_weakrefs")
             cat = self.categorize_object(pointing_to, can_be_in_nursery=True)
             if cat == 0:
                 # the weakref points to a dying object; no need to rememeber it
                 (obj + offset).address[0] = llmemory.NULL
             else:
                 # the weakref points to an object that stays alive
-                if cat == 2:
+                if cat == 2:      # update the pointer if needed
                     pointing_hdr = self.gc.header(pointing_to)
                     (obj + offset).address[0] = pointing_hdr.version
-                new.append(obj)    # stays alive
+                new.append(obj)   # re-register in the new local_weakrefs list
         #
         self.local_weakrefs = new
         old.delete()

From noreply at buildbot.pypy.org  Wed Apr 18 18:07:44 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 18 Apr 2012 18:07:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: There is actually another (rare)
 bytecode combination that makes
Message-ID: <20120418160744.7FC8146E1E6@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54504:b09f20be15fd
Date: 2012-04-18 17:35 +0200
http://bitbucket.org/pypy/pypy/changeset/b09f20be15fd/

Log:	There is actually another (rare) bytecode combination that makes the
	position counter go backward.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_exception.py b/pypy/module/pypyjit/test_pypy_c/test_exception.py
--- a/pypy/module/pypyjit/test_pypy_c/test_exception.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_exception.py
@@ -91,3 +91,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_continue_in_finally(self):
+        # check that 'continue' inside a try:finally: block is correctly
+        # detected as closing a loop
+        py.test.skip("is this case important?")
+        def f(n):
+            i = 0
+            while 1:
+                try:
+                    if i < n:
+                        continue
+                finally:
+                    i += 1
+                return i
+
+        log = self.run(f, [2000])
+        assert log.result == 2001
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i40 = int_add_ovf(i31, 1)
+            guard_no_overflow(descr=...)
+            i41 = int_lt(i40, i33)
+            guard_true(i41, descr=...)
+            --TICK--
+            jump(..., descr=...)
+        """)

From noreply at buildbot.pypy.org  Wed Apr 18 18:07:45 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 18 Apr 2012 18:07:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120418160745.C32CF46E1E7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54505:63f16c08de2e
Date: 2012-04-18 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/63f16c08de2e/

Log:	merge heads

diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,8 +453,8 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, 1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1168,8 +1168,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct

From noreply at buildbot.pypy.org  Wed Apr 18 18:19:12 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 18 Apr 2012 18:19:12 +0200 (CEST)
Subject: [pypy-commit] pypy stdlib-unification/py3k: implement applevel
	urandom for posix os
Message-ID: <20120418161912.26909820B3@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: stdlib-unification/py3k
Changeset: r54506:23aebe5793a4
Date: 2012-04-18 18:18 +0200
http://bitbucket.org/pypy/pypy/changeset/23aebe5793a4/

Log:	implement applevel urandom for posix os

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -35,8 +35,10 @@
     appleveldefs = {
     'error'      : 'app_posix.error',
     'stat_result': 'app_posix.stat_result',
+    'urandom': 'app_posix.urandom',
     }
     if os.name == 'nt':
+        del appleveldefs['urandom'] # at interp on win32
         appleveldefs.update({
                 'popen2' : 'app_posix.popen2',
                 'popen3' : 'app_posix.popen3',
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -1,5 +1,4 @@
 # NOT_RPYTHON
-
 from _structseq import structseqtype, structseqfield
 
 # XXX we need a way to access the current module's globals more directly...
@@ -102,6 +101,19 @@
         from _pypy_wait import wait4
         return wait4(pid, options)
 
+    def urandom(n):
+        """urandom(n) -> str
+
+        Return a string of n random bytes suitable for cryptographic use.
+
+        """
+        try:
+            with open('/dev/urandom', 'rb', buffering=0) as fd:
+                return fd.read(n)
+        except (OSError, IOError):
+            raise NotImplementedError("/dev/urandom (or equivalent) not found")
+
+
 else:
     # Windows implementations
 

From noreply at buildbot.pypy.org  Wed Apr 18 20:04:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 18 Apr 2012 20:04:00 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Allow the JIT operations too. At this
 point, if they are left in the
Message-ID: <20120418180400.54EC3820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54507:bb6e728e7380
Date: 2012-04-18 18:24 +0200
http://bitbucket.org/pypy/pypy/changeset/bb6e728e7380/

Log:	Allow the JIT operations too. At this point, if they are left in
	the code, it means mostly that we don't have a JIT anyway.

diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -12,6 +12,9 @@
     'debug_print', 'debug_assert', 'cast_opaque_ptr', 'hint',
     'indirect_call', 'stack_current', 'gc_stack_bottom',
     'cast_current_ptr_to_int',   # this variant of 'cast_ptr_to_int' is ok
+    'jit_force_virtual', 'jit_force_virtualizable',
+    'jit_force_quasi_immutable', 'jit_marker', 'jit_is_virtual',
+    'jit_record_known_class',
     ])
 ALWAYS_ALLOW_OPERATIONS |= set(lloperation.enum_tryfold_ops())
 

From noreply at buildbot.pypy.org  Wed Apr 18 20:36:26 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Wed, 18 Apr 2012 20:36:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: add a comment explaining why this test
	fails
Message-ID: <20120418183626.547A1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54508:f1921fdff44d
Date: 2012-04-18 20:36 +0200
http://bitbucket.org/pypy/pypy/changeset/f1921fdff44d/

Log:	add a comment explaining why this test fails

diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -53,6 +53,13 @@
         assert sum(cache_counter) == 20
  
     def test_change_methods(self):
+        # this test fails because of the following line in typeobject.py:427
+        #             if cached_name is name:
+        
+        # in py3k, identifiers are stored in W_UnicodeObject and unwrapped by
+        # calling space.str_w, which .encode('ascii') the string, thus
+        # creating new strings all the time. The problem should be solved when
+        # we implement proper unicode identifiers in py3k
         import __pypy__
         class A(object):
             def f(self):

From noreply at buildbot.pypy.org  Wed Apr 18 20:36:33 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 18 Apr 2012 20:36:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: (sbehnel) Improve tests for
	PyUnicode_Tailmatch
Message-ID: <20120418183633.7F6B3820B3@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54509:b58494d41466
Date: 2012-04-18 20:35 +0200
http://bitbucket.org/pypy/pypy/changeset/b58494d41466/

Log:	(sbehnel) Improve tests for PyUnicode_Tailmatch

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,10 +453,22 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, -1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, 1) == 1
+        # prefix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        # suffix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        # type checks
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+        self.raises(space, api, TypeError,
+                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
+                    2, 10, 1)
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")

From noreply at buildbot.pypy.org  Wed Apr 18 21:37:44 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 18 Apr 2012 21:37:44 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge default into branch
Message-ID: <20120418193744.725C6820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54510:991e699f07a6
Date: 2012-04-18 22:37 +0300
http://bitbucket.org/pypy/pypy/changeset/991e699f07a6/

Log:	merge default into branch

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -158,6 +158,12 @@
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
+Note that this difference might show up indirectly in some cases.  For
+example, a generator left pending in the middle is --- again ---
+garbage-collected later in PyPy than in CPython.  You can see the
+difference if the ``yield`` keyword it is suspended at is itself
+enclosed in a ``try:`` or a ``with:`` block.
+
 Using the default GC called ``minimark``, the built-in function ``id()``
 works like it does in CPython.  With other GCs it returns numbers that
 are not real addresses (because an object can move around several times)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -169,9 +173,17 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +198,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +236,20 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +408,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,10 +453,22 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        # prefix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        # suffix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        # type checks
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+        self.raises(space, api, TypeError,
+                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
+                    2, 10, 1)
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")
@@ -479,6 +491,8 @@
                 api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
         assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
                 api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
         assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
                 api.PyUnicode_Splitlines(w_str, 0)))
         assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
@@ -630,6 +630,8 @@
     Otherwise, splits occur at the given separator.  At most maxsplit
     splits will be done.  If negative, no limit is set.  Separators
     are not included in the resulting list."""
+    if w_sep is None:
+        w_sep = space.w_None
     return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
 
 @cpython_api([PyObject, rffi.INT_real], PyObject)
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -513,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1435,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1410,6 +1410,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -770,6 +770,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -780,7 +782,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -793,6 +796,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -804,7 +809,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/module/pypyjit/test_pypy_c/test_exception.py b/pypy/module/pypyjit/test_pypy_c/test_exception.py
--- a/pypy/module/pypyjit/test_pypy_c/test_exception.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_exception.py
@@ -91,3 +91,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_continue_in_finally(self):
+        # check that 'continue' inside a try:finally: block is correctly
+        # detected as closing a loop
+        py.test.skip("is this case important?")
+        def f(n):
+            i = 0
+            while 1:
+                try:
+                    if i < n:
+                        continue
+                finally:
+                    i += 1
+                return i
+
+        log = self.run(f, [2000])
+        assert log.result == 2001
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i40 = int_add_ovf(i31, 1)
+            guard_no_overflow(descr=...)
+            i41 = int_lt(i40, i33)
+            guard_true(i41, descr=...)
+            --TICK--
+            jump(..., descr=...)
+        """)
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,31 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        res = 'bar' in x
+        assert res is True
+        #
+        class MyError(Exception):
+            pass
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise MyError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(MyError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -204,6 +204,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.call_location()
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,9 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +18,17 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +40,7 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1178,8 +1178,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:

From noreply at buildbot.pypy.org  Wed Apr 18 23:22:01 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Wed, 18 Apr 2012 23:22:01 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Only restore temporary register
 on paths where it was saved.
Message-ID: <20120418212201.7A8F1820B3@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r54511:e645b2e016d6
Date: 2012-04-18 17:21 -0400
http://bitbucket.org/pypy/pypy/changeset/e645b2e016d6/

Log:	Only restore temporary register on paths where it was saved.

diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -990,7 +990,7 @@
                 #byteofs
                 s = 3 + descr.jit_wb_card_page_shift
 
-                # use r20 as temporay register, save it in FORCE INDEX slot
+                # use r20 as temporary register, save it in FORCE INDEX slot
                 temp_reg = r.r20
                 ENCODING_AREA = len(r.MANAGED_REGS) * WORD
                 self.mc.store(temp_reg.value, r.SPP.value, ENCODING_AREA)
@@ -1012,15 +1012,15 @@
                 self.mc.stbx(r.SCRATCH.value, loc_base.value, temp_reg.value)
                 # done
 
+                # restore temporary register r20
+                self.mc.load(temp_reg.value, r.SPP.value, ENCODING_AREA)
+
                 # patch the JMP above
                 offset = self.mc.currpos()
                 pmc = OverwritingBuilder(self.mc, jmp_location, 1)
                 pmc.b(offset - jmp_location)
                 pmc.overwrite()
 
-                # restore temporary register r20
-                self.mc.load(temp_reg.value, r.SPP.value, ENCODING_AREA)
-
         # patch the JZ above
         offset = self.mc.currpos() - jz_location
         pmc = OverwritingBuilder(self.mc, jz_location, 1)

From noreply at buildbot.pypy.org  Wed Apr 18 23:50:09 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 18 Apr 2012 23:50:09 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix tests to use translated
	module
Message-ID: <20120418215009.59F30820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54512:4bb7c0cc9cbe
Date: 2012-04-19 00:49 +0300
http://bitbucket.org/pypy/pypy/changeset/4bb7c0cc9cbe/

Log:	fix tests to use translated module

diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -194,53 +194,62 @@
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
-    os.write(fd, 'Hello world')
+    f = getllimpl(os.write)
+    f(fd, 'Hello world')
     os.close(fd)
-    hello = open(fname).read()
-    assert hello == "Hello world"
+    with open(fname) as fid:
+        assert fid.read() == "Hello world"
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     os.close(fd)
-    raises(OSError, os.write, fd, 'Hello world')
+    raises(OSError, f, fd, 'Hello world')
 
 def test_os_close():
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
-    os.close(fd)
-    raises(OSError, os.close, fd)
+    f = getllimpl(os.close)
+    f(fd)
+    raises(OSError, f, fd)
 
 def test_os_lseek():
     fname = str(udir.join('os_test.txt'))
-    fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
+    fd = os.open(fname, os.O_RDWR|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
-    os.lseek(fd,0,0)
-    assert os.tell(fd) == 0
+    f = getllimpl(os.lseek)
+    f(fd,0,0)
+    assert os.read(fd, 11) == 'Hello world'
     os.close(fd)
-    raises(OSError, os.lseek, fd, 0, 0)
+    raises(OSError, f, fd, 0, 0)
 
 def test_os_fsync():
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
-    os.fsync(fd)
+    f = getllimpl(os.fsync)
+    f(fd)
+    os.close(fd)
     fid = open(fname)
     assert fid.read() == 'Hello world'
-    os.close(fd)
-    raises(OSError, os.fsync, fd)
+    fid.close()
+    raises(OSError, f, fd)
 
 def test_os_fdatasync():
+    try:
+        f = getllimpl(os.fdatasync)
+    except:
+        skip('No fdatasync in os')
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777)
     assert fd >= 0
     os.write(fd, 'Hello world')
-    os.fdatasync(fd)
+    f(fd)
     fid = open(fname)
     assert fid.read() == 'Hello world'
     os.close(fd)
-    raises(OSError, os.fdatasync, fd)
+    raises(OSError, f, fd)
 
 class ExpectTestOs:
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Apr 19 09:58:40 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 09:58:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: this test does not make sense for py3k,
 we cannot use % on bytes
Message-ID: <20120419075840.AC47F820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54513:77b126a20fff
Date: 2012-04-19 09:46 +0200
http://bitbucket.org/pypy/pypy/changeset/77b126a20fff/

Log:	this test does not make sense for py3k, we cannot use % on bytes

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -183,8 +183,6 @@
         raises(TypeError, '%c'.__mod__, ("",))
         raises(TypeError, '%c'.__mod__, (['c'],))
     
-    def test_broken_unicode(self):
-        raises(UnicodeDecodeError, 'N&#225;zov: %s'.__mod__, u'Jerry')
 
 class AppTestWidthPrec:
     def test_width(self):

From noreply at buildbot.pypy.org  Thu Apr 19 09:58:42 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 09:58:42 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: slight differences between py2 and py3k
 wrt chars and unicodes
Message-ID: <20120419075842.07ECA820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54514:f215b274d3e1
Date: 2012-04-19 09:49 +0200
http://bitbucket.org/pypy/pypy/changeset/f215b274d3e1/

Log:	slight differences between py2 and py3k wrt chars and unicodes

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -176,9 +176,10 @@
         e = 'e'
         assert '%c' % A == 'A'
         assert '%c' % e == 'e'
-        raises(OverflowError, '%c'.__mod__, (256,))
+        #raises(OverflowError, '%c'.__mod__, (256,)) # py2
+        assert '%c' % 256 == '\u0100'                # py3k
         raises(OverflowError, '%c'.__mod__, (-1,))
-        raises(OverflowError, u'%c'.__mod__, (sys.maxunicode+1,))
+        raises(OverflowError, '%c'.__mod__, (sys.maxunicode+1,))
         raises(TypeError, '%c'.__mod__, ("bla",))
         raises(TypeError, '%c'.__mod__, ("",))
         raises(TypeError, '%c'.__mod__, (['c'],))

From noreply at buildbot.pypy.org  Thu Apr 19 09:58:43 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 09:58:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: kill this test, we no longer have longs
Message-ID: <20120419075843.60419820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54515:2eb703b200a6
Date: 2012-04-19 09:50 +0200
http://bitbucket.org/pypy/pypy/changeset/2eb703b200a6/

Log:	kill this test, we no longer have longs

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -127,15 +127,6 @@
         assert '27' == '%.2o' % n
         assert '027' == '%#.2o' % n
 
-    def test_format_long(self):
-        l = 4800000000L
-        assert '%d' % l == '4800000000'
-        #
-        class SubLong(long):
-            pass
-        sl = SubLong(l)
-        assert '%d' % sl == '4800000000'
-
     def test_format_list(self):
         l = [1,2]
         assert '<[1, 2]>' == '<%s>' % l

From noreply at buildbot.pypy.org  Thu Apr 19 09:58:44 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 09:58:44 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify
Message-ID: <20120419075844.B909A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54516:2f004a1b7830
Date: 2012-04-19 09:51 +0200
http://bitbucket.org/pypy/pypy/changeset/2f004a1b7830/

Log:	py3k-ify

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -44,7 +44,7 @@
             def __getitem__(self, key):
                 return key
         assert '%(key)s'%MyMapping2() == 'key'
-        assert u'%(key)s'%MyMapping2() == u'key'
+        #assert u'%(key)s'%MyMapping2() == u'key'  # no py3k
 
 class AppTestStringObject:
 

From noreply at buildbot.pypy.org  Thu Apr 19 09:58:46 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 09:58:46 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify. The test still fail because we
 cannot have unicode exception messages yet,
 but apart from that it raises the correct exception
Message-ID: <20120419075846.0501A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54517:ae3dff717bbb
Date: 2012-04-19 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/ae3dff717bbb/

Log:	py3k-ify. The test still fail because we cannot have unicode
	exception messages yet, but apart from that it raises the correct
	exception

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,4 +295,4 @@
 
     def test_invalid_char(self):
         f = 4
-        raises(ValueError, 'u"%\u1234" % (f,)')
+        raises(ValueError, '"%\u1234" % (f,)')

From noreply at buildbot.pypy.org  Thu Apr 19 09:58:47 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 09:58:47 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: don't use the L notation (not supported in
 py3k), and use a value which is big enough not to fit a word on 64bit cpus
Message-ID: <20120419075847.44333820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54518:4a2187c22c5e
Date: 2012-04-19 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/4a2187c22c5e/

Log:	don't use the L notation (not supported in py3k), and use a value
	which is big enough not to fit a word on 64bit cpus

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -285,8 +285,8 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
         big = -123456789012345678901234567890L

From noreply at buildbot.pypy.org  Thu Apr 19 09:58:48 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 09:58:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: don't use the L notation (not supported
 in py3k),
 and use a value which is big enough not to fit a word on 64bit cpus
Message-ID: <20120419075848.8E7FA820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54519:fb765e0d956e
Date: 2012-04-19 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/fb765e0d956e/

Log:	don't use the L notation (not supported in py3k), and use a value
	which is big enough not to fit a word on 64bit cpus

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,8 +295,8 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
         big = -123456789012345678901234567890L

From noreply at buildbot.pypy.org  Thu Apr 19 10:03:12 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 10:03:12 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: don't use the L notation (not supported in
 py3k), and kill useless print
Message-ID: <20120419080312.29190820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54520:1695068fd9cf
Date: 2012-04-19 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/1695068fd9cf/

Log:	don't use the L notation (not supported in py3k), and kill useless
	print

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -289,8 +289,7 @@
         assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):

From noreply at buildbot.pypy.org  Thu Apr 19 10:03:13 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 10:03:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: don't use the L notation (not supported
 in py3k), and kill useless print
Message-ID: <20120419080313.69763820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r54521:62b3d1aaf311
Date: 2012-04-19 10:02 +0200
http://bitbucket.org/pypy/pypy/changeset/62b3d1aaf311/

Log:	don't use the L notation (not supported in py3k), and kill useless
	print

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -299,8 +299,7 @@
         assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):

From noreply at buildbot.pypy.org  Thu Apr 19 12:29:45 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 12:29:45 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Play nicely with 'MALLOC_CHECK_',
 the checking done by glibc's malloc, 
Message-ID: <20120419102945.A97D5820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54522:bb87d3a5b07d
Date: 2012-04-19 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/bb87d3a5b07d/

Log:	Play nicely with 'MALLOC_CHECK_', the checking done by glibc's
	malloc, which is unhappy about the overwrite of 4 bytes with
	0xDDDDDDDD if we allocated less than 4 bytes.

diff --git a/pypy/translator/c/src/allocator.h b/pypy/translator/c/src/allocator.h
--- a/pypy/translator/c/src/allocator.h
+++ b/pypy/translator/c/src/allocator.h
@@ -24,8 +24,8 @@
 #ifndef PYPY_NOT_MAIN_FILE
 
 #if defined(TRIVIAL_MALLOC_DEBUG)
-  void *PyObject_Malloc(size_t n) { return malloc(n); }
-  void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n); }
+  void *PyObject_Malloc(size_t n) { return malloc(n<4?4:n)); }
+  void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n<4?4:n); }
   void PyObject_Free(void *p) { if (p) { *((int*)p) = 0xDDDDDDDD; } free(p); }
 
 #elif defined(LINUXMEMCHK)

From noreply at buildbot.pypy.org  Thu Apr 19 12:29:46 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 12:29:46 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Comment.
Message-ID: <20120419102946.DC08682F4F@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54523:691cfbdef6df
Date: 2012-04-19 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/691cfbdef6df/

Log:	Comment.

diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -27,7 +27,8 @@
         self.chained_list = NULL
 
     def malloc_object(self, totalsize):
-        """Malloc.  You must also call add_regular() or add_special() later."""
+        """Malloc.  You must also call add_regular() or add_special() later.
+        Note that it is not zero-filled."""
         adr1 = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize), 0)
         llarena.arena_reserve(adr1, totalsize)
         return adr1 + self.gc.gcheaderbuilder.size_gc_header

From noreply at buildbot.pypy.org  Thu Apr 19 12:29:48 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 12:29:48 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Bah.
Message-ID: <20120419102948.20255820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54524:9332cddf50d1
Date: 2012-04-19 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/9332cddf50d1/

Log:	Bah.

diff --git a/pypy/translator/c/src/allocator.h b/pypy/translator/c/src/allocator.h
--- a/pypy/translator/c/src/allocator.h
+++ b/pypy/translator/c/src/allocator.h
@@ -24,7 +24,7 @@
 #ifndef PYPY_NOT_MAIN_FILE
 
 #if defined(TRIVIAL_MALLOC_DEBUG)
-  void *PyObject_Malloc(size_t n) { return malloc(n<4?4:n)); }
+  void *PyObject_Malloc(size_t n) { return malloc(n<4?4:n); }
   void *PyObject_Realloc(void *p, size_t n) { return realloc(p, n<4?4:n); }
   void PyObject_Free(void *p) { if (p) { *((int*)p) = 0xDDDDDDDD; } free(p); }
 

From noreply at buildbot.pypy.org  Thu Apr 19 12:29:49 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 12:29:49 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Partial fix: don't record orecs when
 running in an inevitable
Message-ID: <20120419102949.5BBBE820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54525:a046810511bf
Date: 2012-04-19 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/a046810511bf/

Log:	Partial fix: don't record orecs when running in an inevitable
	transaction (like the main thread). Should be improved also for
	non-inevitable transactions as documented in lists.c.

diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -412,14 +412,21 @@
 
 /* lazy/lazy read instrumentation */
 #define STM_DO_READ(READ_OPERATION)                                     \
+  if (is_inevitable(d)) {                                               \
+    /* if is_inevitable(), then we don't need to do the checking of  */ \
+    /* o->version done below --- but more importantly, we don't need */ \
+    /* to insert o in the OrecList                                   */ \
+    READ_OPERATION;                                                     \
+  }                                                                     \
+  else {                                                                \
  retry:                                                                 \
   /* read the orec BEFORE we read anything else */                      \
   ovt = o->version;                                                     \
   CFENCE;                                                               \
                                                                         \
-  /* this tx doesn't hold any locks, so if the lock for this addr is    \
-     held, there is contention.  A lock is never hold for too long,     \
-     so spinloop until it is released. */                               \
+  /* this tx doesn't hold any locks, so if the lock for this addr is */ \
+  /* held, there is contention.  A lock is never hold for too long,  */ \
+  /* so spinloop until it is released.                               */ \
   if (IS_LOCKED_OR_NEWER(ovt, d->start_time))                           \
     {                                                                   \
       if (IS_LOCKED(ovt)) {                                             \
@@ -440,7 +447,8 @@
   if (__builtin_expect(o->version != ovt, 0))                           \
     goto retry;       /* oups, try again */                             \
                                                                         \
-  oreclist_insert(&d->reads, (orec_t*)o);
+  oreclist_insert(&d->reads, (orec_t*)o);                               \
+  }
 
 
 #define STM_READ_WORD(SIZE, SUFFIX, TYPE)                               \
@@ -451,8 +459,9 @@
   owner_version_t ovt;                                                  \
                                                                         \
   assert(sizeof(TYPE) == SIZE);                                         \
-                                                                        \
-  /* XXX try to remove this check from the main path */                 \
+  /* XXX try to remove this check from the main path:           */      \
+  /*     d is NULL only when in non-main threads but            */      \
+  /*     outside a transaction.                                 */      \
   if (d == NULL)                                                        \
     return *(TYPE *)(((char *)addr) + offset);                          \
                                                                         \
@@ -468,7 +477,8 @@
     not_found:;                                                         \
     }                                                                   \
                                                                         \
-  STM_DO_READ(TYPE tmp = *(TYPE *)(((char *)addr) + offset));           \
+  TYPE tmp;                                                             \
+  STM_DO_READ(tmp = *(TYPE *)(((char *)addr) + offset));                \
   return tmp;                                                           \
 }
 
diff --git a/pypy/translator/stm/src_stm/lists.c b/pypy/translator/stm/src_stm/lists.c
--- a/pypy/translator/stm/src_stm/lists.c
+++ b/pypy/translator/stm/src_stm/lists.c
@@ -230,6 +230,9 @@
 
 static void oreclist_insert(struct OrecList *oreclist, orec_t *newitem)
 {
+  /* XXX it would be a good idea to try to compactify the list,
+     by removing duplicate entries.  We are only interested in
+     the set of entries present, and the order doesn't matter. */
   if (oreclist->size == oreclist->alloc)
     _oreclist_grow(oreclist);
   oreclist->items[oreclist->size++] = newitem;

From notifications-noreply at bitbucket.org  Thu Apr 19 12:39:49 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Thu, 19 Apr 2012 10:39:49 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120419103949.6376.84510@bitbucket12.managed.contegix.com>

You have received a notification from John Hsing.

Hi, I forked pypy. My fork is at https://bitbucket.org/droid4u/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Thu Apr 19 13:55:49 2012
From: noreply at buildbot.pypy.org (krono)
Date: Thu, 19 Apr 2012 13:55:49 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk default: Re-import a FixedStack
	implementation
Message-ID: <20120419115549.50210820B3@wyvern.cs.uni-duesseldorf.de>

Author: Tobias Pape <tobias at netshed.de>
Branch: 
Changeset: r19:4da7f6a039b9
Date: 2012-04-19 13:37 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/4da7f6a039b9/

Log:	Re-import a FixedStack implementation

	- fix deprecated overflowcheck w/ shift
	- fix r_uints missing

diff --git a/spyvm/fixedstack.py b/spyvm/fixedstack.py
new file mode 100644
--- /dev/null
+++ b/spyvm/fixedstack.py
@@ -0,0 +1,65 @@
+"""
+A Fixed stack for SPy.
+"""
+
+import types
+
+from pypy.rlib.rarithmetic import r_uint
+
+class FixedStack(object):
+    _annspecialcase_ = "specialize:ctr_location" # polymorphic
+    
+    def __init__(self):
+        pass
+    
+    def setup(self, stacksize):
+        self.ptr = r_uint(0) # we point after the last element
+        self.items = [None] * stacksize
+    
+    def clone(self):
+        # this is only needed if we support flow space
+        s = self.__class__()
+        s.setup(len(self.items))
+        for item in self.items[:self.ptr]:
+            try:
+                item = item.clone()
+            except AttributeError:
+                pass
+            s.push(item)
+        return s
+    
+    def push(self, item):
+        ptr = self.ptr
+        self.items[ptr] = item
+        self.ptr = ptr + 1
+    
+    def pop(self):
+        ptr = self.ptr - 1
+        ret = self.items[ptr]   # you get OverflowError if the stack is empty
+        self.items[ptr] = None
+        self.ptr = ptr
+        return ret
+    
+    def drop(self, n):
+        while n > 0:
+            n -= 1
+            self.ptr -= 1
+            self.items[self.ptr] = None
+    
+    def top(self, position=0):
+        # for a fixed stack, we assume correct indices
+        rpos = r_uint(position)
+        return self.items[self.ptr + ~rpos]
+    
+    def set_top(self, value, position=0):
+        # for a fixed stack, we assume correct indices
+        rpos = r_uint(position)
+        self.items[self.ptr + ~rpos] = value
+    
+    def depth(self):
+        return self.ptr
+    
+    def empty(self):
+        return not self.ptr
+    
+
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -424,7 +424,7 @@
         return self.words[n]
         
     def setword(self, n, word):
-        self.words[n] = word        
+        self.words[n] = r_uint(word)
 
     def size(self):
         return len(self.words)   
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -217,7 +217,7 @@
             val = w_value.value
             if val < 0:
                 raise UnwrappingError("got negative integer")
-            return w_value.value
+            return r_uint(w_value.value)
         if isinstance(w_value, model.W_BytesObject):
             # TODO: Completely untested! This failed translation bigtime...
             # XXX Probably we want to allow all subclasses
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -202,7 +202,7 @@
     # left shift, must fail if we lose bits beyond 32
     if argument > 0:
         try:
-            shifted = rarithmetic.ovfcheck_lshift(receiver, argument)
+            shifted = rarithmetic.ovfcheck(receiver << argument)
         except OverflowError:
             raise PrimitiveFailedError()
         return interp.space.wrap_int(shifted)
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -315,7 +315,7 @@
     __metaclass__ = extendabletype
 
     def __init__(self, space, w_self):
-        from pypy.interpreter.miscutils import FixedStack
+        from spyvm.fixedstack import FixedStack
 
         self._w_sender = space.w_nil
         self._stack = FixedStack()

From noreply at buildbot.pypy.org  Thu Apr 19 14:49:26 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 14:49:26 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix for tests.
Message-ID: <20120419124926.88078820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54526:22f9ada8f2f9
Date: 2012-04-19 14:37 +0200
http://bitbucket.org/pypy/pypy/changeset/22f9ada8f2f9/

Log:	Fix for tests.

diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -757,6 +757,8 @@
         checkptr(ptr1)
         return lltype.cast_ptr_to_int(ptr1)
 
+    op_cast_current_ptr_to_int = op_cast_ptr_to_int
+
     def op_cast_opaque_ptr(self, RESTYPE, obj):
         checkptr(obj)
         return lltype.cast_opaque_ptr(RESTYPE, obj)

From noreply at buildbot.pypy.org  Thu Apr 19 14:49:27 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 14:49:27 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix test.
Message-ID: <20120419124927.CD5B1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54527:7b3615f8e6d4
Date: 2012-04-19 14:49 +0200
http://bitbucket.org/pypy/pypy/changeset/7b3615f8e6d4/

Log:	Fix test.

diff --git a/pypy/rpython/test/test_rvirtualizable2.py b/pypy/rpython/test/test_rvirtualizable2.py
--- a/pypy/rpython/test/test_rvirtualizable2.py
+++ b/pypy/rpython/test/test_rvirtualizable2.py
@@ -117,8 +117,8 @@
         TYPE = self.gettype(v_inst)
         accessor = TYPE._hints['virtualizable2_accessor']
         assert accessor.TYPE == TYPE
-        assert accessor.fields == {self.prefix + 'v1': IR_IMMUTABLE,
-                                   self.prefix + 'v2': IR_IMMUTABLE_ARRAY}
+        assert accessor._fields == {self.prefix + 'v1': IR_IMMUTABLE,
+                                    self.prefix + 'v2': IR_IMMUTABLE_ARRAY}
         #
         def fn2(n):
             Base().base1 = 42

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:48 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:48 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: these two tests don't make sense on py3k
 where we have only unicode strings
Message-ID: <20120419133348.3CE94820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54528:580294c5d310
Date: 2012-04-19 10:06 +0200
http://bitbucket.org/pypy/pypy/changeset/580294c5d310/

Log:	these two tests don't make sense on py3k where we have only unicode
	strings

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -256,21 +256,7 @@
         assert "%G" % (nan,) == 'NAN'
 
 class AppTestUnicodeObject:
-    def test_unicode_convert(self):
-        u = u"x"
-        assert isinstance("%s" % u, unicode)
-
-    def test_unicode_nonascii(self):
-        """
-        Interpolating a unicode string with non-ascii characters in it into
-        a string format should decode the format string as ascii and return
-        unicode.
-        """
-        u = u'\x80'
-        result = "%s" % u
-        assert isinstance(result, unicode)
-        assert result == u
-
+    
     def test_unicode_d(self):
         t = 3
         assert u"%.1d" % t == '3'

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:54 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: partial backout of 83fc38544bca: we don't
 want rlib/ to diverge from default
Message-ID: <20120419133354.93F6D82F53@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54533:0000a290ac54
Date: 2012-04-19 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/0000a290ac54/

Log:	partial backout of 83fc38544bca: we don't want rlib/ to diverge from
	default

diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -285,10 +285,10 @@
     def tofloat(self):
         return _AsDouble(self)
 
-    def format(self, digits, prefix=''):
+    def format(self, digits, prefix='', suffix=''):
         # 'digits' is a string whose length is the base to use,
         # and where each character is the corresponding digit.
-        return _format(self, digits, prefix)
+        return _format(self, digits, prefix, suffix)
 
     def repr(self):
         return _format(self, BASE10, '', 'L')
@@ -601,10 +601,10 @@
         if self.sign == 0:
             return '0L'
         else:
-            return _format(self, BASE8, '0')
+            return _format(self, BASE8, '0', 'L')
 
     def hex(self):
-        return _format(self, BASE16, '0x')
+        return _format(self, BASE16, '0x', 'L')
 
     def log(self, base):
         # base is supposed to be positive or 0.0, which means we use e
@@ -1584,7 +1584,7 @@
 BASE10 = '0123456789'
 BASE16 = '0123456789abcdef'
 
-def _format(a, digits, prefix=''):
+def _format(a, digits, prefix='', suffix=''):
     """
     Convert a bigint object to a string, using a given conversion base.
     Return a string object.
@@ -1600,9 +1600,14 @@
     while i > 1:
         bits += 1
         i >>= 1
-    i = 5 + len(prefix) + (size_a*SHIFT + bits-1) // bits
+    i = 5 + len(prefix) + len(suffix) + (size_a*SHIFT + bits-1) // bits
     s = [chr(0)] * i
     p = i
+    j = len(suffix)
+    while j > 0:
+        p -= 1
+        j -= 1
+        s[p] = suffix[j]
 
     if a.sign == 0:
         p -= 1

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:49 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:49 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify
Message-ID: <20120419133349.79DCE82F4F@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54529:35d076905f8f
Date: 2012-04-19 10:07 +0200
http://bitbucket.org/pypy/pypy/changeset/35d076905f8f/

Log:	py3k-ify

diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -259,7 +259,7 @@
     
     def test_unicode_d(self):
         t = 3
-        assert u"%.1d" % t == '3'
+        assert "%.1d" % t == '3'
 
     def test_unicode_overflow(self):
         skip("nicely passes on top of CPython but requires > 2GB of RAM")
@@ -267,8 +267,8 @@
         raises((OverflowError, MemoryError), 'u"%.*d" % (sys.maxint, 1)')
 
     def test_unicode_format_a(self):
-        ten = 10L
-        assert u'%x' % ten == 'a'
+        ten = 10
+        assert '%x' % ten == 'a'
 
     def test_long_no_overflow(self):
         big = 0x1234567890987654321

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:55 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:55 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: re-fix what was broken by 0000a290ac54 and
 0954589ca523: on py3k, the repr() of longs is the same as their str(),
 i.e. the number without the L suffix
Message-ID: <20120419133355.D3B55820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54534:5995e1a6e2a1
Date: 2012-04-19 14:14 +0200
http://bitbucket.org/pypy/pypy/changeset/5995e1a6e2a1/

Log:	re-fix what was broken by 0000a290ac54 and 0954589ca523: on py3k,
	the repr() of longs is the same as their str(), i.e. the number
	without the L suffix

diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -117,7 +117,7 @@
                              space.wrap("long int too large to convert to float"))
 
 def repr__Long(space, w_long):
-    return space.wrap(w_long.num.repr())
+    return space.wrap(w_long.num.str())
 
 def str__Long(space, w_long):
     return space.wrap(w_long.num.str())

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:50 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:50 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix this test which was broken by
	aa1d34b5a927
Message-ID: <20120419133350.B352F82F50@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54530:c9384faead26
Date: 2012-04-19 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/c9384faead26/

Log:	fix this test which was broken by aa1d34b5a927

diff --git a/pypy/rlib/test/test_objectmodel.py b/pypy/rlib/test/test_objectmodel.py
--- a/pypy/rlib/test/test_objectmodel.py
+++ b/pypy/rlib/test/test_objectmodel.py
@@ -160,8 +160,8 @@
 
 def test_compute_hash_float():
     from pypy.rlib.rfloat import INFINITY, NAN
-    assert compute_hash(INFINITY) == 314159
-    assert compute_hash(-INFINITY) == -271828
+    assert compute_hash(INFINITY) == HASH_INF
+    assert compute_hash(-INFINITY) == -HASH_INF
     assert compute_hash(NAN) == 0
 
 def test_compute_identity_hash():

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:57 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:57 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify
Message-ID: <20120419133357.25185820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54535:1795cb766136
Date: 2012-04-19 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/1795cb766136/

Log:	py3k-ify

diff --git a/pypy/objspace/std/test/test_index.py b/pypy/objspace/std/test/test_index.py
--- a/pypy/objspace/std/test/test_index.py
+++ b/pypy/objspace/std/test/test_index.py
@@ -76,7 +76,7 @@
         assert self.n.__index__() == 5
 
     def test_subclasses(self):
-        r = range(10)
+        r = list(range(10))
         assert r[self.TrapInt(5):self.TrapInt(10)] == r[5:10]
         assert slice(self.TrapInt()).indices(0) == (0,0,1)
 

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:51 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:51 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix another test broken by aa1d34b5a927
Message-ID: <20120419133351.ECDD382F51@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54531:a1935ceb3977
Date: 2012-04-19 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/a1935ceb3977/

Log:	fix another test broken by aa1d34b5a927

diff --git a/pypy/rlib/test/test_objectmodel.py b/pypy/rlib/test/test_objectmodel.py
--- a/pypy/rlib/test/test_objectmodel.py
+++ b/pypy/rlib/test/test_objectmodel.py
@@ -332,8 +332,8 @@
             q = Foo()
             assert compute_hash(q) == compute_identity_hash(q)
             from pypy.rlib.rfloat import INFINITY, NAN
-            assert compute_hash(INFINITY) == 314159
-            assert compute_hash(-INFINITY) == -271828
+            assert compute_hash(INFINITY) == HASH_INF
+            assert compute_hash(-INFINITY) == -HASH_INF
             assert compute_hash(NAN) == 0
             return i*2
         res = self.interpret(f, [42])

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:58 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:58 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify: maxint has gone,
 and __getitem__ gets a slice() object now. The test was previously
 written with __getslice__ but got changed in a parent revision
Message-ID: <20120419133358.5ECE6820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54536:1373e06b8e6d
Date: 2012-04-19 15:31 +0200
http://bitbucket.org/pypy/pypy/changeset/1373e06b8e6d/

Log:	py3k-ify: maxint has gone, and __getitem__ gets a slice() object
	now. The test was previously written with __getslice__ but got
	changed in a parent revision

diff --git a/pypy/objspace/std/test/test_index.py b/pypy/objspace/std/test/test_index.py
--- a/pypy/objspace/std/test/test_index.py
+++ b/pypy/objspace/std/test/test_index.py
@@ -268,7 +268,7 @@
         assert self.neg.__index__() == self.neg
 
     def test_getitem(self):
-        from sys import maxint
+        from sys import maxsize as maxint
         class GetItem(object):
             def __len__(self):
                 return maxint
@@ -277,7 +277,7 @@
         x = GetItem()
         assert x[self.pos] == self.pos
         assert x[self.neg] == self.neg
-        assert x[self.neg:self.pos] == (-1, maxint)
+        assert x[self.neg:self.pos] == slice(self.neg, self.pos)
         assert x[self.neg:self.pos:1].indices(maxint) == (0, maxint, 1)
 
     def test_getslice_nolength(self):

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:53 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:53 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: partial backout of f4af6649c12f: we don't
 want rlib/ to diverge from default
Message-ID: <20120419133353.3772682F52@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54532:0954589ca523
Date: 2012-04-19 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/0954589ca523/

Log:	partial backout of f4af6649c12f: we don't want rlib/ to diverge from
	default

diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -291,7 +291,7 @@
         return _format(self, digits, prefix)
 
     def repr(self):
-        return _format(self, BASE10)
+        return _format(self, BASE10, '', 'L')
 
     def str(self):
         return _format(self, BASE10)

From noreply at buildbot.pypy.org  Thu Apr 19 15:33:59 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:33:59 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: classic classes are gone in py3k
Message-ID: <20120419133359.AAC6B820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54537:c577976dd5d2
Date: 2012-04-19 15:32 +0200
http://bitbucket.org/pypy/pypy/changeset/c577976dd5d2/

Log:	classic classes are gone in py3k

diff --git a/pypy/objspace/std/test/test_index.py b/pypy/objspace/std/test/test_index.py
--- a/pypy/objspace/std/test/test_index.py
+++ b/pypy/objspace/std/test/test_index.py
@@ -287,20 +287,6 @@
 
         assert X()[-2:1] == (-2, 1)
 
-    def test_getitem_classic(self):
-        from sys import maxint
-        class Empty: pass
-        class GetItem(Empty):
-            def __len__(self):
-                return maxint
-            def __getitem__(self, key):
-                return key
-        x = GetItem()
-        assert x[self.pos] == self.pos
-        assert x[self.neg] == self.neg
-        assert x[self.neg:self.pos] == (-1, maxint)
-        assert x[self.neg:self.pos:1].indices(maxint) == (0, maxint, 1)
-
     def test_sequence_repeat(self):
         raises(OverflowError, lambda: "a" * self.pos)
         raises(OverflowError, lambda: "a" * self.neg)

From noreply at buildbot.pypy.org  Thu Apr 19 15:34:00 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 15:34:00 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify, __getslice__ vs __getitem__
Message-ID: <20120419133400.E4432820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54538:ab9676ffcac3
Date: 2012-04-19 15:33 +0200
http://bitbucket.org/pypy/pypy/changeset/ab9676ffcac3/

Log:	py3k-ify, __getslice__ vs __getitem__

diff --git a/pypy/objspace/std/test/test_index.py b/pypy/objspace/std/test/test_index.py
--- a/pypy/objspace/std/test/test_index.py
+++ b/pypy/objspace/std/test/test_index.py
@@ -285,7 +285,7 @@
             def __getitem__(self, slice):
                 return slice
 
-        assert X()[-2:1] == (-2, 1)
+        assert X()[-2:1] == slice(-2, 1)
 
     def test_sequence_repeat(self):
         raises(OverflowError, lambda: "a" * self.pos)

From noreply at buildbot.pypy.org  Thu Apr 19 16:52:12 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 16:52:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: Expand a bit the comments.
Message-ID: <20120419145212.689F2820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54539:b89254c1241f
Date: 2012-04-19 16:49 +0200
http://bitbucket.org/pypy/pypy/changeset/b89254c1241f/

Log:	Expand a bit the comments.

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1657,14 +1657,17 @@
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
             #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower 16 bits read there
-            #   - multiply by 4 and use it as an offset in type_info_group
-            #   - add 16 bytes, to go past the TYPE_INFO structure
+            #   - keep the lower half-word read there
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
             loc = locs[1]
             assert isinstance(loc, ImmedLoc)
             classptr = loc.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the 16 bits in the object header
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 or 4 on a big-endian machine.
             from pypy.rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)

From noreply at buildbot.pypy.org  Thu Apr 19 17:30:24 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Thu, 19 Apr 2012 17:30:24 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: (edelsohn,
 arigato): Complete guard_class support for GC.
Message-ID: <20120419153024.7B030820B3@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r54540:5b05d7de13d5
Date: 2012-04-19 11:30 -0400
http://bitbucket.org/pypy/pypy/changeset/5b05d7de13d5/

Log:	(edelsohn, arigato): Complete guard_class support for GC.

diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -259,14 +259,22 @@
     def _cmp_guard_class(self, op, locs, regalloc):
         offset = locs[2]
         if offset is not None:
-            #self.mc.LDR_ri(r.ip.value, locs[0].value, offset.value, cond=fcond)
-            #self.mc.CMP_rr(r.ip.value, locs[1].value, cond=fcond)
             with scratch_reg(self.mc):
                 self.mc.load(r.SCRATCH.value, locs[0].value, offset.value)
                 self.mc.cmp_op(0, r.SCRATCH.value, locs[1].value)
         else:
-            raise NotImplementedError
-            # XXX port from x86 backend once gc support is in place
+            typeid = locs[1]
+            # here, we have to go back from 'classptr' to the value expected
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 (32 bit) or 4 (64 bit) on a
+            # big-endian machine.
+            with scratch_reg(self.mc):
+                if IS_PPC_32:
+                    self.mc.lhz(r.SCRATCH.value, locs[0].value, 2)
+                else:
+                    self.mc.lwz(r.SCRATCH.value, locs[0].value, 4)
+                self.mc.cmp_op(0, r.SCRATCH.value, typeid.value, imm=typeid.is_imm())
 
     def emit_guard_not_invalidated(self, op, locs, regalloc):
         return self._emit_guard(op, locs, c.EQ, is_guard_not_invalidated=True)
diff --git a/pypy/jit/backend/ppc/regalloc.py b/pypy/jit/backend/ppc/regalloc.py
--- a/pypy/jit/backend/ppc/regalloc.py
+++ b/pypy/jit/backend/ppc/regalloc.py
@@ -1,6 +1,6 @@
 from pypy.jit.backend.llsupport.regalloc import (RegisterManager, FrameManager,
                                                  TempBox, compute_vars_longevity)
-from pypy.jit.backend.ppc.arch import (WORD, MY_COPY_OF_REGS)
+from pypy.jit.backend.ppc.arch import (WORD, MY_COPY_OF_REGS, IS_PPC_32)
 from pypy.jit.backend.ppc.jump import remap_frame_layout
 from pypy.jit.backend.ppc.locations import imm
 from pypy.jit.backend.ppc.helper.regalloc import (_check_imm_arg,
@@ -15,7 +15,8 @@
 from pypy.jit.metainterp.history import JitCellToken, TargetToken
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.ppc import locations
-from pypy.rpython.lltypesystem import rffi, lltype, rstr
+from pypy.rpython.lltypesystem import rffi, lltype, rstr, llmemory
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.descr import ArrayDescr
 import pypy.jit.backend.ppc.register as r
@@ -452,17 +453,45 @@
         boxes = op.getarglist()
 
         x = self._ensure_value_is_boxed(boxes[0], boxes)
-        y = self.get_scratch_reg(INT, forbidden_vars=boxes)
         y_val = rffi.cast(lltype.Signed, op.getarg(1).getint())
-        self.assembler.load(y, imm(y_val))
+
+        arglocs = [x, None, None]
 
         offset = self.cpu.vtable_offset
-        assert offset is not None
-        assert _check_imm_arg(offset)
-        offset_loc = imm(offset)
-        arglocs = self._prepare_guard(op, [x, y, offset_loc])
+        if offset is not None:
+            y = self.get_scratch_reg(INT, forbidden_vars=boxes)
+            self.assembler.load(y, imm(y_val))
 
-        return arglocs
+            assert _check_imm_arg(offset)
+            offset_loc = imm(offset)
+
+            arglocs[1] = y
+            arglocs[2] = offset_loc
+
+        else:
+            # XXX hard-coded assumption: to go from an object to its class
+            # we use the following algorithm:
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0
+            #   - keep the lower half-word read there
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
+            classptr = y_val
+            from pypy.rpython.memory.gctypelayout import GCData
+            sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
+            type_info_group = llop.gc_get_type_info_group(llmemory.Address)
+            type_info_group = rffi.cast(lltype.Signed, type_info_group)
+            expected_typeid = classptr - sizeof_ti - type_info_group
+            if IS_PPC_32:
+                expected_typeid >>= 2
+            if _check_imm_arg(expected_typeid):
+                arglocs[1] = imm(expected_typeid)
+            else:
+                y = self.get_scratch_reg(INT, forbidden_vars=boxes)
+                self.assembler.load(y, imm(expected_typeid))
+                arglocs[1] = y
+
+        return self._prepare_guard(op, arglocs)
 
     prepare_guard_nonnull_class = prepare_guard_class
 

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:35 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:35 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k_skip two tests which fails right now
 because of missing strategies
Message-ID: <20120419161935.863F1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54541:572c64c9e79a
Date: 2012-04-19 15:59 +0200
http://bitbucket.org/pypy/pypy/changeset/572c64c9e79a/

Log:	py3k_skip two tests which fails right now because of missing
	strategies

diff --git a/pypy/objspace/std/test/test_celldict.py b/pypy/objspace/std/test/test_celldict.py
--- a/pypy/objspace/std/test/test_celldict.py
+++ b/pypy/objspace/std/test/test_celldict.py
@@ -65,6 +65,7 @@
         assert "ModuleDictStrategy" in __pypy__.internal_repr(obj)
 
     def test_check_module_uses_module_dict(self):
+        py3k_skip("ModuleDictStrategy is immediately turned into ObjectDictStrategy because we use unicode keys now")
         m = type(__builtins__)("abc")
         self.impl_used(m.__dict__)
 
@@ -142,6 +143,7 @@
         assert x == ("a", 3)
 
     def test_degenerate(self):
+        py3k_skip("ModuleDictStrategy is immediately turned into ObjectDictStrategy because we use unicode keys now")
         import __pypy__
 
         d = self.d
@@ -149,4 +151,4 @@
         d["a"] = 3
         del d["a"]
         d[object()] = 5
-        assert d.values() == [5]
+        assert list(d.values()) == [5]

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:42 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:42 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k_skip these two tests
Message-ID: <20120419161942.241C682F53@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54546:abe0df9a5ff4
Date: 2012-04-19 16:34 +0200
http://bitbucket.org/pypy/pypy/changeset/abe0df9a5ff4/

Log:	py3k_skip these two tests

diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -1,3 +1,4 @@
+import py
 from pypy.objspace.std.listobject import W_ListObject, EmptyListStrategy, ObjectListStrategy, IntegerListStrategy, FloatListStrategy, UnicodeListStrategy, RangeListStrategy, make_range_list
 from pypy.objspace.std import listobject
 from pypy.objspace.std.test.test_listobject import TestW_ListObject
@@ -139,6 +140,7 @@
         assert isinstance(l.strategy, EmptyListStrategy)
 
     def test_setslice(self):
+        py.test.py3k_skip("missing the correct list strategy")
         l = W_ListObject(self.space, [])
         assert isinstance(l.strategy, EmptyListStrategy)
         l.setslice(0, 1, 2, W_ListObject(self.space, [self.space.wrap(1), self.space.wrap(2), self.space.wrap(3)]))
@@ -466,6 +468,8 @@
         assert space.unwrap(w_res) == 3
 
     def test_create_list_from_set(self):
+        # this test fails because of the "w_set.iter = None" line below
+        py.test.py3k_skip("missing the correct list strategy")
         from pypy.objspace.std.setobject import W_SetObject
         from pypy.objspace.std.setobject import _initialize_set
 

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:36 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:36 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: allow lookup of str subclasses in
 dictproxies. Comment out py2-only checks
Message-ID: <20120419161936.D188F82F4F@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54542:f5af4b8cad47
Date: 2012-04-19 16:11 +0200
http://bitbucket.org/pypy/pypy/changeset/f5af4b8cad47/

Log:	allow lookup of str subclasses in dictproxies. Comment out py2-only
	checks

diff --git a/pypy/objspace/std/dictproxyobject.py b/pypy/objspace/std/dictproxyobject.py
--- a/pypy/objspace/std/dictproxyobject.py
+++ b/pypy/objspace/std/dictproxyobject.py
@@ -20,7 +20,7 @@
     def getitem(self, w_dict, w_key):
         space = self.space
         w_lookup_type = space.type(w_key)
-        if space.is_w(w_lookup_type, space.w_unicode):
+        if space.is_true(space.issubtype(w_lookup_type, space.w_unicode)):
             return self.getitem_str(w_dict, space.str_w(w_key))
         else:
             return None
diff --git a/pypy/objspace/std/test/test_dictproxy.py b/pypy/objspace/std/test/test_dictproxy.py
--- a/pypy/objspace/std/test/test_dictproxy.py
+++ b/pypy/objspace/std/test/test_dictproxy.py
@@ -31,9 +31,10 @@
         assert 'a' in NotEmpty.__dict__
         class substr(str): pass
         assert substr('a') in NotEmpty.__dict__
-        assert u'a' in NotEmpty.__dict__
-        assert NotEmpty.__dict__[u'a'] == 1
-        assert u'\xe9' not in NotEmpty.__dict__
+        # the following are only for py2
+        ## assert u'a' in NotEmpty.__dict__
+        ## assert NotEmpty.__dict__[u'a'] == 1
+        ## assert u'\xe9' not in NotEmpty.__dict__
 
     def test_dictproxyeq(self):
         class a(object):

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:43 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: comment out an assert which is no longer
	valid in py3k
Message-ID: <20120419161943.5E3DB820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54547:552ef9b9b6d0
Date: 2012-04-19 16:34 +0200
http://bitbucket.org/pypy/pypy/changeset/552ef9b9b6d0/

Log:	comment out an assert which is no longer valid in py3k

diff --git a/pypy/objspace/std/test/test_versionedtype.py b/pypy/objspace/std/test/test_versionedtype.py
--- a/pypy/objspace/std/test/test_versionedtype.py
+++ b/pypy/objspace/std/test/test_versionedtype.py
@@ -29,7 +29,9 @@
         btag = w_B.version_tag()
         assert atag is not None
         assert btag is not None
-        assert w_C.version_tag() is None
+        # the following assert is true only in py2 because C is an old-style
+        #class
+        # assert w_C.version_tag() is None
         assert atag is not btag
         w_types = space.appexec([w_A, w_B], """(A, B):
             B.g = lambda self: None

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:38 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:38 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: we no longer have oldstyle classes,
	kill the test
Message-ID: <20120419161938.6387182F50@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54543:c20cf887558b
Date: 2012-04-19 16:12 +0200
http://bitbucket.org/pypy/pypy/changeset/c20cf887558b/

Log:	we no longer have oldstyle classes, kill the test

diff --git a/pypy/objspace/std/test/test_identitydict.py b/pypy/objspace/std/test/test_identitydict.py
--- a/pypy/objspace/std/test/test_identitydict.py
+++ b/pypy/objspace/std/test/test_identitydict.py
@@ -130,9 +130,3 @@
         assert d1 == d2
         assert self.uses_identity_strategy(d1)
         assert not self.uses_identity_strategy(d2)
-
-    def test_old_style_classes(self):
-        class X:
-            pass
-        d = {X(): 1}
-        assert not self.uses_identity_strategy(d)

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:44 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:44 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: oldstyle classes are gone, kill this test
Message-ID: <20120419161944.98165820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54548:823e45ade349
Date: 2012-04-19 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/823e45ade349/

Log:	oldstyle classes are gone, kill this test

diff --git a/pypy/objspace/std/test/test_versionedtype.py b/pypy/objspace/std/test/test_versionedtype.py
--- a/pypy/objspace/std/test/test_versionedtype.py
+++ b/pypy/objspace/std/test/test_versionedtype.py
@@ -147,39 +147,6 @@
         assert w_MODULE.version_tag() is not None
         assert w_OBJECT.version_tag() is not None
 
-    def test_version_tag_mixes_oldnew(self):
-        space = self.space
-        w_types = space.appexec([], """():
-        class A:
-            pass
-
-        class B(A, object):
-            pass
-
-        return A, B
-        """)
-        w_A, w_B = space.unpackiterable(w_types)
-        oldtag = w_B.version_tag()
-        space.setattr(w_A, space.wrap("x"), space.w_None)
-        newtag = w_B.version_tag()
-        if oldtag is not None:
-            assert newtag != oldtag
-
-        w_types = space.appexec([], """():
-        class A:
-            pass
-        class B(object):
-            pass
-
-        return A, B
-        """)
-        w_A, w_B = space.unpackiterable(w_types)
-        oldtag = w_B.version_tag()
-        assert oldtag is not None
-        space.setattr(w_B, space.wrap("__bases__"), space.newtuple([w_A, space.w_object]))
-        newtag = w_B.version_tag()
-        assert newtag is None
-
     def test_version_tag_of_modules(self):
         space = self.space
         w_mod = space.appexec([], """():

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:39 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k-ify
Message-ID: <20120419161939.A4AF182F51@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54544:cad97b461205
Date: 2012-04-19 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/cad97b461205/

Log:	py3k-ify

diff --git a/pypy/objspace/std/test/test_instmethobject.py b/pypy/objspace/std/test/test_instmethobject.py
--- a/pypy/objspace/std/test/test_instmethobject.py
+++ b/pypy/objspace/std/test/test_instmethobject.py
@@ -20,7 +20,7 @@
             def m(self): pass
         try:
             C.m(1)
-        except TypeError, e:
+        except TypeError as e:
             assert str(e) == 'unbound method m() must be called with C instance as first argument (got int instance instead)'
 
     def test_getBound(self):

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:45 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:45 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k_skip for now the tests which fails
 because they need to handle unicode and longs now
Message-ID: <20120419161945.D95CA820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54549:15d867fe89e9
Date: 2012-04-19 18:12 +0200
http://bitbucket.org/pypy/pypy/changeset/15d867fe89e9/

Log:	py3k_skip for now the tests which fails because they need to handle
	unicode and longs now

diff --git a/pypy/module/_ffi/test/test__ffi.py b/pypy/module/_ffi/test/test__ffi.py
--- a/pypy/module/_ffi/test/test__ffi.py
+++ b/pypy/module/_ffi/test/test__ffi.py
@@ -121,6 +121,7 @@
                 return x+y;
             }
         """
+        py3k_skip('missing support for longs')
         import sys
         from _ffi import CDLL, types
         libfoo = CDLL(self.libfoo_name)
@@ -202,6 +203,7 @@
                 return len;
             }
         """
+        py3k_skip('missing support for unicode')
         from _ffi import CDLL, types
         import _rawffi
         libfoo = CDLL(self.libfoo_name)
@@ -252,6 +254,7 @@
                 return s;
             }
         """
+        py3k_skip('missing support for unicode')
         from _ffi import CDLL, types
         import _rawffi
         libfoo = CDLL(self.libfoo_name)
@@ -323,6 +326,7 @@
                 return x+y;
             }
         """
+        py3k_skip('missing support for longs')
         import sys
         from _ffi import CDLL, types
         libfoo = CDLL(self.libfoo_name)
@@ -449,6 +453,7 @@
                 return x+y;
             }
         """
+        py3k_skip('missing support for ulonglong')
         from _ffi import CDLL, types
         maxint64 = 9223372036854775807 # maxint64+1 does not fit into a
                                        # longlong, but it does into a

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:40 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k_skip this test while waiting for a
	UnicodeDictStrategy
Message-ID: <20120419161940.DE08B82F52@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54545:d63caac36a57
Date: 2012-04-19 16:15 +0200
http://bitbucket.org/pypy/pypy/changeset/d63caac36a57/

Log:	py3k_skip this test while waiting for a UnicodeDictStrategy

diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -113,6 +113,7 @@
         return r[r.find("(") + 1: r.find(")")]
 
     def test_create(self):
+        py3k_skip("need UnicodeDictStrategy to work in py3k")
         def f(**args):
             return args
         d = f(a=1)

From noreply at buildbot.pypy.org  Thu Apr 19 18:19:47 2012
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 19 Apr 2012 18:19:47 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: py3k_skip for now the tests which fails
 because they need to handle unicode and longs now
Message-ID: <20120419161947.335A0820B3@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: py3k
Changeset: r54550:55deb3442c05
Date: 2012-04-19 18:19 +0200
http://bitbucket.org/pypy/pypy/changeset/55deb3442c05/

Log:	py3k_skip for now the tests which fails because they need to handle
	unicode and longs now

diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -262,6 +262,7 @@
         assert lib.ptr(1, [], 'i')()[0] == 42
 
     def test_getchar(self):
+        py3k_skip('bytes vs unicode')
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)
         get_char = lib.ptr('get_char', ['P', 'H'], 'c')
@@ -290,6 +291,7 @@
         assert buf[:8] == b'*' + b'\x00'*6 + b'a'
 
     def test_returning_str(self):
+        py3k_skip('bytes vs unicode')
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)
         char_check = lib.ptr('char_check', ['c', 'c'], 's')
@@ -558,6 +560,7 @@
         raises(ValueError, "_rawffi.Array('xx')")
 
     def test_longs_ulongs(self):
+        py3k_skip('fails on 32bit')
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)
         some_huge_value = lib.ptr('some_huge_value', [], 'q')
@@ -610,6 +613,7 @@
         cb.free()
 
     def test_another_callback(self):
+        py3k_skip('fails on 32bit')
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)
         runcallback = lib.ptr('runcallback', ['P'], 'q')
@@ -781,6 +785,7 @@
         a.free()
 
     def test_truncate(self):
+        py3k_skip('int vs long')
         import _rawffi, struct
         a = _rawffi.Array('b')(1)
         a[0] = -5
@@ -1049,6 +1054,7 @@
         assert oldnum == _rawffi._num_of_allocated_objects()
 
     def test_array_autofree(self):
+        py3k_skip('bytes vs unicode')
         import gc, _rawffi
         gc.collect()
         oldnum = _rawffi._num_of_allocated_objects()

From noreply at buildbot.pypy.org  Thu Apr 19 18:21:53 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 18:21:53 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: Comment that it's a Very Bad Idea
 to make tests pass by adding
Message-ID: <20120419162153.7F128820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: arm-backend-2
Changeset: r54551:ab1b025eb51b
Date: 2012-04-19 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/ab1b025eb51b/

Log:	Comment that it's a Very Bad Idea to make tests pass by adding "if
	not we_are_translated():" sections. It just means that the same
	tests won't pass after transaltion.

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -357,6 +357,7 @@
         cond = self._emit_call(force_index, adr, arglist, fcond, resloc)
         descr = op.getdescr()
         #XXX Hack, Hack, Hack
+        # XXX NEEDS TO BE FIXED
         if (op.result and not we_are_translated()):
             #XXX check result type
             loc = regalloc.rm.call_result_location(op.result)
@@ -657,11 +658,16 @@
             else:
                 self.mc.LDR_rr(res.value, base_loc.value, ofs.value)
         elif size.value == 2:
+            # XXX NEEDS TO BE FIXED
+            # XXX this doesn't get the correct result: it needs to know
+            # XXX if we want a signed or unsigned result
             if ofs.is_imm():
                 self.mc.LDRH_ri(res.value, base_loc.value, ofs.value)
             else:
                 self.mc.LDRH_rr(res.value, base_loc.value, ofs.value)
         elif size.value == 1:
+            # XXX this doesn't get the correct result: it needs to know
+            # XXX if we want a signed or unsigned result
             if ofs.is_imm():
                 self.mc.LDRB_ri(res.value, base_loc.value, ofs.value)
             else:
@@ -700,8 +706,13 @@
         elif fieldsize.value == 4:
             self.mc.LDR_rr(res_loc.value, base_loc.value, r.ip.value)
         elif fieldsize.value == 2:
+            # XXX NEEDS TO BE FIXED
+            # XXX this doesn't get the correct result: it needs to know
+            # XXX if we want a signed or unsigned result
             self.mc.LDRH_rr(res_loc.value, base_loc.value, r.ip.value)
         elif fieldsize.value == 1:
+            # XXX this doesn't get the correct result: it needs to know
+            # XXX if we want a signed or unsigned result
             self.mc.LDRB_rr(res_loc.value, base_loc.value, r.ip.value)
         else:
             assert 0
@@ -804,9 +815,14 @@
             self.mc.LDR_rr(res.value, base_loc.value, scale_loc.value,
                                                                 cond=fcond)
         elif scale.value == 1:
+            # XXX NEEDS TO BE FIXED
+            # XXX this doesn't get the correct result: it needs to know
+            # XXX if we want a signed or unsigned result
             self.mc.LDRH_rr(res.value, base_loc.value, scale_loc.value,
                                                                 cond=fcond)
         elif scale.value == 0:
+            # XXX this doesn't get the correct result: it needs to know
+            # XXX if we want a signed or unsigned result
             self.mc.LDRB_rr(res.value, base_loc.value, scale_loc.value,
                                                                 cond=fcond)
         else:

From noreply at buildbot.pypy.org  Thu Apr 19 19:48:50 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 19:48:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Expand a bit more the comment
Message-ID: <20120419174850.82D02820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54552:d9b5f8a7d8fe
Date: 2012-04-19 19:46 +0200
http://bitbucket.org/pypy/pypy/changeset/d9b5f8a7d8fe/

Log:	Expand a bit more the comment

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1656,8 +1656,11 @@
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
-            #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower half-word read there
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0;
+            #     this is a complete word (N=4 bytes on 32-bit, N=8 on
+            #     64-bits)
+            #   - keep the lower half of what is read there (i.e.
+            #     truncate to an unsigned 'N / 2' bytes value)
             #   - multiply by 4 (on 32-bits only) and use it as an
             #     offset in type_info_group
             #   - add 16/32 bytes, to go past the TYPE_INFO structure

From noreply at buildbot.pypy.org  Thu Apr 19 19:48:51 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 19:48:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: Argh
Message-ID: <20120419174851.BB10D820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54553:d65a34cca537
Date: 2012-04-19 19:47 +0200
http://bitbucket.org/pypy/pypy/changeset/d65a34cca537/

Log:	Argh

diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -229,8 +229,8 @@
                     return thing
                 elif hasattr(thing, '__name__'): # mostly types and functions
                     return thing.__name__
-                elif hasattr(thing, 'name'): # mostly ClassDescs
-                    return thing.name
+                elif hasattr(thing, 'name') and isinstance(thing.name, str):
+                    return thing.name            # mostly ClassDescs
                 elif isinstance(thing, tuple):
                     return '_'.join(map(nameof, thing))
                 else:

From noreply at buildbot.pypy.org  Thu Apr 19 19:48:53 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 19:48:53 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120419174853.6F181820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54554:bc86c14c03bd
Date: 2012-04-19 19:47 +0200
http://bitbucket.org/pypy/pypy/changeset/bc86c14c03bd/

Log:	hg merge default

diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -229,8 +229,8 @@
                     return thing
                 elif hasattr(thing, '__name__'): # mostly types and functions
                     return thing.__name__
-                elif hasattr(thing, 'name'): # mostly ClassDescs
-                    return thing.name
+                elif hasattr(thing, 'name') and isinstance(thing.name, str):
+                    return thing.name            # mostly ClassDescs
                 elif isinstance(thing, tuple):
                     return '_'.join(map(nameof, thing))
                 else:
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1656,15 +1656,21 @@
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
-            #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower 16 bits read there
-            #   - multiply by 4 and use it as an offset in type_info_group
-            #   - add 16 bytes, to go past the TYPE_INFO structure
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0;
+            #     this is a complete word (N=4 bytes on 32-bit, N=8 on
+            #     64-bits)
+            #   - keep the lower half of what is read there (i.e.
+            #     truncate to an unsigned 'N / 2' bytes value)
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
             loc = locs[1]
             assert isinstance(loc, ImmedLoc)
             classptr = loc.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the 16 bits in the object header
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 or 4 on a big-endian machine.
             from pypy.rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,10 +453,22 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        # prefix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        # suffix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        # type checks
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+        self.raises(space, api, TypeError,
+                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
+                    2, 10, 1)
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_exception.py b/pypy/module/pypyjit/test_pypy_c/test_exception.py
--- a/pypy/module/pypyjit/test_pypy_c/test_exception.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_exception.py
@@ -91,3 +91,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_continue_in_finally(self):
+        # check that 'continue' inside a try:finally: block is correctly
+        # detected as closing a loop
+        py.test.skip("is this case important?")
+        def f(n):
+            i = 0
+            while 1:
+                try:
+                    if i < n:
+                        continue
+                finally:
+                    i += 1
+                return i
+
+        log = self.run(f, [2000])
+        assert log.result == 2001
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i40 = int_add_ovf(i31, 1)
+            guard_no_overflow(descr=...)
+            i41 = int_lt(i40, i33)
+            guard_true(i41, descr=...)
+            --TICK--
+            jump(..., descr=...)
+        """)
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,12 +295,11 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,31 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        res = 'bar' in x
+        assert res is True
+        #
+        class MyError(Exception):
+            pass
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise MyError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(MyError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1168,8 +1168,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct

From noreply at buildbot.pypy.org  Thu Apr 19 20:00:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 20:00:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix two more cases that could, in theory,
 give strange results if
Message-ID: <20120419180034.8CD9A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54555:96124bafacbe
Date: 2012-04-19 19:59 +0200
http://bitbucket.org/pypy/pypy/changeset/96124bafacbe/

Log:	Fix two more cases that could, in theory, give strange results if
	'name' happens to be something else than a string (like some
	method).

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -49,7 +49,9 @@
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
         # maybe we want this shorter:
-        name = getattr(self, 'name', '?')
+        name = getattr(self, 'name', None)
+        if not isinstance(name, str):
+            name = '?'
         return "<%s %s>" % (self.__class__.__name__, name)
 
     def call_args(self, args):
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -378,7 +378,10 @@
     __slots__ = ()
 
     def __repr__(self):
-        s = '%s(%s)' % (self.__class__.__name__, getattr(self, 'name', ''))
+        name = getattr(self, 'name', '')
+        if not isinstance(name, str):
+            name = ''
+        s = '%s(%s)' % (self.__class__.__name__, name)
         w_cls = getattr(self, 'w__class__', None)
         if w_cls is not None and w_cls is not self:
             s += ' instance of %s' % self.w__class__

From noreply at buildbot.pypy.org  Thu Apr 19 20:04:18 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 19 Apr 2012 20:04:18 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: stacklets are not compatible with stm
	right now.
Message-ID: <20120419180418.61592820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54556:ffae6c7c3160
Date: 2012-04-19 20:03 +0200
http://bitbucket.org/pypy/pypy/changeset/ffae6c7c3160/

Log:	stacklets are not compatible with stm right now.

diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py
--- a/pypy/config/translationoption.py
+++ b/pypy/config/translationoption.py
@@ -107,7 +107,9 @@
                default=False, cmdline="--thread"),
     BoolOption("stm", "enable use of Software Transactional Memory",
                default=False, cmdline="--stm",
-               requires=[("translation.gc", "stmgc")]),
+               requires=[("translation.gc", "stmgc"),
+                         ("translation.continuation", False),  # XXX for now
+                         ]),
     BoolOption("sandbox", "Produce a fully-sandboxed executable",
                default=False, cmdline="--sandbox",
                requires=[("translation.thread", False)],

From noreply at buildbot.pypy.org  Thu Apr 19 20:24:38 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 19 Apr 2012 20:24:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: add import, fixes issue 1133
Message-ID: <20120419182438.C8FAD820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54557:3afde514d040
Date: 2012-04-19 21:23 +0300
http://bitbucket.org/pypy/pypy/changeset/3afde514d040/

Log:	add import, fixes issue 1133

diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno

From noreply at buildbot.pypy.org  Fri Apr 20 10:43:04 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 10:43:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Ignore PyErr_SetInterrupt() if no
	'signal' module.
Message-ID: <20120420084304.4740A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54558:f4977370180a
Date: 2012-04-20 08:04 +0200
http://bitbucket.org/pypy/pypy/changeset/f4977370180a/

Log:	Ignore PyErr_SetInterrupt() if no 'signal' module.

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):

From noreply at buildbot.pypy.org  Fri Apr 20 10:43:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 10:43:05 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120420084305.C3D3C820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54559:a144a7717de1
Date: 2012-04-20 08:04 +0200
http://bitbucket.org/pypy/pypy/changeset/a144a7717de1/

Log:	hg merge default

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -49,7 +49,9 @@
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
         # maybe we want this shorter:
-        name = getattr(self, 'name', '?')
+        name = getattr(self, 'name', None)
+        if not isinstance(name, str):
+            name = '?'
         return "<%s %s>" % (self.__class__.__name__, name)
 
     def call_args(self, args):
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -378,7 +378,10 @@
     __slots__ = ()
 
     def __repr__(self):
-        s = '%s(%s)' % (self.__class__.__name__, getattr(self, 'name', ''))
+        name = getattr(self, 'name', '')
+        if not isinstance(name, str):
+            name = ''
+        s = '%s(%s)' % (self.__class__.__name__, name)
         w_cls = getattr(self, 'w__class__', None)
         if w_cls is not None and w_cls is not self:
             s += ' instance of %s' % self.w__class__

From noreply at buildbot.pypy.org  Fri Apr 20 10:43:07 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 10:43:07 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Disable the 'thread' module if
	'transaction' is enabled.
Message-ID: <20120420084307.09EFD820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54560:1a2a28c8c0e8
Date: 2012-04-20 08:12 +0200
http://bitbucket.org/pypy/pypy/changeset/1a2a28c8c0e8/

Log:	Disable the 'thread' module if 'transaction' is enabled.

diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py
--- a/pypy/translator/goal/targetpypystandalone.py
+++ b/pypy/translator/goal/targetpypystandalone.py
@@ -149,6 +149,14 @@
         # expose the following variables to ease debugging
         global space, entry_point
 
+        if config.translation.stm:
+            config.objspace.usemodules.transaction = True
+            config.objspace.usemodules.signal = False     # XXX! FIXME
+            config.objspace.usemodules.thread = False
+            config.translation.thread = False
+        elif config.objspace.usemodules.transaction:
+            raise Exception("use --stm, not --withmod-transaction alone")
+
         if config.objspace.allworkingmodules:
             from pypy.config.pypyoption import enable_allworkingmodules
             enable_allworkingmodules(config)
@@ -185,12 +193,6 @@
                 # module if translation.continuation cannot be enabled
                 config.objspace.usemodules._continuation = False
 
-        if config.translation.stm:
-            config.objspace.usemodules.transaction = True
-            config.objspace.usemodules.signal = False     # XXX!
-        elif config.objspace.usemodules.transaction:
-            raise Exception("use --stm, not --withmod-transaction alone")
-
         if not config.translation.rweakref:
             config.objspace.usemodules._weakref = False
 

From noreply at buildbot.pypy.org  Fri Apr 20 10:43:08 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 10:43:08 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix comment.
Message-ID: <20120420084308.3FADF820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54561:5caa1edd8f86
Date: 2012-04-20 09:47 +0200
http://bitbucket.org/pypy/pypy/changeset/5caa1edd8f86/

Log:	Fix comment.

diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py
--- a/pypy/rpython/memory/gc/stmshared.py
+++ b/pypy/rpython/memory/gc/stmshared.py
@@ -27,8 +27,8 @@
         self.chained_list = NULL
 
     def malloc_object(self, totalsize):
-        """Malloc.  You must also call add_regular() or add_special() later.
-        Note that it is not zero-filled."""
+        """Malloc.  You should also call add_regular() later, or keep it in
+        some other data structure.  Note that it is not zero-filled."""
         adr1 = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize), 0)
         llarena.arena_reserve(adr1, totalsize)
         return adr1 + self.gc.gcheaderbuilder.size_gc_header

From noreply at buildbot.pypy.org  Fri Apr 20 10:43:09 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 10:43:09 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Thread-local data in the transaction
 module. See docstring for
Message-ID: <20120420084309.D3B28820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54562:5ccd3d3146e0
Date: 2012-04-20 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/5ccd3d3146e0/

Log:	Thread-local data in the transaction module. See docstring for two
	foreseen usage patterns.

diff --git a/pypy/module/transaction/__init__.py b/pypy/module/transaction/__init__.py
--- a/pypy/module/transaction/__init__.py
+++ b/pypy/module/transaction/__init__.py
@@ -11,6 +11,7 @@
         'run': 'interp_transaction.run',
         'add_epoll': 'interp_epoll.add_epoll',        # xxx linux only
         'remove_epoll': 'interp_epoll.remove_epoll',  # xxx linux only
+        'local': 'interp_local.W_Local',
     }
 
     appleveldefs = {
diff --git a/pypy/module/transaction/interp_local.py b/pypy/module/transaction/interp_local.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/interp_local.py
@@ -0,0 +1,53 @@
+from pypy.interpreter.baseobjspace import Wrappable
+from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty,
+    descr_get_dict)
+from pypy.module.transaction.interp_transaction import state
+
+
+class W_Local(Wrappable):
+    """Thread-local data.  Behaves like a regular object, but its content
+    is not shared between multiple concurrently-running transactions.
+    It can be accessed without conflicts.
+
+    It can be used for purely transaction-local data.
+
+    It can also be used for long-living caches that store values that
+    are (1) not too costly to compute and (2) not too memory-hungry,
+    because they will end up being computed and stored once per actual
+    thread.
+    """
+
+    def __init__(self, space):
+        self.dicts = []
+        self._update_dicts(space)
+        # unless we call transaction.set_num_threads() afterwards, this
+        # 'local' object is now initialized with the correct number of
+        # dictionaries, to avoid conflicts later if _update_dicts() is
+        # called in a transaction.
+
+    def _update_dicts(self, space):
+        new = state.get_number_of_threads() - len(self.dicts)
+        if new <= 0:
+            return
+        # update the list without appending to it (to keep it non-resizable)
+        self.dicts = self.dicts + [space.newdict(instance=True)
+                                   for i in range(new)]
+
+    def getdict(self, space):
+        n = state.get_thread_number()
+        try:
+            return self.dicts[n]
+        except IndexError:
+            self._update_dicts(space)
+            assert n < len(self.dicts)
+            return self.dicts[n]
+
+def descr_local__new__(space, w_subtype):
+    local = W_Local(space)
+    return space.wrap(local)
+
+W_Local.typedef = TypeDef("transaction.local",
+            __new__ = interp2app(descr_local__new__),
+            __dict__ = GetSetProperty(descr_get_dict, cls=W_Local),
+            )
+W_Local.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -23,11 +23,13 @@
         self.ll_no_tasks_pending_lock = threadintf.null_ll_lock
         self.ll_unfinished_lock = threadintf.null_ll_lock
         self.threadobjs = {}      # empty during translation
+        self.threadnums = {}      # empty during translation
         self.epolls = None
         self.pending = Fifo()
 
     def _freeze_(self):
         self.threadobjs.clear()
+        self.threadnums.clear()
         return False
 
     def startup(self, space, w_module):
@@ -52,6 +54,7 @@
         assert id not in self.threadobjs
         ec._transaction_pending = Fifo()
         self.threadobjs[id] = ec
+        self.threadnums[id] = len(self.threadnums)
 
     # ---------- interface for ThreadLocals ----------
     # This works really like a thread-local, which may have slightly
@@ -69,6 +72,7 @@
         id = rstm.thread_id()
         assert id == MAIN_THREAD_ID   # should not be used from a transaction
         self.threadobjs[id] = value
+        self.threadnums = {id: 0}
 
     def getmainthreadvalue(self):
         return self.threadobjs.get(MAIN_THREAD_ID, None)
@@ -80,6 +84,14 @@
         for id in self.threadobjs.keys():
             if id != MAIN_THREAD_ID:
                 del self.threadobjs[id]
+        self.threadnums = {MAIN_THREAD_ID: 0}
+
+    def get_thread_number(self):
+        id = rstm.thread_id()
+        return self.threadnums[id]
+
+    def get_number_of_threads(self):
+        return 1 + self.num_threads
 
     # ----------
 
diff --git a/pypy/module/transaction/test/test_local.py b/pypy/module/transaction/test/test_local.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/transaction/test/test_local.py
@@ -0,0 +1,75 @@
+import py
+from pypy.conftest import gettestobjspace
+
+
+class AppTestLocal:
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['transaction'])
+
+    def test_simple(self):
+        import transaction
+        x = transaction.local()
+        x.foo = 42
+        assert x.foo == 42
+        assert hasattr(x, 'foo')
+        assert not hasattr(x, 'bar')
+        assert getattr(x, 'foo', 84) == 42
+        assert getattr(x, 'bar', 84) == 84
+
+    def test_transaction_local(self):
+        import transaction
+        transaction.set_num_threads(2)
+        x = transaction.local()
+        all_lists = []
+
+        def f(n):
+            if not hasattr(x, 'lst'):
+                x.lst = []
+                all_lists.append(x.lst)
+            x.lst.append(n)
+            if n > 0:
+                transaction.add(f, n - 1)
+                transaction.add(f, n - 1)
+        transaction.add(f, 5)
+        transaction.run()
+
+        assert not hasattr(x, 'lst')
+        assert len(all_lists) == 2
+        total = all_lists[0] + all_lists[1]
+        assert total.count(5) == 1
+        assert total.count(4) == 2
+        assert total.count(3) == 4
+        assert total.count(2) == 8
+        assert total.count(1) == 16
+        assert total.count(0) == 32
+        assert len(total) == 63
+
+    def test_transaction_local_growing(self):
+        import transaction
+        transaction.set_num_threads(1)
+        x = transaction.local()
+        all_lists = []
+
+        def f(n):
+            if not hasattr(x, 'lst'):
+                x.lst = []
+                all_lists.append(x.lst)
+            x.lst.append(n)
+            if n > 0:
+                transaction.add(f, n - 1)
+                transaction.add(f, n - 1)
+        transaction.add(f, 5)
+
+        transaction.set_num_threads(2)    # more than 1 specified above
+        transaction.run()
+
+        assert not hasattr(x, 'lst')
+        assert len(all_lists) == 2
+        total = all_lists[0] + all_lists[1]
+        assert total.count(5) == 1
+        assert total.count(4) == 2
+        assert total.count(3) == 4
+        assert total.count(2) == 8
+        assert total.count(1) == 16
+        assert total.count(0) == 32
+        assert len(total) == 63

From noreply at buildbot.pypy.org  Fri Apr 20 10:43:11 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 10:43:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120420084311.54F74820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54563:9c07ef6786f8
Date: 2012-04-20 10:42 +0200
http://bitbucket.org/pypy/pypy/changeset/9c07ef6786f8/

Log:	merge heads

diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno

From noreply at buildbot.pypy.org  Fri Apr 20 11:10:33 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 11:10:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: Hack hack more to make the ssl tests
 pass (at least a bit more) when
Message-ID: <20120420091033.E83BB820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54564:3c320ce4ad61
Date: 2012-04-20 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/3c320ce4ad61/

Log:	Hack hack more to make the ssl tests pass (at least a bit more) when
	untranslated.

diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -6,6 +6,7 @@
 import sys, os
 
 link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -27,6 +28,7 @@
         # amount of troubles due to symbol versions
         # and 0.9.8/1.0.0
         link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
     else:
         libraries += ['ssl', 'crypto']
 
@@ -41,6 +43,7 @@
 eci = ExternalCompilationInfo(
     libraries = libraries,
     link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Fri Apr 20 11:38:18 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 11:38:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: Change the default mapping of the keys
	Home and End
Message-ID: <20120420093818.7DBB7820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54565:771401a72a83
Date: 2012-04-20 11:37 +0200
http://bitbucket.org/pypy/pypy/changeset/771401a72a83/

Log:	Change the default mapping of the keys Home and End to go to the
	start/end of the current line instead of the start/end of the whole
	multi-line block.

diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal

From noreply at buildbot.pypy.org  Fri Apr 20 12:01:14 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Apr 2012 12:01:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix include order
Message-ID: <20120420100114.48176820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54566:9a563fca7bf2
Date: 2012-04-20 12:36 +0300
http://bitbucket.org/pypy/pypy/changeset/9a563fca7bf2/

Log:	fix include order

diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(

From noreply at buildbot.pypy.org  Fri Apr 20 12:01:15 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Apr 2012 12:01:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: clean up test, fix for windows
Message-ID: <20120420100115.8F29F820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54567:b1d67b6d3c43
Date: 2012-04-20 12:59 +0300
http://bitbucket.org/pypy/pypy/changeset/b1d67b6d3c43/

Log:	clean up test, fix for windows

diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 

From noreply at buildbot.pypy.org  Fri Apr 20 12:01:16 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Apr 2012 12:01:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120420100116.CCAC0820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54568:8217d20a5697
Date: 2012-04-20 13:00 +0300
http://bitbucket.org/pypy/pypy/changeset/8217d20a5697/

Log:	merge heads

diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -6,6 +6,7 @@
 import sys, os
 
 link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -27,6 +28,7 @@
         # amount of troubles due to symbol versions
         # and 0.9.8/1.0.0
         link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
     else:
         libraries += ['ssl', 'crypto']
 
@@ -41,6 +43,7 @@
 eci = ExternalCompilationInfo(
     libraries = libraries,
     link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Fri Apr 20 12:04:35 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Apr 2012 12:04:35 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge from default
Message-ID: <20120420100435.D3F7B820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54569:cadf8ce4b0db
Date: 2012-04-20 13:03 +0300
http://bitbucket.org/pypy/pypy/changeset/cadf8ce4b0db/

Log:	merge from default

diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -229,8 +229,8 @@
                     return thing
                 elif hasattr(thing, '__name__'): # mostly types and functions
                     return thing.__name__
-                elif hasattr(thing, 'name'): # mostly ClassDescs
-                    return thing.name
+                elif hasattr(thing, 'name') and isinstance(thing.name, str):
+                    return thing.name            # mostly ClassDescs
                 elif isinstance(thing, tuple):
                     return '_'.join(map(nameof, thing))
                 else:
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -49,7 +49,9 @@
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
         # maybe we want this shorter:
-        name = getattr(self, 'name', '?')
+        name = getattr(self, 'name', None)
+        if not isinstance(name, str):
+            name = '?'
         return "<%s %s>" % (self.__class__.__name__, name)
 
     def call_args(self, args):
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1656,15 +1656,21 @@
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
-            #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower 16 bits read there
-            #   - multiply by 4 and use it as an offset in type_info_group
-            #   - add 16 bytes, to go past the TYPE_INFO structure
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0;
+            #     this is a complete word (N=4 bytes on 32-bit, N=8 on
+            #     64-bits)
+            #   - keep the lower half of what is read there (i.e.
+            #     truncate to an unsigned 'N / 2' bytes value)
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
             loc = locs[1]
             assert isinstance(loc, ImmedLoc)
             classptr = loc.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the 16 bits in the object header
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 or 4 on a big-endian machine.
             from pypy.rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -378,7 +378,10 @@
     __slots__ = ()
 
     def __repr__(self):
-        s = '%s(%s)' % (self.__class__.__name__, getattr(self, 'name', ''))
+        name = getattr(self, 'name', '')
+        if not isinstance(name, str):
+            name = ''
+        s = '%s(%s)' % (self.__class__.__name__, name)
         w_cls = getattr(self, 'w__class__', None)
         if w_cls is not None and w_cls is not self:
             s += ' instance of %s' % self.w__class__
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,12 +295,11 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -6,6 +6,7 @@
 import sys, os
 
 link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -27,6 +28,7 @@
         # amount of troubles due to symbol versions
         # and 0.9.8/1.0.0
         link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
     else:
         libraries += ['ssl', 'crypto']
 
@@ -41,6 +43,7 @@
 eci = ExternalCompilationInfo(
     libraries = libraries,
     link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Fri Apr 20 12:08:45 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 12:08:45 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: Update the $$.
Message-ID: <20120420100845.8D9F9820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r355:a497f05d778b
Date: 2012-04-20 12:08 +0200
http://bitbucket.org/pypy/pypy.org/changeset/a497f05d778b/

Log:	Update the $$.

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,12 +9,12 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 41.4
+      value: 41.5
    });
   });
 </script>
 
-   $43438 of $105000 (41.4%)
+   $43563 of $105000 (41.5%)
    <div id="progressbar">
    </div>
 
diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -9,12 +9,12 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 73.5
+      value: 74.2
    });
   });
 </script>
 
-   $44099 of $60000 (73.5%)
+   $44502 of $60000 (74.2%)
    <div id="progressbar">
    </div>
 
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -9,12 +9,12 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 3.8
+      value: 13.5
    });
   });
 </script>
 
-   $1925 of $50400 (3.8%)
+   $6791 of $50400 (13.5%)
    <div id="progressbar">
    </div>
 

From noreply at buildbot.pypy.org  Fri Apr 20 12:27:59 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 12:27:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: write numpy.where
Message-ID: <20120420102759.F2E4A820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54570:257a319fcf72
Date: 2012-04-20 12:25 +0200
http://bitbucket.org/pypy/pypy/changeset/257a319fcf72/

Log:	write numpy.where

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -30,6 +30,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(arr, x, y)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,62 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.x.eq(other.x, compare_array_no) and
+                self.y.eq(other.y, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Call2(%s, %s, %s)' % (self.name, self.left.debug_repr(),
+                                      self.right.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,9 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement

From noreply at buildbot.pypy.org  Fri Apr 20 12:28:01 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 12:28:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120420102801.44C6C820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54571:483b9f63a068
Date: 2012-04-20 12:27 +0200
http://bitbucket.org/pypy/pypy/changeset/483b9f63a068/

Log:	merge

diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -6,6 +6,7 @@
 import sys, os
 
 link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -27,6 +28,7 @@
         # amount of troubles due to symbol versions
         # and 0.9.8/1.0.0
         link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
     else:
         libraries += ['ssl', 'crypto']
 
@@ -41,6 +43,7 @@
 eci = ExternalCompilationInfo(
     libraries = libraries,
     link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Fri Apr 20 12:32:14 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 12:32:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: uh typos, untested so far
Message-ID: <20120420103214.E0C33820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54572:a7e85fda3731
Date: 2012-04-20 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/a7e85fda3731/

Log:	uh typos, untested so far

diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -519,8 +519,8 @@
             return False
         assert isinstance(other, WhereSignature)
         return (self.arrsig.eq(other.arrsig, compare_array_no) and
-                self.x.eq(other.x, compare_array_no) and
-                self.y.eq(other.y, compare_array_no))
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
 
     def _invent_array_numbering(self, arr, cache):
         from pypy.module.micronumpy.interp_arrayops import WhereArray

From noreply at buildbot.pypy.org  Fri Apr 20 12:34:30 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 12:34:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix debug_repr
Message-ID: <20120420103430.ED2F9820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54573:5c0472d2d20a
Date: 2012-04-20 12:34 +0200
http://bitbucket.org/pypy/pypy/changeset/5c0472d2d20a/

Log:	fix debug_repr

diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -555,5 +555,6 @@
             return rhs
 
     def debug_repr(self):
-        return 'Call2(%s, %s, %s)' % (self.name, self.left.debug_repr(),
-                                      self.right.debug_repr())
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())

From noreply at buildbot.pypy.org  Fri Apr 20 13:38:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 13:38:34 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add 'transaction.local' in the emulator
 too. (This is a complete
Message-ID: <20120420113834.0E699820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54574:48347aefd009
Date: 2012-04-20 13:23 +0200
http://bitbucket.org/pypy/pypy/changeset/48347aefd009/

Log:	Add 'transaction.local' in the emulator too. (This is a complete
	implementation.)

diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -106,3 +106,17 @@
     finally:
         _in_transaction = False
         pending.clear()   # this is the behavior we get with interp_transaction
+
+
+class local(object):
+    """Thread-local data.  Behaves like a regular object, but its content
+    is not shared between multiple concurrently-running transactions.
+    It can be accessed without conflicts.
+
+    It can be used for purely transaction-local data.
+
+    It can also be used for long-living caches that store values that
+    are (1) not too costly to compute and (2) not too memory-hungry,
+    because they will end up being computed and stored once per actual
+    thread.
+    """

From noreply at buildbot.pypy.org  Fri Apr 20 14:02:17 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 20 Apr 2012 14:02:17 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: raise correct error for invalid
	fd
Message-ID: <20120420120217.6AB73820B3@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54575:e7ce29040264
Date: 2012-04-20 15:01 +0300
http://bitbucket.org/pypy/pypy/changeset/e7ce29040264/

Log:	raise correct error for invalid fd

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -5,6 +5,7 @@
 from pypy.rlib import streamio
 from pypy.rlib.rarithmetic import r_longlong
 from pypy.rlib.rstring import StringBuilder
+from pypy.rlib.rposix import validate_fd
 from pypy.module._file.interp_stream import W_AbstractStream, StreamErrors
 from pypy.module.posix.interp_posix import dispatch_filename
 from pypy.interpreter.error import OperationError, operationerrfmt
@@ -271,6 +272,15 @@
 
     def file_fdopen(self, fd, mode="r", buffering=-1):
         try:
+            # Catch invalid fd early to get an OSError rather than an
+            # IOError from streamio
+            validate_fd(fd)
+        except OSError, e:
+            w_error = self.space.call_function(self.space.w_OSError,
+                    self.space.wrap(e.errno), self.space.wrap(e.strerror),
+                    self.space.wrap(e.filename))
+            raise OperationError(self.space.w_OSError, w_error)
+        try:
             self.direct_fdopen(fd, mode, buffering)
         except StreamErrors, e:
             raise wrap_streamerror(self.space, e, self.w_name)

From noreply at buildbot.pypy.org  Fri Apr 20 14:06:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 14:06:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Bug fix in the absence of 'thread'.
Message-ID: <20120420120641.C34A1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54576:e59cb3b60396
Date: 2012-04-20 13:53 +0200
http://bitbucket.org/pypy/pypy/changeset/e59cb3b60396/

Log:	Bug fix in the absence of 'thread'.

diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
 try:
     from thread import _local as local
 except ImportError:
-    local = object    # no threads
+    class local(object):    # no threads
+        pass
 
 class ConvMode:
     encoding = 'ascii'

From noreply at buildbot.pypy.org  Fri Apr 20 14:06:43 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 14:06:43 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120420120643.10E26820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54577:e352f141e7fc
Date: 2012-04-20 13:55 +0200
http://bitbucket.org/pypy/pypy/changeset/e352f141e7fc/

Log:	hg merge default

diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
 try:
     from thread import _local as local
 except ImportError:
-    local = object    # no threads
+    class local(object):    # no threads
+        pass
 
 class ConvMode:
     encoding = 'ascii'
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -6,6 +6,7 @@
 import sys, os
 
 link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -27,6 +28,7 @@
         # amount of troubles due to symbol versions
         # and 0.9.8/1.0.0
         link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
     else:
         libraries += ['ssl', 'crypto']
 
@@ -41,6 +43,7 @@
 eci = ExternalCompilationInfo(
     libraries = libraries,
     link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Fri Apr 20 14:06:44 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 14:06:44 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Allow gc_identityhash and gc_id, too.
Message-ID: <20120420120644.5DA53820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54578:34e0998cf7c5
Date: 2012-04-20 14:01 +0200
http://bitbucket.org/pypy/pypy/changeset/34e0998cf7c5/

Log:	Allow gc_identityhash and gc_id, too.

diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -15,6 +15,7 @@
     'jit_force_virtual', 'jit_force_virtualizable',
     'jit_force_quasi_immutable', 'jit_marker', 'jit_is_virtual',
     'jit_record_known_class',
+    'gc_identityhash', 'gc_id',
     ])
 ALWAYS_ALLOW_OPERATIONS |= set(lloperation.enum_tryfold_ops())
 

From noreply at buildbot.pypy.org  Fri Apr 20 14:06:45 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 14:06:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120420120645.9CFA4820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54579:9e290b906bf5
Date: 2012-04-20 14:05 +0200
http://bitbucket.org/pypy/pypy/changeset/9e290b906bf5/

Log:	merge heads

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -30,6 +30,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(arr, x, y)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,9 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 

From noreply at buildbot.pypy.org  Fri Apr 20 14:55:18 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 14:55:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: debugging feature and a passing test
Message-ID: <20120420125518.75A68820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54580:50fcaa0b8eef
Date: 2012-04-20 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/50fcaa0b8eef/

Log:	debugging feature and a passing test

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -4,7 +4,7 @@
 from pypy.module.micronumpy import signature
 
 class WhereArray(VirtualArray):
-    def __init__(self, arr, x, y):
+    def __init__(self, space, arr, x, y):
         self.arr = arr
         self.x = x
         self.y = y
@@ -87,4 +87,4 @@
     arr = convert_to_array(space, w_arr)
     x = convert_to_array(space, w_x)
     y = convert_to_array(space, w_y)
-    return WhereArray(arr, x, y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -1583,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -7,3 +7,10 @@
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5), zeros(5))
         assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()

From noreply at buildbot.pypy.org  Fri Apr 20 14:55:37 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 14:55:37 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add '__pypy__.local',
 re-exposing either 'thread._local', or
Message-ID: <20120420125537.96C37820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54581:ce0532024f73
Date: 2012-04-20 14:15 +0200
http://bitbucket.org/pypy/pypy/changeset/ce0532024f73/

Log:	Add '__pypy__.local', re-exposing either 'thread._local', or
	'transaction.local', or just a dummy class if we have neither.

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -29,6 +29,7 @@
 
 class Module(MixedModule):
     appleveldefs = {
+        'local'                     : 'app_local.local',
     }
 
     interpleveldefs = {
diff --git a/pypy/module/__pypy__/app_local.py b/pypy/module/__pypy__/app_local.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/app_local.py
@@ -0,0 +1,11 @@
+import sys
+
+if 'thread' in sys.builtin_module_names:
+    from thread import _local as local
+elif 'transaction' in sys.builtin_module_names:
+    from transaction import local
+else:
+    class local(object):
+        """A pseudo-thread-local class.
+        As this interpreter does not have threads, it is a regular class.
+        """
diff --git a/pypy/module/__pypy__/test/test_local.py b/pypy/module/__pypy__/test/test_local.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_local.py
@@ -0,0 +1,19 @@
+import py
+from pypy.conftest import gettestobjspace
+
+
+class AppTestLocal(object):
+    def test_local(self):
+        import __pypy__
+        x = __pypy__.local()
+        x.foo = 42
+        assert x.foo == 42     # hard to test more :-)
+
+
+class AppTestLocalWithThreads(object):
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['thread'])
+
+    def test_local(self):
+        import __pypy__, thread
+        assert __pypy__.local is thread._local

From noreply at buildbot.pypy.org  Fri Apr 20 14:55:38 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 14:55:38 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Use __pypy__.local here.
Message-ID: <20120420125538.CDB75820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54582:b5e2dbeb1aaa
Date: 2012-04-20 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/b5e2dbeb1aaa/

Log:	Use __pypy__.local here.

diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -1,10 +1,6 @@
 
 import _rawffi, sys
-try:
-    from thread import _local as local
-except ImportError:
-    class local(object):    # no threads
-        pass
+from __pypy__ import local
 
 class ConvMode:
     encoding = 'ascii'

From noreply at buildbot.pypy.org  Fri Apr 20 19:02:32 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:02:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Debugging help.
Message-ID: <20120420170232.4BB41820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54583:290898449fd2
Date: 2012-04-20 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/290898449fd2/

Log:	Debugging help.

diff --git a/pypy/translator/c/src/exception.h b/pypy/translator/c/src/exception.h
--- a/pypy/translator/c/src/exception.h
+++ b/pypy/translator/c/src/exception.h
@@ -43,6 +43,16 @@
           filename, lineno, functionname);
 }
 #endif
+#else   /* !DO_LOG_EXC: define the function anyway, so that we can shut
+           off the prints of a debug_exc by remaking only testing_1.o */
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname);
+#ifndef PYPY_NOT_MAIN_FILE
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname)
+{
+}
+#endif
 #endif  /* DO_LOG_EXC */
 
 /* Hint: functions and macros not defined here, like RPyRaiseException,

From noreply at buildbot.pypy.org  Fri Apr 20 19:02:33 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:02:33 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Print another message when we are
 leaving transactional mode.
Message-ID: <20120420170233.8F47982369@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54584:f25378799d99
Date: 2012-04-20 17:59 +0200
http://bitbucket.org/pypy/pypy/changeset/f25378799d99/

Log:	Print another message when we are leaving transactional mode.

diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -276,6 +276,7 @@
                 self.log.event('specializing transactionally %d blocks' %
                                (len(pending),))
                 transaction.run()
+                self.log.event('left transactional mode')
                 blockcount += len(pending)
                 self.already_seen.update(dict.fromkeys(pending, True))
 

From noreply at buildbot.pypy.org  Fri Apr 20 19:02:38 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:02:38 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Rename this method
Message-ID: <20120420170238.9C06382F52@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54588:2c6d4f49051b
Date: 2012-04-20 18:53 +0200
http://bitbucket.org/pypy/pypy/changeset/2c6d4f49051b/

Log:	Rename this method

diff --git a/pypy/module/transaction/interp_local.py b/pypy/module/transaction/interp_local.py
--- a/pypy/module/transaction/interp_local.py
+++ b/pypy/module/transaction/interp_local.py
@@ -26,7 +26,7 @@
         # called in a transaction.
 
     def _update_dicts(self, space):
-        new = state.get_number_of_threads() - len(self.dicts)
+        new = state.get_total_number_of_threads() - len(self.dicts)
         if new <= 0:
             return
         # update the list without appending to it (to keep it non-resizable)
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -100,7 +100,7 @@
         id = rstm.thread_id()
         return self.threadnums[id]
 
-    def get_number_of_threads(self):
+    def get_total_number_of_threads(self):
         return 1 + self.num_threads
 
     # ----------

From noreply at buildbot.pypy.org  Fri Apr 20 19:02:39 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:02:39 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix epoll too.
Message-ID: <20120420170239.D644E820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54589:c255347d4061
Date: 2012-04-20 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/c255347d4061/

Log:	Fix epoll too.

diff --git a/pypy/module/transaction/interp_epoll.py b/pypy/module/transaction/interp_epoll.py
--- a/pypy/module/transaction/interp_epoll.py
+++ b/pypy/module/transaction/interp_epoll.py
@@ -45,8 +45,8 @@
 
     def run(self):
         # This code is run non-transactionally.  Careful, no GC available.
-        state = interp_transaction.state
-        if state.has_exception() or self.force_quit:
+        ts = interp_transaction.state.transactionalstate
+        if ts.has_exception() or self.force_quit:
             return
         fd = rffi.cast(rffi.INT, self.epoller.epfd)
         maxevents = rffi.cast(rffi.INT, self.maxevents)
@@ -60,9 +60,9 @@
                 nfds = 0    # ignore, just wait for more later
             else:
                 # unsure how to trigger this case
-                state = interp_transaction.state
-                state.got_exception_errno = errno
-                state.must_reraise_exception(_reraise_from_errno)
+                ts = interp_transaction.state.transactionalstate
+                ts.got_exception_errno = errno
+                ts.must_reraise_exception(_reraise_from_errno)
                 return
         # We have to allocate new PendingCallback objects, but we can't
         # allocate anything here because we are not running transactionally.
@@ -96,10 +96,9 @@
                                              space.wrap(self.events))
 
 
-def _reraise_from_errno():
-    state = interp_transaction.state
-    space = state.space
-    errno = state.got_exception_errno
+def _reraise_from_errno(transactionalstate):
+    space = interp_transaction.state.space
+    errno = transactionalstate.got_exception_errno
     msg = os.strerror(errno)
     w_type = space.w_IOError
     w_error = space.call_function(w_type, space.wrap(errno), space.wrap(msg))
@@ -109,9 +108,7 @@
 @unwrap_spec(epoller=W_Epoll)
 def add_epoll(space, epoller, w_callback):
     state = interp_transaction.state
-    if state.epolls is None:
-        state.epolls = {}
-    elif epoller in state.epolls:
+    if epoller in state.epolls:
         raise OperationError(state.w_error,
             space.wrap("add_epoll(ep): ep is already registered"))
     pending = EPollPending(space, epoller, w_callback)
@@ -121,10 +118,7 @@
 @unwrap_spec(epoller=W_Epoll)
 def remove_epoll(space, epoller):
     state = interp_transaction.state
-    if state.epolls is None:
-        pending = None
-    else:
-        pending = state.epolls.get(epoller, None)
+    pending = state.epolls.get(epoller, None)
     if pending is None:
         raise OperationError(state.w_error,
             space.wrap("remove_epoll(ep): ep is not registered"))
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -34,6 +34,7 @@
         self.ll_not_ready_to_start_lock = threadintf.null_ll_lock
         self.threadobjs = {}      # empty during translation
         self.threadnums = {}      # empty during translation
+        self.epolls = {}
         self.pending = Fifo()
 
     def _freeze_(self):
@@ -95,6 +96,7 @@
             if id != MAIN_THREAD_ID:
                 del self.threadobjs[id]
         self.threadnums = {MAIN_THREAD_ID: 0}
+        self.epolls.clear()
 
     def get_thread_number(self):
         id = rstm.thread_id()
@@ -163,7 +165,6 @@
 
     def __init__(self):
         self._reraise_exception = None
-        self.epolls = None
 
     def has_exception(self):
         return self._reraise_exception is not None

From noreply at buildbot.pypy.org  Fri Apr 20 19:02:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:02:34 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add an essential assert. I think it is
	broken right now.
Message-ID: <20120420170234.D009A82F4F@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54585:ef84ff47459d
Date: 2012-04-20 17:59 +0200
http://bitbucket.org/pypy/pypy/changeset/ef84ff47459d/

Log:	Add an essential assert. I think it is broken right now. Will write
	comments...

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -223,6 +223,7 @@
 
 def _setup_thread(_, retry_counter):
     """Setup a thread.  Run as a transaction because it allocates."""
+    assert state.running
     my_thread_id = rstm.thread_id()
     my_ec = state.space.createexecutioncontext()
     state.add_thread(my_thread_id, my_ec)

From noreply at buildbot.pypy.org  Fri Apr 20 19:02:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:02:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120420170241.1E98B820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54590:ae2bca32fdd5
Date: 2012-04-20 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/ae2bca32fdd5/

Log:	merge heads

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -4,7 +4,7 @@
 from pypy.module.micronumpy import signature
 
 class WhereArray(VirtualArray):
-    def __init__(self, arr, x, y):
+    def __init__(self, space, arr, x, y):
         self.arr = arr
         self.x = x
         self.y = y
@@ -87,4 +87,4 @@
     arr = convert_to_array(space, w_arr)
     x = convert_to_array(space, w_x)
     y = convert_to_array(space, w_y)
-    return WhereArray(arr, x, y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -1583,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -7,3 +7,10 @@
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5), zeros(5))
         assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()

From noreply at buildbot.pypy.org  Fri Apr 20 19:02:36 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:02:36 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120420170236.233D482F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54586:d932b70587e7
Date: 2012-04-20 17:59 +0200
http://bitbucket.org/pypy/pypy/changeset/d932b70587e7/

Log:	hg merge default

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -30,6 +30,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(arr, x, y)
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,9 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 
diff --git a/pypy/translator/c/src/exception.h b/pypy/translator/c/src/exception.h
--- a/pypy/translator/c/src/exception.h
+++ b/pypy/translator/c/src/exception.h
@@ -43,6 +43,16 @@
           filename, lineno, functionname);
 }
 #endif
+#else   /* !DO_LOG_EXC: define the function anyway, so that we can shut
+           off the prints of a debug_exc by remaking only testing_1.o */
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname);
+#ifndef PYPY_NOT_MAIN_FILE
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname)
+{
+}
+#endif
 #endif  /* DO_LOG_EXC */
 
 /* Hint: functions and macros not defined here, like RPyRaiseException,

From noreply at buildbot.pypy.org  Fri Apr 20 19:02:37 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:02:37 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix.  See comment.
Message-ID: <20120420170237.612ED82F51@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54587:d1daac6a89d2
Date: 2012-04-20 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/d1daac6a89d2/

Log:	Fix. See comment.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -12,19 +12,28 @@
 
 
 class State(object):
+    # Warning: this is the class of a singleton.  It Must Not Be Used
+    # Inside Any Transaction!!!!  This code is written with the
+    # expectation that it is really a global singleton used for
+    # synchronization between transactions.  So if the STM logic kicks
+    # in and makes local copies of it, we loose.
+    #
+    # Instead, we have a TransactionalState instance that can be
+    # written to by transactions (possibly causing conflicts).
 
     def initialize(self, space):
         self.space = space
         self.running = False
         self.num_threads = NUM_THREADS_DEFAULT
+        self.transactionalstate = None
         #
         self.w_error = None
         self.ll_lock = threadintf.null_ll_lock
         self.ll_no_tasks_pending_lock = threadintf.null_ll_lock
         self.ll_unfinished_lock = threadintf.null_ll_lock
+        self.ll_not_ready_to_start_lock = threadintf.null_ll_lock
         self.threadobjs = {}      # empty during translation
         self.threadnums = {}      # empty during translation
-        self.epolls = None
         self.pending = Fifo()
 
     def _freeze_(self):
@@ -41,6 +50,7 @@
         self.ll_no_tasks_pending_lock = threadintf.allocate_lock()
         self.ll_unfinished_lock = threadintf.allocate_lock()
         self.lock_unfinished()
+        self.ll_not_ready_to_start_lock = threadintf.allocate_lock()
         self.startup_run()
 
     def startup_run(self):
@@ -139,8 +149,21 @@
         """Release ll_unfinished_lock."""
         threadintf.release(self.ll_unfinished_lock)
 
-    def init_exceptions(self):
+    def lock_not_ready_to_start(self):
+        """This lock is acquired when the threads are still starting.
+        It is released when all threads are past their initialization."""
+        threadintf.acquire(self.ll_not_ready_to_start_lock, True)
+
+    def unlock_not_ready_to_start(self):
+        """Release ll_not_ready_to_start_lock."""
+        threadintf.release(self.ll_not_ready_to_start_lock)
+
+
+class TransactionalState(object):
+
+    def __init__(self):
         self._reraise_exception = None
+        self.epolls = None
 
     def has_exception(self):
         return self._reraise_exception is not None
@@ -150,7 +173,7 @@
 
     def close_exceptions(self):
         if self._reraise_exception is not None:
-            self._reraise_exception()
+            self._reraise_exception(self)
 
 
 state = State()
@@ -184,13 +207,15 @@
         if retry_counter > 0:
             pending.register() # retrying: will be done later, try others first
             return
-        if state.has_exception():
+        ts = state.transactionalstate
+        if ts.has_exception():
             return   # return early if there is already an exception to reraise
         try:
             pending.run_in_transaction(state.space)
         except Exception, e:
-            state.got_exception_applevel = e
-            state.must_reraise_exception(_reraise_from_applevel)
+            ts = state.transactionalstate
+            ts.got_exception_applevel = e
+            ts.must_reraise_exception(_reraise_from_applevel)
 
 
 class Pending(AbstractPending):
@@ -202,9 +227,9 @@
         space.call_args(self.w_callback, self.args)
 
 
-def _reraise_from_applevel():
-    e = state.got_exception_applevel
-    state.got_exception_applevel = None
+def _reraise_from_applevel(transactionalstate):
+    e = transactionalstate.got_exception_applevel
+    transactionalstate.got_exception_applevel = None
     raise e
 
 
@@ -235,8 +260,17 @@
     # so we need to be very careful.
     rstm.descriptor_init()
     state.lock()
+    rstm.perform_transaction(_setup_thread, AbstractPending, None)
+    # wait until all threads reach this point to continue
+    if state.num_waiting_threads + 1 < state.num_threads:
+        state.num_waiting_threads += 1
+        state.unlock()
+        state.lock_not_ready_to_start()
+        state.lock()
+    else:
+        state.num_waiting_threads = 0
+    state.unlock_not_ready_to_start()
     #
-    rstm.perform_transaction(_setup_thread, AbstractPending, None)
     my_transactions_pending = state.getvalue()._transaction_pending
     #
     while True:
@@ -312,8 +346,9 @@
     state.num_waiting_threads = 0
     state.finished = False
     #
+    state.transactionalstate = TransactionalState()
+    state.lock_not_ready_to_start()
     state.running = True
-    state.init_exceptions()
     #
     # start the threads and wait for all of them to finish
     _run()
@@ -323,7 +358,8 @@
     assert not state.is_locked_no_tasks_pending()
     state.clear_all_values_apart_from_main()
     state.running = False
-    state.epolls = None
+    ts = state.transactionalstate
+    state.transactionalstate = None
     #
     # now re-raise the exception that we got in a transaction
-    state.close_exceptions()
+    ts.close_exceptions()

From noreply at buildbot.pypy.org  Fri Apr 20 19:19:48 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 19:19:48 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: merge default
Message-ID: <20120420171948.D0DF3820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54591:77d35f6e6b22
Date: 2012-04-20 18:16 +0200
http://bitbucket.org/pypy/pypy/changeset/77d35f6e6b22/

Log:	merge default

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
 try:
     from thread import _local as local
 except ImportError:
-    local = object    # no threads
+    class local(object):    # no threads
+        pass
 
 class ConvMode:
     encoding = 'ascii'
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -229,8 +229,8 @@
                     return thing
                 elif hasattr(thing, '__name__'): # mostly types and functions
                     return thing.__name__
-                elif hasattr(thing, 'name'): # mostly ClassDescs
-                    return thing.name
+                elif hasattr(thing, 'name') and isinstance(thing.name, str):
+                    return thing.name            # mostly ClassDescs
                 elif isinstance(thing, tuple):
                     return '_'.join(map(nameof, thing))
                 else:
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -169,9 +173,17 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +198,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +236,20 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +408,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -49,7 +49,9 @@
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
         # maybe we want this shorter:
-        name = getattr(self, 'name', '?')
+        name = getattr(self, 'name', None)
+        if not isinstance(name, str):
+            name = '?'
         return "<%s %s>" % (self.__class__.__name__, name)
 
     def call_args(self, args):
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1656,15 +1656,21 @@
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
-            #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower 16 bits read there
-            #   - multiply by 4 and use it as an offset in type_info_group
-            #   - add 16 bytes, to go past the TYPE_INFO structure
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0;
+            #     this is a complete word (N=4 bytes on 32-bit, N=8 on
+            #     64-bits)
+            #   - keep the lower half of what is read there (i.e.
+            #     truncate to an unsigned 'N / 2' bytes value)
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
             loc = locs[1]
             assert isinstance(loc, ImmedLoc)
             classptr = loc.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the 16 bits in the object header
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 or 4 on a big-endian machine.
             from pypy.rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,10 +453,22 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        # prefix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        # suffix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        # type checks
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+        self.raises(space, api, TypeError,
+                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
+                    2, 10, 1)
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
@@ -30,6 +31,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, space, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -513,7 +514,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1436,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
@@ -1559,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,16 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
@@ -1410,6 +1409,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -762,6 +762,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -772,7 +774,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -785,6 +788,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -796,7 +801,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/module/pypyjit/test_pypy_c/test_exception.py b/pypy/module/pypyjit/test_pypy_c/test_exception.py
--- a/pypy/module/pypyjit/test_pypy_c/test_exception.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_exception.py
@@ -91,3 +91,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_continue_in_finally(self):
+        # check that 'continue' inside a try:finally: block is correctly
+        # detected as closing a loop
+        py.test.skip("is this case important?")
+        def f(n):
+            i = 0
+            while 1:
+                try:
+                    if i < n:
+                        continue
+                finally:
+                    i += 1
+                return i
+
+        log = self.run(f, [2000])
+        assert log.result == 2001
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i40 = int_add_ovf(i31, 1)
+            guard_no_overflow(descr=...)
+            i41 = int_lt(i40, i33)
+            guard_true(i41, descr=...)
+            --TICK--
+            jump(..., descr=...)
+        """)
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -378,7 +378,10 @@
     __slots__ = ()
 
     def __repr__(self):
-        s = '%s(%s)' % (self.__class__.__name__, getattr(self, 'name', ''))
+        name = getattr(self, 'name', '')
+        if not isinstance(name, str):
+            name = ''
+        s = '%s(%s)' % (self.__class__.__name__, name)
         w_cls = getattr(self, 'w__class__', None)
         if w_cls is not None and w_cls is not self:
             s += ' instance of %s' % self.w__class__
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,12 +295,11 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,31 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        res = 'bar' in x
+        assert res is True
+        #
+        class MyError(Exception):
+            pass
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise MyError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(MyError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -204,6 +204,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.call_location()
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,10 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +19,18 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +42,8 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1162,8 +1162,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Fri Apr 20 19:19:50 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 19:19:50 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: count the number of pinnedo
 bjects in the nursery
Message-ID: <20120420171950.25BE8820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54592:b949f7afc02a
Date: 2012-04-20 18:19 +0200
http://bitbucket.org/pypy/pypy/changeset/b949f7afc02a/

Log:	count the number of pinnedo bjects in the nursery

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1334,6 +1334,7 @@
         self.surviving_pinned_objects.sort()
         while self.surviving_pinned_objects.non_empty():
             next = self.surviving_pinned_objects.pop()
+            self.pinned_objects_in_nursery += 1
             assert next >= prev
             size = llarena.getfakearenaaddress(next) - prev
             llarena.arena_reset(prev, size, 2)
@@ -1360,6 +1361,8 @@
         #
         debug_print("minor collect, total memory used:",
                     self.get_total_memory_used())
+        debug_print("number of pinned objects:",
+                    self.pinned_objects_in_nursery)
         if self.DEBUG >= 2:
             self.debug_check_consistency()     # expensive!
         debug_stop("gc-minor")

From noreply at buildbot.pypy.org  Fri Apr 20 19:33:58 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:33:58 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: Don't decrement
 'pinned_objects_in_nursery' if the object is not pinned
Message-ID: <20120420173358.615C1820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-minimark-pinning
Changeset: r54593:50b176a35634
Date: 2012-04-20 19:33 +0200
http://bitbucket.org/pypy/pypy/changeset/50b176a35634/

Log:	Don't decrement 'pinned_objects_in_nursery' if the object is not
	pinned (e.g. if a previous pin() failed).

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -810,8 +810,9 @@
         self.header(obj).tid |= GCFLAG_PINNED
 
     def unpin(self, obj):
-        self.pinned_objects_in_nursery -= 1
-        self.header(obj).tid &= ~GCFLAG_PINNED
+        if self.header(obj).tid & GCFLAG_PINNED:
+            self.pinned_objects_in_nursery -= 1
+            self.header(obj).tid &= ~GCFLAG_PINNED
 
     def shrink_array(self, obj, smallerlength):
         #

From noreply at buildbot.pypy.org  Fri Apr 20 19:35:34 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 20 Apr 2012 19:35:34 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Better _freeze_,
 to make sure everything is clear during translation.
Message-ID: <20120420173534.65E12820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54594:fc2c0855b755
Date: 2012-04-20 19:35 +0200
http://bitbucket.org/pypy/pypy/changeset/fc2c0855b755/

Log:	Better _freeze_, to make sure everything is clear during
	translation.

diff --git a/pypy/module/transaction/fifo.py b/pypy/module/transaction/fifo.py
--- a/pypy/module/transaction/fifo.py
+++ b/pypy/module/transaction/fifo.py
@@ -4,6 +4,8 @@
         self.first = None
         self.last = None
 
+    clear = __init__
+
     def append(self, newitem):
         newitem.next = None
         if self.last is None:
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -28,18 +28,21 @@
         self.transactionalstate = None
         #
         self.w_error = None
+        self.threadobjs = {}      # empty during translation
+        self.threadnums = {}      # empty during translation
+        self.epolls = {}
+        self.pending = Fifo()
+        self._freeze_()
+
+    def _freeze_(self):
         self.ll_lock = threadintf.null_ll_lock
         self.ll_no_tasks_pending_lock = threadintf.null_ll_lock
         self.ll_unfinished_lock = threadintf.null_ll_lock
         self.ll_not_ready_to_start_lock = threadintf.null_ll_lock
-        self.threadobjs = {}      # empty during translation
-        self.threadnums = {}      # empty during translation
-        self.epolls = {}
-        self.pending = Fifo()
-
-    def _freeze_(self):
         self.threadobjs.clear()
         self.threadnums.clear()
+        self.epolls.clear()
+        self.pending.clear()
         return False
 
     def startup(self, space, w_module):

From noreply at buildbot.pypy.org  Fri Apr 20 19:48:02 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 19:48:02 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: finish support for pinned
 objects with shadows
Message-ID: <20120420174802.BE78C820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54595:13aecb5e5a44
Date: 2012-04-20 19:44 +0200
http://bitbucket.org/pypy/pypy/changeset/13aecb5e5a44/

Log:	finish support for pinned objects with shadows

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1262,6 +1262,12 @@
                 self.old_objects_with_cards_set.append(dest_addr)
                 dest_hdr.tid |= GCFLAG_CARDS_SET
 
+    def record_pinned_object(self, obj, new_dict):
+        obj = obj + self.gcheaderbuilder.size_gc_header
+        shadow = self.nursery_objects_shadows.get(obj)
+        if shadow != NULL:
+            new_dict.setitem(obj, shadow)
+
     # ----------
     # Nursery collection
 
@@ -1318,7 +1324,14 @@
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
-            self.nursery_objects_shadows.clear()
+            if self.surviving_pinned_objects.non_empty():
+                new_shadows = self.AddressDict()
+                self.surviving_pinned_objects.foreach(self.record_pinned_object,
+                                                      new_shadows)
+                self.nursery_objects_shadows.delete()
+                self.nursery_objects_shadows = new_shadows
+            else:
+                self.nursery_objects_shadows.clear()
         #
         # Walk the list of young raw-malloced objects, and either free
         # them or make them old.
diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -542,9 +542,11 @@
         self.gc.pin(llmemory.cast_ptr_to_adr(s))
         self.gc.minor_collection(1)
         self.gc.unpin(llmemory.cast_ptr_to_adr(s))
-        self.gc.minor_collection(1)
         assert self.gc.nursery_free != self.gc.nursery
         # we still have a pinned object
+        self.gc.minor_collection(1)
+        assert self.gc.nursery_free == self.gc.nursery
+        # we don't have a pinned object any more
 
     def test_writebarrier_before_copy(self):
         from pypy.rpython.memory.gc import minimark

From noreply at buildbot.pypy.org  Fri Apr 20 19:48:04 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 20 Apr 2012 19:48:04 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: merge
Message-ID: <20120420174804.06591820B3@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54596:a21db6fede56
Date: 2012-04-20 19:47 +0200
http://bitbucket.org/pypy/pypy/changeset/a21db6fede56/

Log:	merge

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -810,8 +810,9 @@
         self.header(obj).tid |= GCFLAG_PINNED
 
     def unpin(self, obj):
-        self.pinned_objects_in_nursery -= 1
-        self.header(obj).tid &= ~GCFLAG_PINNED
+        if self.header(obj).tid & GCFLAG_PINNED:
+            self.pinned_objects_in_nursery -= 1
+            self.header(obj).tid &= ~GCFLAG_PINNED
 
     def shrink_array(self, obj, smallerlength):
         #

From noreply at buildbot.pypy.org  Fri Apr 20 21:04:46 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 20 Apr 2012 21:04:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: start of cppyy documentation
Message-ID: <20120420190446.D7762820B3@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r54597:1d560d5a8154
Date: 2012-04-20 12:05 -0700
http://bitbucket.org/pypy/pypy/changeset/1d560d5a8154/

Log:	start of cppyy documentation

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/cppyy.rst
@@ -0,0 +1,109 @@
+============================
+cppyy: C++ bindings for pypy
+============================
+
+The cppyy module provides C++ bindings for PyPy by using the reflection
+information extracted from C++ header files by means of the
+`Reflex package`_.
+For this to work, you have to both install Reflex and build PyPy from the
+reflex-support branch.
+As indicated by this being a branch, support for Reflex is still
+experimental.
+However, it is functional enough to put it in the hands of those who want
+to give it a try.
+In the medium term, cppyy will move away from Reflex and instead use
+`cling`_ as its backend, which is based on `llvm`_.
+Although that will change the logistics on the generation of reflection
+information, it will not change the python-side interface.
+
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`cling`: http://root.cern.ch/drupal/content/cling
+.. _`llvm`: http://llvm.org/
+
+
+Installation
+============
+
+For now, the easiest way of getting the latest version of Reflex, is by
+installing the ROOT package.
+`Download`_ a binary or install from `source`_.
+Some Linux and Mac systems may have ROOT provided in the list of scientific
+software of their packager.
+A current, standalone version of Reflex should be provided at some point,
+once the dependencies and general packaging have been thought out.
+Also, make sure you have a version of `gccxml`_ installed, which is most
+easily provided by the packager of your system.
+If you read up on gccxml, you'll probably notice that it is no longer being
+developed and hence will not provide C++11 support.
+That's why the medium plan is to move to `cling`_.
+
+.. _`Download`: http://root.cern.ch/drupal/content/downloading-root
+.. _`source`: http://root.cern.ch/drupal/content/installing-root-source
+.. _`gccxml`: http://www.gccxml.org
+
+Next, get the `PyPy sources`_, select the reflex-support branch, and build
+pypy-c.
+For the build to succeed, the ``$ROOTSYS`` environment variable must point to
+the location of your ROOT installation::
+
+    $ hg clone https://bitbucket.org/pypy/pypy
+    $ cd pypy
+    $ hg up reflex-support
+    $ cd pypy/translator/goal
+    $ python translate.py -O jit --gcrootfinder=shadowstack targetpypystandalone.py --withmod-cppyy
+
+This will build a ``pypy-c`` that includes the cppyy module, and through that,
+Reflex support.
+Of course, if you already have a prebuilt version of the ``pypy`` interpreter,
+you can use that for the translation rather than ``python``.
+
+.. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
+
+
+Basic example
+=============
+
+Now test with a trivial example whether all packages are properly installed
+and functional.
+First, create a C++ header file with some class in it (note that all functions
+are made inline for convenience; a real-world example would of course have a
+corresponding source file)::
+
+    $ cat MyClass.h
+    class MyClass {
+    public:
+        MyClass( int i = -99) : m_myint( i ) {}
+
+        int GetMyInt() { return m_myint; }
+        void SetMyInt( int i ) { m_myint = i; }
+
+    public:
+       int m_myint;
+    };
+
+Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
+code::
+
+    $ genreflex MyClass.h
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+
+Now you're ready to use the bindings.
+Since the bindings are designed to look pythonistic, it should be
+straightforward::
+
+    $ pypy-c
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libMyClassDict.so")
+    <CPPLibrary object at 0xb6fd7c4c>
+    >>>> myinst = cppyy.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> myinst.SetMyInt(33)
+    >>>> print myinst.m_myint
+    33
+    >>>> myinst.m_myint = 77
+    >>>> print myinst.GetMyInt()
+    77
+    >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
+
+That's all there is to it!
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -23,9 +23,12 @@
 
 * Write them in RPython as mixedmodule_, using *rffi* as bindings.
 
+* Write them in C++ and bind them through Reflex_ (EXPERIMENTAL)
+
 .. _ctypes: #CTypes
 .. _\_ffi: #LibFFI
 .. _mixedmodule: #Mixed Modules
+.. _Reflex: #Reflex
 
 CTypes
 ======
@@ -110,3 +113,34 @@
 
 XXX we should provide detailed docs about lltype and rffi, especially if we
     want people to follow that way.
+
+Reflex
+======
+
+This method is only experimental for now, and is being exercised on a branch,
+`reflex-support`_, so you will have to build PyPy yourself.
+The method works by using the `Reflex package`_ to provide reflection
+information of the C++ code, which is then used to automatically generate
+bindings at runtime, which can then be used from python.
+Full details are `available here`_.
+
+.. _`reflex-support`: cppyy.html
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`available here`: cppyy.html
+
+Pros
+----
+
+If it works, it is mostly automatic, and hence easy in use.
+The bindings can make use of direct pointers, in which case the calls are
+very fast.
+
+Cons
+----
+
+C++ is a large language, and these bindings are not yet feature-complete.
+Although missing features should do no harm if you don't use them, if you do
+need a particular feature, it may be necessary to work around it in python
+or with a C++ helper function.
+Although Reflex works on various platforms, the bindings with PyPy have only
+been tested on Linux.

From noreply at buildbot.pypy.org  Fri Apr 20 23:53:39 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 20 Apr 2012 23:53:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: cppyy documentation: feature list and
 short intro to PyCintex
Message-ID: <20120420215339.99455820B3@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r54598:e1487d12d116
Date: 2012-04-20 14:54 -0700
http://bitbucket.org/pypy/pypy/changeset/e1487d12d116/

Log:	cppyy documentation: feature list and short intro to PyCintex

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -1,5 +1,5 @@
 ============================
-cppyy: C++ bindings for pypy
+cppyy: C++ bindings for PyPy
 ============================
 
 The cppyy module provides C++ bindings for PyPy by using the reflection
@@ -26,6 +26,8 @@
 
 For now, the easiest way of getting the latest version of Reflex, is by
 installing the ROOT package.
+Besides getting the latest version of Reflex, another advantage is that with
+the full ROOT package, you can also use your Reflex-bound code on `CPython`_.
 `Download`_ a binary or install from `source`_.
 Some Linux and Mac systems may have ROOT provided in the list of scientific
 software of their packager.
@@ -54,7 +56,7 @@
 
 This will build a ``pypy-c`` that includes the cppyy module, and through that,
 Reflex support.
-Of course, if you already have a prebuilt version of the ``pypy`` interpreter,
+Of course, if you already have a pre-built version of the ``pypy`` interpreter,
 you can use that for the translation rather than ``python``.
 
 .. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
@@ -72,10 +74,10 @@
     $ cat MyClass.h
     class MyClass {
     public:
-        MyClass( int i = -99) : m_myint( i ) {}
+        MyClass(int i = -99) : m_myint(i) {}
 
         int GetMyInt() { return m_myint; }
-        void SetMyInt( int i ) { m_myint = i; }
+        void SetMyInt(int i) { m_myint = i; }
 
     public:
        int m_myint;
@@ -107,3 +109,185 @@
     >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
 
 That's all there is to it!
+
+
+Features
+========
+
+The following is not meant to be an exhaustive list, since cppyy is still
+under active development.
+Furthermore, the intention is that every feature is as natural as possible on
+the python side, so if you find something missing in the list below, simply
+try it out.
+It is not always possible to provide exact mapping between python and C++
+(active memory management is one such case), but by and large, if the use of a
+feature does not strike you as obvious, it is more likely to simply be a bug.
+That is a strong statement to make, but also a worthy goal.
+
+* **abstract classes**: Are represented as python classes, since they are
+  needed to complete the inheritance hierarchies, but will raise an exception
+  if an attempt is made to instantiate from them.
+
+* **arrays**: Supported for builtin data types only, as used from module
+  ``array``.
+  Out-of-bounds checking is limited to those cases where the size is known at
+  compile time (and hence part of the reflection info).
+
+* **builtin data types**: Map onto the expected equivalent python types, with
+  the caveat that there may be size differences, and thus it is possible that
+  exceptions are raised if an overflow is detected.
+
+* **casting**: Is supposed to be unnecessary.
+  Object pointer returns from functions provide the most derived class known
+  in the hierarchy of the object being returned.
+  This is important to preserve object identity as well as to make casting,
+  a pure C++ feature after all, superfluous.
+
+* **classes and structs**: Get mapped onto python classes, where they can be
+  instantiated as expected.
+  If classes are inner classes or live in a namespace, their naming and
+  location will reflect that.
+
+* **data members**: Public data members are represented as python properties
+  and provide read and write access on instances as expected.
+
+* **default arguments**: C++ default arguments work as expected, but python
+  keywords are not supported.
+  It is technically possible to support keywords, but for the C++ interface,
+  the formal argument names have no meaning and are not considered part of the
+  API, hence it is not a good idea to use keywords.
+
+* **doc strings**: The doc string of a method or function contains the C++
+  arguments and return types of all overloads of that name, as applicable.
+
+* **functions**: Work as expected and live in their appropriate namespace
+  (which can be the global one, ``cppyy.gbl``).
+
+* **inheritance**: All combinations of inheritance on the C++ (single,
+  multiple, virtual) are supported in the binding.
+  However, new python classes can only use single inheritance from a bound C++
+  class.
+  Multiple inheritance would introduce two "this" pointers in the binding.
+  This is a current, not a fundamental, limitation.
+  The C++ side will not see any overridden methods on the python side, as
+  cross-inheritance is planned but not yet supported.
+
+* **methods**: Are represented as python methods and work as expected.
+  They are first class objects and can be bound to an instance.
+  Virtual C++ methods work as expected.
+  To select a specific virtual method, do like with normal python classes
+  that override methods: select it from the class that you need, rather than
+  calling the method on the instance.
+
+* **namespaces**: Are represented as python classes.
+  Namespaces are open-ended than classes, so sometimes initial access may
+  result in updates as data and functions are looked up and constructed
+  lazily.
+  Thus the result of ``dir()`` on a namespace should not be relied upon: it
+  only shows the already accessed members. (TODO: to be fixed by implementing
+  __dir__.)
+  The global namespace is ``cppyy.gbl``.
+
+* **operator conversions**: If defined in the C++ class and a python
+  equivalent exists (i.e. all builtin integer and floating point types, as well
+  as ``bool``), it will map onto that python conversion.
+  Note that ``char*`` is mapped onto ``__str__``.
+
+* **operator overloads**: If defined in the C++ class and if a python
+  equivalent is available (not always the case, think e.g. of ``operator||``),
+  then they work as expected.
+  Special care needs to be taken for global operator overloads in C++: first,
+  make sure that they are actually reflected, especially for the global
+  overloads for ``operator==`` and ``operator!=`` of STL iterators in the case
+  of gcc.
+  Second, make sure that reflection info is loaded in the proper order.
+  I.e. that these global overloads are available before use.
+
+* **pointers**: For builtin data types, see arrays.
+  For objects, a pointer to an object and an object looks the same, unless
+  the pointer is a data member.
+  In that case, assigning to the data member will cause a copy of the pointer
+  and care should be taken about the object's live time.
+  If a pointer is a global variable, the C++ side can replace the underlying
+  object and the python side will immediately reflect that.
+
+* **static data members**: Are represented as python property objects on the
+  class and the meta-class.
+  Both reading and write access is as expected.
+
+* **static methods**: Are represented as python's ``staticmethod`` objects
+  and can be called both from the class as well as from instances.
+
+* **strings**: The std::string class is considered a builtin C++ type and
+  mixes quite well with python's str.
+  Python's str can be passed where a ``const char*`` is expected, and an str
+  will be returned if the return type is ``const char*``.
+
+* **templated classes**: Are represented in a meta-class style in python.
+  This looks a little bit confusing, but conceptually is rather natural.
+  For example, given the class ``std::vector<int>``, the meta-class part would
+  be ``std.vector`` in python.
+  Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
+  create an instance of that class, do ``std.vector(int)()``.
+  Note that templates can be build up by handing actual classes to the class
+  instantiation, or by passing in the list of template arguments as a string.
+  The former is a lot easier to work with if you have template instantiations
+  using classes that themselves are templates (etc.) in the arguments.
+  All classes must already exist in the loaded reflection info.
+
+* **unary operators**: Are supported if a python equivalent exists, and if the
+  operator is defined in the C++ class.
+
+You can always find more detailed examples and see the full of supported
+features by looking at the tests in pypy/module/cppyy/test.
+
+CPython
+=======
+
+Most of the ideas in cppyy come originally from the `PyROOT`_ project.
+Although PyROOT does not support Reflex directly, it has an alter ego called
+"PyCintex" that, in a somewhat roundabout way, does.
+If you installed ROOT, rather than just Reflex, PyCintex should be available
+immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment
+variable.
+
+.. _`PyROOT`: http://root.cern.ch/drupal/content/pyroot
+
+There are a couple of minor differences between PyCintex and cppyy, most to do
+with naming.
+The one that you will run into directly, is that PyCintex uses a function
+called ``loadDictionary`` rather than ``load_reflection_info``.
+The reason for this is that Reflex calls the shared libraries that contain
+reflection info "dictionaries."
+However, in python, the name `dictionary` already has a well-defined meaning,
+so a more descriptive name was chosen for cppyy.
+In addition, PyCintex requires that the names of shared libraries so loaded
+start with "lib" in their name.
+The basic example above, rewritten for PyCintex thus goes like this::
+
+    $ python
+    >>>> import PyCintex
+    >>>> PyCintex.loadDictionary("libMyClassDict.so")
+    >>>> myinst = PyCintex.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> myinst.SetMyInt(33)
+    >>>> print myinst.m_myint
+    33
+    >>>> myinst.m_myint = 77
+    >>>> print myinst.GetMyInt()
+    77
+    >>>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
+
+Other naming differences are such things as taking an address of an object.
+In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
+made to follow the naming as in ``ctypes`` and hence use ``addressof``
+(PyROOT/PyCintex predate ``ctypes`` by several years, and the ROOT project
+follows camel-case, hence the differences).
+
+Of course, this is python, so if any of the naming is not to your liking, all
+you have to do is provide a wrapper script that you import instead of
+importing the ``cppyy`` or ``PyCintex`` modules directly.
+In that wrapper script you can rename methods exactly the way you need it.
+
+In the Cling world, all these differences will be resolved.
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -28,7 +28,6 @@
 .. _ctypes: #CTypes
 .. _\_ffi: #LibFFI
 .. _mixedmodule: #Mixed Modules
-.. _Reflex: #Reflex
 
 CTypes
 ======

From noreply at buildbot.pypy.org  Fri Apr 20 23:58:30 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Fri, 20 Apr 2012 23:58:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: oops, python has three >, not four ...
Message-ID: <20120420215830.D8D6F820B3@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r54599:442a3ea22328
Date: 2012-04-20 14:58 -0700
http://bitbucket.org/pypy/pypy/changeset/442a3ea22328/

Log:	oops, python has three >, not four ...

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -266,18 +266,18 @@
 The basic example above, rewritten for PyCintex thus goes like this::
 
     $ python
-    >>>> import PyCintex
-    >>>> PyCintex.loadDictionary("libMyClassDict.so")
-    >>>> myinst = PyCintex.gbl.MyClass(42)
-    >>>> print myinst.GetMyInt()
+    >>> import PyCintex
+    >>> PyCintex.loadDictionary("libMyClassDict.so")
+    >>> myinst = PyCintex.gbl.MyClass(42)
+    >>> print myinst.GetMyInt()
     42
-    >>>> myinst.SetMyInt(33)
-    >>>> print myinst.m_myint
+    >>> myinst.SetMyInt(33)
+    >>> print myinst.m_myint
     33
-    >>>> myinst.m_myint = 77
-    >>>> print myinst.GetMyInt()
+    >>> myinst.m_myint = 77
+    >>> print myinst.GetMyInt()
     77
-    >>>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
+    >>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
 
 Other naming differences are such things as taking an address of an object.
 In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
@@ -290,4 +290,4 @@
 importing the ``cppyy`` or ``PyCintex`` modules directly.
 In that wrapper script you can rename methods exactly the way you need it.
 
-In the Cling world, all these differences will be resolved.
+In the cling world, all these differences will be resolved.

From noreply at buildbot.pypy.org  Sat Apr 21 00:28:04 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Sat, 21 Apr 2012 00:28:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: cppyy doc: introduction of the fast path
Message-ID: <20120420222804.42E53820B3@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r54600:e6931441abcb
Date: 2012-04-20 15:28 -0700
http://bitbucket.org/pypy/pypy/changeset/e6931441abcb/

Log:	cppyy doc: introduction of the fast path

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -37,7 +37,7 @@
 easily provided by the packager of your system.
 If you read up on gccxml, you'll probably notice that it is no longer being
 developed and hence will not provide C++11 support.
-That's why the medium plan is to move to `cling`_.
+That's why the medium term plan is to move to `cling`_.
 
 .. _`Download`: http://root.cern.ch/drupal/content/downloading-root
 .. _`source`: http://root.cern.ch/drupal/content/installing-root-source
@@ -241,6 +241,35 @@
 You can always find more detailed examples and see the full of supported
 features by looking at the tests in pypy/module/cppyy/test.
 
+
+The fast lane
+=============
+
+The following is an experimental feature of cppyy, and that makes it doubly
+experimental, so caveat emptor.
+With a slight modification of Reflex, it can provide function pointers for
+C++ methods, and hence allow PyPy to call those pointers directly, rather than
+calling C++ through a Reflex stub.
+This results in a rather significant speed-up.
+Mind you, the normal stub path is not exactly slow, so for now only use this
+out of curiosity or if you really need it.
+
+To install this patch of Reflex, locate the file genreflex-methptrgetter.patch
+in pypy/module/cppyy and apply it to the genreflex python scripts found in
+``$ROOTSYS/lib``::
+
+    $ cd $ROOTSYS/lib
+    $ patch -p2 < genreflex-methptrgetter.patch
+
+With this patch, ``genreflex`` will have grown the ``--with-methptrgetter``
+option.
+Use this option when running ``genreflex``, and add the
+``-Wno-pmf-conversions`` option to ``g++`` when compiling.
+The rest works the same way: the fast path will be used transparently (which
+also means that you can't actually find out whether it is in use, other than
+by running a micro-benchmark).
+
+
 CPython
 =======
 

From noreply at buildbot.pypy.org  Sat Apr 21 00:48:29 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Sat, 21 Apr 2012 00:48:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: cppyy doc: fix typos and a short
 elaboration on missing features
Message-ID: <20120420224829.63D16820B3@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r54601:ac83f51817bc
Date: 2012-04-20 15:48 -0700
http://bitbucket.org/pypy/pypy/changeset/ac83f51817bc/

Log:	cppyy doc: fix typos and a short elaboration on missing features

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -180,7 +180,7 @@
   calling the method on the instance.
 
 * **namespaces**: Are represented as python classes.
-  Namespaces are open-ended than classes, so sometimes initial access may
+  Namespaces are more open-ended than classes, so sometimes initial access may
   result in updates as data and functions are looked up and constructed
   lazily.
   Thus the result of ``dir()`` on a namespace should not be relied upon: it
@@ -207,13 +207,13 @@
   For objects, a pointer to an object and an object looks the same, unless
   the pointer is a data member.
   In that case, assigning to the data member will cause a copy of the pointer
-  and care should be taken about the object's live time.
+  and care should be taken about the object's life time.
   If a pointer is a global variable, the C++ side can replace the underlying
   object and the python side will immediately reflect that.
 
 * **static data members**: Are represented as python property objects on the
   class and the meta-class.
-  Both reading and write access is as expected.
+  Both read and write access is as expected.
 
 * **static methods**: Are represented as python's ``staticmethod`` objects
   and can be called both from the class as well as from instances.
@@ -229,8 +229,9 @@
   be ``std.vector`` in python.
   Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
   create an instance of that class, do ``std.vector(int)()``.
-  Note that templates can be build up by handing actual classes to the class
-  instantiation, or by passing in the list of template arguments as a string.
+  Note that templates can be build up by handing actual types to the class
+  instantiation (as done in this vector example), or by passing in the list of
+  template arguments as a string.
   The former is a lot easier to work with if you have template instantiations
   using classes that themselves are templates (etc.) in the arguments.
   All classes must already exist in the loaded reflection info.
@@ -241,6 +242,16 @@
 You can always find more detailed examples and see the full of supported
 features by looking at the tests in pypy/module/cppyy/test.
 
+If a feature or reflection info is missing, this is supposed to be handled
+gracefully.
+In fact, there are unit tests explicitly for this purpose (even as their use
+becomes less interesting over time, as the number of missing features
+decreases).
+Only when a missing feature is used, should there be an exception.
+For example, if no reflection info is available for a return type, then a
+class that has a method with that return type can still be used.
+Only that one specific method can not be used.
+
 
 The fast lane
 =============

From notifications-noreply at bitbucket.org  Sat Apr 21 04:21:01 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sat, 21 Apr 2012 02:21:01 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120421022101.13924.7184@bitbucket15.managed.contegix.com>

You have received a notification from William Ting.

Hi, I forked pypy. My fork is at https://bitbucket.org/wting/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Sat Apr 21 08:05:21 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Apr 2012 08:05:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Re-enable this part of the code and fix
	it.
Message-ID: <20120421060521.5BFBF46E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54602:ebca5c7c17dd
Date: 2012-04-21 08:04 +0200
http://bitbucket.org/pypy/pypy/changeset/ebca5c7c17dd/

Log:	Re-enable this part of the code and fix it.

diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -40,7 +40,7 @@
 # In that case, do 5 bits at a time.  The potential drawback is that
 # a table of 2**5 intermediate results is computed.
 
-## FIVEARY_CUTOFF = 8   disabled for now
+FIVEARY_CUTOFF = 8
 
 
 def _mask_digit(x):
@@ -456,7 +456,7 @@
 
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
-        if 1:   ## b.numdigits() <= FIVEARY_CUTOFF:
+        if b.numdigits() <= FIVEARY_CUTOFF:
             # Left-to-right binary exponentiation (HAC Algorithm 14.79)
             # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
             i = b.numdigits() - 1
@@ -469,30 +469,51 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 i -= 1
-##        else:
-##            This code is disabled for now, because it assumes that
-##            SHIFT is a multiple of 5.  It could be fixed but it looks
-##            like it's more troubles than benefits...
-##
-##            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-##            # This is only useful in the case where c != None.
-##            # z still holds 1L
-##            table = [z] * 32
-##            table[0] = z
-##            for i in range(1, 32):
-##                table[i] = _help_mult(table[i-1], a, c)
-##            i = b.numdigits() - 1
-##            while i >= 0:
-##                bi = b.digit(i)
-##                j = SHIFT - 5
-##                while j >= 0:
-##                    index = (bi >> j) & 0x1f
-##                    for k in range(5):
-##                        z = _help_mult(z, z, c)
-##                    if index:
-##                        z = _help_mult(z, table[index], c)
-##                    j -= 5
-##                i -= 1
+        else:
+            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
+            # This is only useful in the case where c != None.
+            # z still holds 1L
+            table = [z] * 32
+            table[0] = z
+            for i in range(1, 32):
+                table[i] = _help_mult(table[i-1], a, c)
+            i = b.numdigits()
+            # Note that here SHIFT is not a multiple of 5.  The difficulty
+            # is to extract 5 bits at a time from 'b', starting from the
+            # most significant digits, so that at the end of the algorithm
+            # it falls exactly to zero.
+            # m  = max number of bits = i * SHIFT
+            # m+ = m rounded up to the next multiple of 5
+            # j  = (m+) % SHIFT = (m+) - (i * SHIFT)
+            # (computed without doing "i * SHIFT", which might overflow)
+            j = i % 5
+            if j != 0:
+                j = 5 - j
+            if not we_are_translated():
+                assert j == (i*SHIFT+4)//5*5 - i*SHIFT
+            #
+            accum = r_uint(0)
+            while True:
+                j -= 5
+                if j >= 0:
+                    index = (accum >> j) & 0x1f
+                else:
+                    # 'accum' does not have enough digit.
+                    # must get the next digit from 'b' in order to complete
+                    i -= 1
+                    if i < 0:
+                        break    # done
+                    bi = b.udigit(i)
+                    index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f
+                    accum = bi
+                    j += SHIFT
+                #
+                for k in range(5):
+                    z = _help_mult(z, z, c)
+                if index:
+                    z = _help_mult(z, table[index], c)
+            #
+            assert j == -5
 
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint.py
@@ -379,6 +379,18 @@
         for n, expected in [(37, 9), (1291, 931), (67889, 39464)]:
             v = two.pow(t, rbigint.fromint(n))
             assert v.toint() == expected
+        #
+        # more tests, comparing against CPython's answer
+        enabled = sample(range(5*32), 10)
+        for i in range(5*32):
+            t = t.mul(two)      # add one random bit
+            if random() >= 0.5:
+                t = t.add(rbigint.fromint(1))
+            if i not in enabled:
+                continue    # don't take forever
+            n = randint(1, sys.maxint)
+            v = two.pow(t, rbigint.fromint(n))
+            assert v.toint() == pow(2, t.tolong(), n)
 
     def test_pow_lln(self):
         x = 10L

From noreply at buildbot.pypy.org  Sat Apr 21 10:01:53 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Apr 2012 10:01:53 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: A random change...
Message-ID: <20120421080153.66A9546E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54603:8af7db90a52d
Date: 2012-04-21 10:01 +0200
http://bitbucket.org/pypy/pypy/changeset/8af7db90a52d/

Log:	A random change...

diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -405,12 +405,12 @@
     'stm_become_inevitable':  LLOp(),
     'stm_descriptor_init':    LLOp(canrun=True),
     'stm_descriptor_done':    LLOp(canrun=True),
-    'stm_enter_transactional_mode': LLOp(canrun=True),
-    'stm_leave_transactional_mode': LLOp(canrun=True),
+    'stm_enter_transactional_mode': LLOp(canrun=True, canmallocgc=True),
+    'stm_leave_transactional_mode': LLOp(canrun=True, canmallocgc=True),
     'stm_writebarrier':       LLOp(sideeffects=False),
     'stm_normalize_global':   LLOp(),
-    'stm_start_transaction':  LLOp(canrun=True),
-    'stm_commit_transaction': LLOp(canrun=True),
+    'stm_start_transaction':  LLOp(canrun=True, canmallocgc=True),
+    'stm_commit_transaction': LLOp(canrun=True, canmallocgc=True),
 
     # __________ address operations __________
 

From noreply at buildbot.pypy.org  Sat Apr 21 10:01:54 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Apr 2012 10:01:54 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: A failing test
Message-ID: <20120421080154.C4CD346E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54604:1772a9a9264a
Date: 2012-04-21 10:01 +0200
http://bitbucket.org/pypy/pypy/changeset/1772a9a9264a/

Log:	A failing test

diff --git a/pypy/translator/c/test/test_standalone.py b/pypy/translator/c/test/test_standalone.py
--- a/pypy/translator/c/test/test_standalone.py
+++ b/pypy/translator/c/test/test_standalone.py
@@ -18,11 +18,14 @@
     config = None
 
     def compile(self, entry_point, debug=True, shared=False,
-                stackcheck=False):
+                stackcheck=False, backendopt=False):
         t = TranslationContext(self.config)
         t.buildannotator().build_types(entry_point, [s_list_of_strings])
         t.buildrtyper().specialize()
 
+        if backendopt:
+            all.backend_optimizations(t)
+
         if stackcheck:
             from pypy.translator.transform import insert_ll_stackcheck
             insert_ll_stackcheck(t)
diff --git a/pypy/translator/stm/test/support.py b/pypy/translator/stm/test/support.py
--- a/pypy/translator/stm/test/support.py
+++ b/pypy/translator/stm/test/support.py
@@ -6,7 +6,7 @@
 class CompiledSTMTests(StandaloneTests):
     gc = "stmgc"
 
-    def compile(self, entry_point):
+    def compile(self, entry_point, **kwds):
         from pypy.config.pypyoption import get_pypy_config
         self.config = get_pypy_config(translating=True)
         self.config.translation.stm = True
@@ -17,7 +17,8 @@
         from pypy.translator.backendopt.canraise import RaiseAnalyzer
         RaiseAnalyzer.fail_on_unknown_operation = True
         try:
-            res = StandaloneTests.compile(self, entry_point, debug=True)
+            res = StandaloneTests.compile(self, entry_point, debug=True,
+                                          **kwds)
         finally:
             RaiseAnalyzer.fail_on_unknown_operation = False
         return res
diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py
--- a/pypy/translator/stm/test/test_ztranslated.py
+++ b/pypy/translator/stm/test/test_ztranslated.py
@@ -10,3 +10,40 @@
         data, dataerr = cbuilder.cmdexec('4 5000', err=True)
         assert 'done sleeping.' in dataerr
         assert 'check ok!' in data
+
+    def test_bug1(self):
+        from pypy.rlib import rstm, rgc
+        from pypy.module.transaction import threadintf
+        #
+        class State:
+            pass
+        state = State()
+        #
+        def _foo(_, retry_counter):
+            rgc.collect(0)
+        def _run_thread():
+            rstm.descriptor_init()
+            rstm.perform_transaction(_foo, X, None)
+            threadintf.release(state.ll_unfinished_lock)
+            rstm.descriptor_done()
+        #
+        class X:
+            def __init__(self, count):
+                self.count = count
+        def g():
+            x = X(1000)
+            rstm.enter_transactional_mode()
+            threadintf.start_new_thread(_run_thread)
+            threadintf.acquire(state.ll_unfinished_lock, True)
+            rstm.leave_transactional_mode()
+            return x
+        def entry_point(argv):
+            state.ll_unfinished_lock = threadintf.allocate_lock()
+            x = X(len(argv))
+            y = g()
+            print '<', x.count, y.count, '>'
+            return 0
+        #
+        t, cbuilder = self.compile(entry_point, backendopt=True)
+        data = cbuilder.cmdexec('a b c d')
+        assert '< 5 1000 >' in data, "got: %r" % (data,)

From noreply at buildbot.pypy.org  Sat Apr 21 10:26:18 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Apr 2012 10:26:18 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix the test.
Message-ID: <20120421082618.ADF7046E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54605:bc9b1f1e564d
Date: 2012-04-21 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/bc9b1f1e564d/

Log:	Fix the test.

diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -70,12 +70,16 @@
         hop.genop("direct_call", [self.teardown_thread_ptr, self.c_const_gc])
 
     def gct_stm_enter_transactional_mode(self, hop):
+        livevars = self.push_roots(hop)
         hop.genop("direct_call", [self.stm_enter_transactional_mode_ptr,
                                   self.c_const_gc])
+        self.pop_roots(hop, livevars)
 
     def gct_stm_leave_transactional_mode(self, hop):
+        livevars = self.push_roots(hop)
         hop.genop("direct_call", [self.stm_leave_transactional_mode_ptr,
                                   self.c_const_gc])
+        self.pop_roots(hop, livevars)
 
     def gct_stm_writebarrier(self, hop):
         op = hop.spaceop
@@ -96,10 +100,14 @@
         hop.genop('cast_adr_to_ptr', [v_globaladr], resultvar=op.result)
 
     def gct_stm_start_transaction(self, hop):
+        livevars = self.push_roots(hop)
         hop.genop("direct_call", [self.stm_start_ptr, self.c_const_gc])
+        self.pop_roots(hop, livevars)
 
     def gct_stm_commit_transaction(self, hop):
+        livevars = self.push_roots(hop)
         hop.genop("direct_call", [self.stm_commit_ptr, self.c_const_gc])
+        self.pop_roots(hop, livevars)
 
 
 class StmShadowStackRootWalker(BaseRootWalker):

From noreply at buildbot.pypy.org  Sat Apr 21 12:03:06 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 21 Apr 2012 12:03:06 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix.
Message-ID: <20120421100306.6264646E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54606:5ec30c19bc26
Date: 2012-04-21 11:31 +0200
http://bitbucket.org/pypy/pypy/changeset/5ec30c19bc26/

Log:	Fix.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -320,14 +320,12 @@
 @rgc.no_collect
 def _run():
     # --- start the threads --- don't use the GC here any more! ---
-    rstm.enter_transactional_mode()
     #
     for i in range(state.num_threads):
         threadintf.start_new_thread(_run_thread)
     #
     state.lock_unfinished()  # wait for all threads to finish
     #
-    rstm.leave_transactional_mode()
     # --- done, we can use the GC again ---
 
 
@@ -355,7 +353,9 @@
     state.running = True
     #
     # start the threads and wait for all of them to finish
+    rstm.enter_transactional_mode()
     _run()
+    rstm.leave_transactional_mode()
     #
     assert state.num_waiting_threads == 0
     assert state.pending.is_empty()

From noreply at buildbot.pypy.org  Sat Apr 21 20:38:36 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 21 Apr 2012 20:38:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20120421183836.2C2E746E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54607:aa342d02a5a2
Date: 2012-04-21 21:34 +0300
http://bitbucket.org/pypy/pypy/changeset/aa342d02a5a2/

Log:	fix test

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,6 @@
 
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_ufunc_instance(self):
         from _numpypy import add, ufunc
@@ -149,7 +148,11 @@
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+        # Since comparisons with nnan and nan all return false,
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
+
 
     def test_fmin(self):
         from _numpypy import fmin
@@ -165,7 +168,9 @@
         assert math.isnan(fmin(0, nan))
         assert math.isnan(fmin(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
     def test_fmod(self):
         from _numpypy import fmod

From noreply at buildbot.pypy.org  Sat Apr 21 20:38:37 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 21 Apr 2012 20:38:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20120421183837.6A62446E1E6@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54608:5fd058965bf7
Date: 2012-04-21 21:36 +0300
http://bitbucket.org/pypy/pypy/changeset/5fd058965bf7/

Log:	fix test

diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -596,7 +596,7 @@
         import sys
         size = 0x14FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
@@ -618,7 +618,7 @@
         import sys
         size = 0x17FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)

From noreply at buildbot.pypy.org  Sat Apr 21 20:43:52 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 21 Apr 2012 20:43:52 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: merge default into branch
Message-ID: <20120421184352.BC28646E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54609:af9cef062227
Date: 2012-04-21 21:39 +0300
http://bitbucket.org/pypy/pypy/changeset/af9cef062227/

Log:	merge default into branch

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
 try:
     from thread import _local as local
 except ImportError:
-    local = object    # no threads
+    class local(object):    # no threads
+        pass
 
 class ConvMode:
     encoding = 'ascii'
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -229,8 +229,8 @@
                     return thing
                 elif hasattr(thing, '__name__'): # mostly types and functions
                     return thing.__name__
-                elif hasattr(thing, 'name'): # mostly ClassDescs
-                    return thing.name
+                elif hasattr(thing, 'name') and isinstance(thing.name, str):
+                    return thing.name            # mostly ClassDescs
                 elif isinstance(thing, tuple):
                     return '_'.join(map(nameof, thing))
                 else:
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/cppyy.rst
@@ -0,0 +1,333 @@
+============================
+cppyy: C++ bindings for PyPy
+============================
+
+The cppyy module provides C++ bindings for PyPy by using the reflection
+information extracted from C++ header files by means of the
+`Reflex package`_.
+For this to work, you have to both install Reflex and build PyPy from the
+reflex-support branch.
+As indicated by this being a branch, support for Reflex is still
+experimental.
+However, it is functional enough to put it in the hands of those who want
+to give it a try.
+In the medium term, cppyy will move away from Reflex and instead use
+`cling`_ as its backend, which is based on `llvm`_.
+Although that will change the logistics on the generation of reflection
+information, it will not change the python-side interface.
+
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`cling`: http://root.cern.ch/drupal/content/cling
+.. _`llvm`: http://llvm.org/
+
+
+Installation
+============
+
+For now, the easiest way of getting the latest version of Reflex, is by
+installing the ROOT package.
+Besides getting the latest version of Reflex, another advantage is that with
+the full ROOT package, you can also use your Reflex-bound code on `CPython`_.
+`Download`_ a binary or install from `source`_.
+Some Linux and Mac systems may have ROOT provided in the list of scientific
+software of their packager.
+A current, standalone version of Reflex should be provided at some point,
+once the dependencies and general packaging have been thought out.
+Also, make sure you have a version of `gccxml`_ installed, which is most
+easily provided by the packager of your system.
+If you read up on gccxml, you'll probably notice that it is no longer being
+developed and hence will not provide C++11 support.
+That's why the medium term plan is to move to `cling`_.
+
+.. _`Download`: http://root.cern.ch/drupal/content/downloading-root
+.. _`source`: http://root.cern.ch/drupal/content/installing-root-source
+.. _`gccxml`: http://www.gccxml.org
+
+Next, get the `PyPy sources`_, select the reflex-support branch, and build
+pypy-c.
+For the build to succeed, the ``$ROOTSYS`` environment variable must point to
+the location of your ROOT installation::
+
+    $ hg clone https://bitbucket.org/pypy/pypy
+    $ cd pypy
+    $ hg up reflex-support
+    $ cd pypy/translator/goal
+    $ python translate.py -O jit --gcrootfinder=shadowstack targetpypystandalone.py --withmod-cppyy
+
+This will build a ``pypy-c`` that includes the cppyy module, and through that,
+Reflex support.
+Of course, if you already have a pre-built version of the ``pypy`` interpreter,
+you can use that for the translation rather than ``python``.
+
+.. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
+
+
+Basic example
+=============
+
+Now test with a trivial example whether all packages are properly installed
+and functional.
+First, create a C++ header file with some class in it (note that all functions
+are made inline for convenience; a real-world example would of course have a
+corresponding source file)::
+
+    $ cat MyClass.h
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_myint(i) {}
+
+        int GetMyInt() { return m_myint; }
+        void SetMyInt(int i) { m_myint = i; }
+
+    public:
+       int m_myint;
+    };
+
+Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
+code::
+
+    $ genreflex MyClass.h
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+
+Now you're ready to use the bindings.
+Since the bindings are designed to look pythonistic, it should be
+straightforward::
+
+    $ pypy-c
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libMyClassDict.so")
+    <CPPLibrary object at 0xb6fd7c4c>
+    >>>> myinst = cppyy.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> myinst.SetMyInt(33)
+    >>>> print myinst.m_myint
+    33
+    >>>> myinst.m_myint = 77
+    >>>> print myinst.GetMyInt()
+    77
+    >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
+
+That's all there is to it!
+
+
+Features
+========
+
+The following is not meant to be an exhaustive list, since cppyy is still
+under active development.
+Furthermore, the intention is that every feature is as natural as possible on
+the python side, so if you find something missing in the list below, simply
+try it out.
+It is not always possible to provide exact mapping between python and C++
+(active memory management is one such case), but by and large, if the use of a
+feature does not strike you as obvious, it is more likely to simply be a bug.
+That is a strong statement to make, but also a worthy goal.
+
+* **abstract classes**: Are represented as python classes, since they are
+  needed to complete the inheritance hierarchies, but will raise an exception
+  if an attempt is made to instantiate from them.
+
+* **arrays**: Supported for builtin data types only, as used from module
+  ``array``.
+  Out-of-bounds checking is limited to those cases where the size is known at
+  compile time (and hence part of the reflection info).
+
+* **builtin data types**: Map onto the expected equivalent python types, with
+  the caveat that there may be size differences, and thus it is possible that
+  exceptions are raised if an overflow is detected.
+
+* **casting**: Is supposed to be unnecessary.
+  Object pointer returns from functions provide the most derived class known
+  in the hierarchy of the object being returned.
+  This is important to preserve object identity as well as to make casting,
+  a pure C++ feature after all, superfluous.
+
+* **classes and structs**: Get mapped onto python classes, where they can be
+  instantiated as expected.
+  If classes are inner classes or live in a namespace, their naming and
+  location will reflect that.
+
+* **data members**: Public data members are represented as python properties
+  and provide read and write access on instances as expected.
+
+* **default arguments**: C++ default arguments work as expected, but python
+  keywords are not supported.
+  It is technically possible to support keywords, but for the C++ interface,
+  the formal argument names have no meaning and are not considered part of the
+  API, hence it is not a good idea to use keywords.
+
+* **doc strings**: The doc string of a method or function contains the C++
+  arguments and return types of all overloads of that name, as applicable.
+
+* **functions**: Work as expected and live in their appropriate namespace
+  (which can be the global one, ``cppyy.gbl``).
+
+* **inheritance**: All combinations of inheritance on the C++ (single,
+  multiple, virtual) are supported in the binding.
+  However, new python classes can only use single inheritance from a bound C++
+  class.
+  Multiple inheritance would introduce two "this" pointers in the binding.
+  This is a current, not a fundamental, limitation.
+  The C++ side will not see any overridden methods on the python side, as
+  cross-inheritance is planned but not yet supported.
+
+* **methods**: Are represented as python methods and work as expected.
+  They are first class objects and can be bound to an instance.
+  Virtual C++ methods work as expected.
+  To select a specific virtual method, do like with normal python classes
+  that override methods: select it from the class that you need, rather than
+  calling the method on the instance.
+
+* **namespaces**: Are represented as python classes.
+  Namespaces are more open-ended than classes, so sometimes initial access may
+  result in updates as data and functions are looked up and constructed
+  lazily.
+  Thus the result of ``dir()`` on a namespace should not be relied upon: it
+  only shows the already accessed members. (TODO: to be fixed by implementing
+  __dir__.)
+  The global namespace is ``cppyy.gbl``.
+
+* **operator conversions**: If defined in the C++ class and a python
+  equivalent exists (i.e. all builtin integer and floating point types, as well
+  as ``bool``), it will map onto that python conversion.
+  Note that ``char*`` is mapped onto ``__str__``.
+
+* **operator overloads**: If defined in the C++ class and if a python
+  equivalent is available (not always the case, think e.g. of ``operator||``),
+  then they work as expected.
+  Special care needs to be taken for global operator overloads in C++: first,
+  make sure that they are actually reflected, especially for the global
+  overloads for ``operator==`` and ``operator!=`` of STL iterators in the case
+  of gcc.
+  Second, make sure that reflection info is loaded in the proper order.
+  I.e. that these global overloads are available before use.
+
+* **pointers**: For builtin data types, see arrays.
+  For objects, a pointer to an object and an object looks the same, unless
+  the pointer is a data member.
+  In that case, assigning to the data member will cause a copy of the pointer
+  and care should be taken about the object's life time.
+  If a pointer is a global variable, the C++ side can replace the underlying
+  object and the python side will immediately reflect that.
+
+* **static data members**: Are represented as python property objects on the
+  class and the meta-class.
+  Both read and write access is as expected.
+
+* **static methods**: Are represented as python's ``staticmethod`` objects
+  and can be called both from the class as well as from instances.
+
+* **strings**: The std::string class is considered a builtin C++ type and
+  mixes quite well with python's str.
+  Python's str can be passed where a ``const char*`` is expected, and an str
+  will be returned if the return type is ``const char*``.
+
+* **templated classes**: Are represented in a meta-class style in python.
+  This looks a little bit confusing, but conceptually is rather natural.
+  For example, given the class ``std::vector<int>``, the meta-class part would
+  be ``std.vector`` in python.
+  Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
+  create an instance of that class, do ``std.vector(int)()``.
+  Note that templates can be build up by handing actual types to the class
+  instantiation (as done in this vector example), or by passing in the list of
+  template arguments as a string.
+  The former is a lot easier to work with if you have template instantiations
+  using classes that themselves are templates (etc.) in the arguments.
+  All classes must already exist in the loaded reflection info.
+
+* **unary operators**: Are supported if a python equivalent exists, and if the
+  operator is defined in the C++ class.
+
+You can always find more detailed examples and see the full of supported
+features by looking at the tests in pypy/module/cppyy/test.
+
+If a feature or reflection info is missing, this is supposed to be handled
+gracefully.
+In fact, there are unit tests explicitly for this purpose (even as their use
+becomes less interesting over time, as the number of missing features
+decreases).
+Only when a missing feature is used, should there be an exception.
+For example, if no reflection info is available for a return type, then a
+class that has a method with that return type can still be used.
+Only that one specific method can not be used.
+
+
+The fast lane
+=============
+
+The following is an experimental feature of cppyy, and that makes it doubly
+experimental, so caveat emptor.
+With a slight modification of Reflex, it can provide function pointers for
+C++ methods, and hence allow PyPy to call those pointers directly, rather than
+calling C++ through a Reflex stub.
+This results in a rather significant speed-up.
+Mind you, the normal stub path is not exactly slow, so for now only use this
+out of curiosity or if you really need it.
+
+To install this patch of Reflex, locate the file genreflex-methptrgetter.patch
+in pypy/module/cppyy and apply it to the genreflex python scripts found in
+``$ROOTSYS/lib``::
+
+    $ cd $ROOTSYS/lib
+    $ patch -p2 < genreflex-methptrgetter.patch
+
+With this patch, ``genreflex`` will have grown the ``--with-methptrgetter``
+option.
+Use this option when running ``genreflex``, and add the
+``-Wno-pmf-conversions`` option to ``g++`` when compiling.
+The rest works the same way: the fast path will be used transparently (which
+also means that you can't actually find out whether it is in use, other than
+by running a micro-benchmark).
+
+
+CPython
+=======
+
+Most of the ideas in cppyy come originally from the `PyROOT`_ project.
+Although PyROOT does not support Reflex directly, it has an alter ego called
+"PyCintex" that, in a somewhat roundabout way, does.
+If you installed ROOT, rather than just Reflex, PyCintex should be available
+immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment
+variable.
+
+.. _`PyROOT`: http://root.cern.ch/drupal/content/pyroot
+
+There are a couple of minor differences between PyCintex and cppyy, most to do
+with naming.
+The one that you will run into directly, is that PyCintex uses a function
+called ``loadDictionary`` rather than ``load_reflection_info``.
+The reason for this is that Reflex calls the shared libraries that contain
+reflection info "dictionaries."
+However, in python, the name `dictionary` already has a well-defined meaning,
+so a more descriptive name was chosen for cppyy.
+In addition, PyCintex requires that the names of shared libraries so loaded
+start with "lib" in their name.
+The basic example above, rewritten for PyCintex thus goes like this::
+
+    $ python
+    >>> import PyCintex
+    >>> PyCintex.loadDictionary("libMyClassDict.so")
+    >>> myinst = PyCintex.gbl.MyClass(42)
+    >>> print myinst.GetMyInt()
+    42
+    >>> myinst.SetMyInt(33)
+    >>> print myinst.m_myint
+    33
+    >>> myinst.m_myint = 77
+    >>> print myinst.GetMyInt()
+    77
+    >>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
+
+Other naming differences are such things as taking an address of an object.
+In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
+made to follow the naming as in ``ctypes`` and hence use ``addressof``
+(PyROOT/PyCintex predate ``ctypes`` by several years, and the ROOT project
+follows camel-case, hence the differences).
+
+Of course, this is python, so if any of the naming is not to your liking, all
+you have to do is provide a wrapper script that you import instead of
+importing the ``cppyy`` or ``PyCintex`` modules directly.
+In that wrapper script you can rename methods exactly the way you need it.
+
+In the cling world, all these differences will be resolved.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -158,6 +158,12 @@
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
+Note that this difference might show up indirectly in some cases.  For
+example, a generator left pending in the middle is --- again ---
+garbage-collected later in PyPy than in CPython.  You can see the
+difference if the ``yield`` keyword it is suspended at is itself
+enclosed in a ``try:`` or a ``with:`` block.
+
 Using the default GC called ``minimark``, the built-in function ``id()``
 works like it does in CPython.  With other GCs it returns numbers that
 are not real addresses (because an object can move around several times)
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -23,6 +23,8 @@
 
 * Write them in RPython as mixedmodule_, using *rffi* as bindings.
 
+* Write them in C++ and bind them through Reflex_ (EXPERIMENTAL)
+
 .. _ctypes: #CTypes
 .. _\_ffi: #LibFFI
 .. _mixedmodule: #Mixed Modules
@@ -110,3 +112,34 @@
 
 XXX we should provide detailed docs about lltype and rffi, especially if we
     want people to follow that way.
+
+Reflex
+======
+
+This method is only experimental for now, and is being exercised on a branch,
+`reflex-support`_, so you will have to build PyPy yourself.
+The method works by using the `Reflex package`_ to provide reflection
+information of the C++ code, which is then used to automatically generate
+bindings at runtime, which can then be used from python.
+Full details are `available here`_.
+
+.. _`reflex-support`: cppyy.html
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`available here`: cppyy.html
+
+Pros
+----
+
+If it works, it is mostly automatic, and hence easy in use.
+The bindings can make use of direct pointers, in which case the calls are
+very fast.
+
+Cons
+----
+
+C++ is a large language, and these bindings are not yet feature-complete.
+Although missing features should do no harm if you don't use them, if you do
+need a particular feature, it may be necessary to work around it in python
+or with a C++ helper function.
+Although Reflex works on various platforms, the bindings with PyPy have only
+been tested on Linux.
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -169,9 +173,17 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +198,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +236,20 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +408,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -49,7 +49,9 @@
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
         # maybe we want this shorter:
-        name = getattr(self, 'name', '?')
+        name = getattr(self, 'name', None)
+        if not isinstance(name, str):
+            name = '?'
         return "<%s %s>" % (self.__class__.__name__, name)
 
     def call_args(self, args):
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1656,15 +1656,21 @@
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
-            #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower 16 bits read there
-            #   - multiply by 4 and use it as an offset in type_info_group
-            #   - add 16 bytes, to go past the TYPE_INFO structure
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0;
+            #     this is a complete word (N=4 bytes on 32-bit, N=8 on
+            #     64-bits)
+            #   - keep the lower half of what is read there (i.e.
+            #     truncate to an unsigned 'N / 2' bytes value)
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
             loc = locs[1]
             assert isinstance(loc, ImmedLoc)
             classptr = loc.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the 16 bits in the object header
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 or 4 on a big-endian machine.
             from pypy.rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,10 +453,22 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        # prefix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        # suffix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        # type checks
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+        self.raises(space, api, TypeError,
+                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
+                    2, 10, 1)
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")
@@ -479,6 +491,8 @@
                 api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
         assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
                 api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
         assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
                 api.PyUnicode_Splitlines(w_str, 0)))
         assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
@@ -630,6 +630,8 @@
     Otherwise, splits occur at the given separator.  At most maxsplit
     splits will be done.  If negative, no limit is set.  Separators
     are not included in the resulting list."""
+    if w_sep is None:
+        w_sep = space.w_None
     return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
 
 @cpython_api([PyObject, rffi.INT_real], PyObject)
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
@@ -30,6 +31,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, space, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -513,7 +514,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1436,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
@@ -1559,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,16 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
@@ -1410,6 +1409,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,6 @@
 
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_ufunc_instance(self):
         from _numpypy import add, ufunc
@@ -149,7 +148,11 @@
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+        # Since comparisons with nnan and nan all return false,
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
+
 
     def test_fmin(self):
         from _numpypy import fmin
@@ -165,7 +168,9 @@
         assert math.isnan(fmin(0, nan))
         assert math.isnan(fmin(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
     def test_fmod(self):
         from _numpypy import fmod
@@ -762,6 +767,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -772,7 +779,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -785,6 +793,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -796,7 +806,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -596,7 +596,7 @@
         import sys
         size = 0x14FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
@@ -618,7 +618,7 @@
         import sys
         size = 0x17FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/module/pypyjit/test_pypy_c/test_exception.py b/pypy/module/pypyjit/test_pypy_c/test_exception.py
--- a/pypy/module/pypyjit/test_pypy_c/test_exception.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_exception.py
@@ -91,3 +91,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_continue_in_finally(self):
+        # check that 'continue' inside a try:finally: block is correctly
+        # detected as closing a loop
+        py.test.skip("is this case important?")
+        def f(n):
+            i = 0
+            while 1:
+                try:
+                    if i < n:
+                        continue
+                finally:
+                    i += 1
+                return i
+
+        log = self.run(f, [2000])
+        assert log.result == 2001
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i40 = int_add_ovf(i31, 1)
+            guard_no_overflow(descr=...)
+            i41 = int_lt(i40, i33)
+            guard_true(i41, descr=...)
+            --TICK--
+            jump(..., descr=...)
+        """)
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -378,7 +378,10 @@
     __slots__ = ()
 
     def __repr__(self):
-        s = '%s(%s)' % (self.__class__.__name__, getattr(self, 'name', ''))
+        name = getattr(self, 'name', '')
+        if not isinstance(name, str):
+            name = ''
+        s = '%s(%s)' % (self.__class__.__name__, name)
         w_cls = getattr(self, 'w__class__', None)
         if w_cls is not None and w_cls is not self:
             s += ' instance of %s' % self.w__class__
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,12 +295,11 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,31 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        res = 'bar' in x
+        assert res is True
+        #
+        class MyError(Exception):
+            pass
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise MyError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(MyError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -204,6 +204,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.call_location()
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -40,7 +40,7 @@
 # In that case, do 5 bits at a time.  The potential drawback is that
 # a table of 2**5 intermediate results is computed.
 
-## FIVEARY_CUTOFF = 8   disabled for now
+FIVEARY_CUTOFF = 8
 
 
 def _mask_digit(x):
@@ -456,7 +456,7 @@
 
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
-        if 1:   ## b.numdigits() <= FIVEARY_CUTOFF:
+        if b.numdigits() <= FIVEARY_CUTOFF:
             # Left-to-right binary exponentiation (HAC Algorithm 14.79)
             # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
             i = b.numdigits() - 1
@@ -469,30 +469,51 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 i -= 1
-##        else:
-##            This code is disabled for now, because it assumes that
-##            SHIFT is a multiple of 5.  It could be fixed but it looks
-##            like it's more troubles than benefits...
-##
-##            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-##            # This is only useful in the case where c != None.
-##            # z still holds 1L
-##            table = [z] * 32
-##            table[0] = z
-##            for i in range(1, 32):
-##                table[i] = _help_mult(table[i-1], a, c)
-##            i = b.numdigits() - 1
-##            while i >= 0:
-##                bi = b.digit(i)
-##                j = SHIFT - 5
-##                while j >= 0:
-##                    index = (bi >> j) & 0x1f
-##                    for k in range(5):
-##                        z = _help_mult(z, z, c)
-##                    if index:
-##                        z = _help_mult(z, table[index], c)
-##                    j -= 5
-##                i -= 1
+        else:
+            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
+            # This is only useful in the case where c != None.
+            # z still holds 1L
+            table = [z] * 32
+            table[0] = z
+            for i in range(1, 32):
+                table[i] = _help_mult(table[i-1], a, c)
+            i = b.numdigits()
+            # Note that here SHIFT is not a multiple of 5.  The difficulty
+            # is to extract 5 bits at a time from 'b', starting from the
+            # most significant digits, so that at the end of the algorithm
+            # it falls exactly to zero.
+            # m  = max number of bits = i * SHIFT
+            # m+ = m rounded up to the next multiple of 5
+            # j  = (m+) % SHIFT = (m+) - (i * SHIFT)
+            # (computed without doing "i * SHIFT", which might overflow)
+            j = i % 5
+            if j != 0:
+                j = 5 - j
+            if not we_are_translated():
+                assert j == (i*SHIFT+4)//5*5 - i*SHIFT
+            #
+            accum = r_uint(0)
+            while True:
+                j -= 5
+                if j >= 0:
+                    index = (accum >> j) & 0x1f
+                else:
+                    # 'accum' does not have enough digit.
+                    # must get the next digit from 'b' in order to complete
+                    i -= 1
+                    if i < 0:
+                        break    # done
+                    bi = b.udigit(i)
+                    index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f
+                    accum = bi
+                    j += SHIFT
+                #
+                for k in range(5):
+                    z = _help_mult(z, z, c)
+                if index:
+                    z = _help_mult(z, table[index], c)
+            #
+            assert j == -5
 
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,10 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +19,18 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +42,8 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint.py
@@ -379,6 +379,18 @@
         for n, expected in [(37, 9), (1291, 931), (67889, 39464)]:
             v = two.pow(t, rbigint.fromint(n))
             assert v.toint() == expected
+        #
+        # more tests, comparing against CPython's answer
+        enabled = sample(range(5*32), 10)
+        for i in range(5*32):
+            t = t.mul(two)      # add one random bit
+            if random() >= 0.5:
+                t = t.add(rbigint.fromint(1))
+            if i not in enabled:
+                continue    # don't take forever
+            n = randint(1, sys.maxint)
+            v = two.pow(t, rbigint.fromint(n))
+            assert v.toint() == pow(2, t.tolong(), n)
 
     def test_pow_lln(self):
         x = 10L
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1168,8 +1168,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct
diff --git a/pypy/translator/c/src/exception.h b/pypy/translator/c/src/exception.h
--- a/pypy/translator/c/src/exception.h
+++ b/pypy/translator/c/src/exception.h
@@ -43,6 +43,16 @@
           filename, lineno, functionname);
 }
 #endif
+#else   /* !DO_LOG_EXC: define the function anyway, so that we can shut
+           off the prints of a debug_exc by remaking only testing_1.o */
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname);
+#ifndef PYPY_NOT_MAIN_FILE
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname)
+{
+}
+#endif
 #endif  /* DO_LOG_EXC */
 
 /* Hint: functions and macros not defined here, like RPyRaiseException,
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Sat Apr 21 20:54:56 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 21 Apr 2012 20:54:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: try harder to find export lib
Message-ID: <20120421185456.C760D46E1E4@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54610:92cfbb56d39e
Date: 2012-04-21 21:52 +0300
http://bitbucket.org/pypy/pypy/changeset/92cfbb56d39e/

Log:	try harder to find export lib

diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -21,7 +21,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC']
     res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
@@ -34,6 +34,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST'] # needed for VC10
     else:
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -16,7 +16,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
     res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
@@ -29,6 +29,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
                         '/EXPORT:init_testcapi']

From noreply at buildbot.pypy.org  Sun Apr 22 12:17:22 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 12:17:22 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Move these too.
Message-ID: <20120422101722.70A248203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54611:b3728c83978e
Date: 2012-04-21 14:49 +0200
http://bitbucket.org/pypy/pypy/changeset/b3728c83978e/

Log:	Move these too.

diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -100,7 +100,6 @@
 
 @rgc.no_collect     # don't use the gc as long as other threads are running
 def _run():
-    rstm.enter_transactional_mode()
     i = 0
     while i < glob.NUM_THREADS:
         glob._arg = glob._arglist[i]
@@ -112,7 +111,6 @@
     while glob.done < glob.NUM_THREADS:    # poor man's lock
         _sleep(rffi.cast(rffi.ULONG, 1))
     debug_print("done sleeping.")
-    rstm.leave_transactional_mode()
 
 
 # Posix only
@@ -135,7 +133,11 @@
     glob.lock = ll_thread.allocate_ll_lock()
     ll_thread.acquire_NOAUTO(glob.lock, True)
     glob._arglist = [Arg() for i in range(glob.NUM_THREADS)]
+    #
+    rstm.enter_transactional_mode()
     _run()
+    rstm.leave_transactional_mode()
+    #
     check_chained_list(glob.anchor.next)
     return 0
 

From noreply at buildbot.pypy.org  Sun Apr 22 12:17:23 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 12:17:23 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: A failing test.
Message-ID: <20120422101723.A862E8204C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54612:a9866b7ee8ed
Date: 2012-04-21 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/a9866b7ee8ed/

Log:	A failing test.

diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py
--- a/pypy/translator/stm/test/test_ztranslated.py
+++ b/pypy/translator/stm/test/test_ztranslated.py
@@ -47,3 +47,30 @@
         t, cbuilder = self.compile(entry_point, backendopt=True)
         data = cbuilder.cmdexec('a b c d')
         assert '< 5 1000 >' in data, "got: %r" % (data,)
+
+    def test_bug2(self):
+        from pypy.rlib import rstm
+        #
+        class X2:
+            pass
+        prebuilt2 = [X2(), X2()]
+        #
+        def bug2(count):
+            x = prebuilt2[count]
+            x.foobar = 2                    # 'x' becomes a local
+            #
+            rstm.enter_transactional_mode() # 'x' becomes the global again
+            rstm.leave_transactional_mode()
+            #
+            y = prebuilt2[count]            # same prebuilt obj
+            y.foobar += 10                  # 'y' becomes a local
+            return x.foobar                 # read from the global, thinking
+        bug2._dont_inline_ = True           #    that it is still a local
+        def entry_point(argv):
+            print bug2(0)
+            print bug2(1)
+            return 0
+        #
+        t, cbuilder = self.compile(entry_point, backendopt=True)
+        data = cbuilder.cmdexec('')
+        assert '12\n12\n' in data, "got: %r" % (data,)

From noreply at buildbot.pypy.org  Sun Apr 22 12:17:24 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 12:17:24 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Start to refactor the RPython interface
 to be higher-level. This should
Message-ID: <20120422101724.E7E368203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54613:9290ce3f4d40
Date: 2012-04-22 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/9290ce3f4d40/

Log:	Start to refactor the RPython interface to be higher-level. This
	should make it easier to call directly C code from the higher-level
	code without passing through lower-level interfaces. No need to
	access from RPython the ll_thread module with locks and so on; this
	should all be done in C.

	Untested code so far (it doesn't run at all, missing levels, but I'd
	like to add proper testing anyway).

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -1,79 +1,146 @@
-import threading
-from pypy.rlib.objectmodel import specialize, we_are_translated
+from pypy.rpython.lltypesystem import lltype, llmemory, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.rpython.annlowlevel import llhelper, cast_instance_to_base_ptr
+from pypy.rpython.annlowlevel import base_ptr_lltype, cast_base_ptr_to_instance
 from pypy.rlib.objectmodel import keepalive_until_here
-from pypy.rlib.debug import ll_assert
-from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.annlowlevel import (cast_base_ptr_to_instance,
-                                      cast_instance_to_base_ptr,
-                                      llhelper)
 from pypy.translator.stm.stmgcintf import StmOperations
 
-_global_lock = threading.RLock()
 
- at specialize.memo()
-def _get_stm_callback(func, argcls):
-    def _stm_callback(llarg, retry_counter):
-        llop.stm_start_transaction(lltype.Void)
-        if we_are_translated():
-            llarg = rffi.cast(rclass.OBJECTPTR, llarg)
-            arg = cast_base_ptr_to_instance(argcls, llarg)
+
+NUM_THREADS_DEFAULT = 4     # XXX for now
+
+
+class TransactionError(Exception):
+    pass
+
+class Transaction(object):
+    _next_transaction = None
+    retry_counter = 0
+
+    def run(self):
+        raise NotImplementedError
+
+
+def run_all_transactions(initial_transaction,
+                         num_threads = NUM_THREADS_DEFAULT):
+    if StmOperations.in_transaction():
+        raise TransactionError("nested call to rstm.run_all_transactions()")
+    #
+    _transactionalstate.initialize()
+    #
+    # Tell the GC we are entering transactional mode.  This makes
+    # sure that 'initial_transaction' is flagged as GLOBAL.
+    # No more GC operation afterwards!
+    llop.stm_enter_transactional_mode(lltype.Void)
+    #
+    # Keep alive 'initial_transaction'.  In truth we would like it to
+    # survive a little bit longer, for the beginning of the C code in
+    # run_all_transactions().  This should be equivalent because there
+    # is no possibility of having a GC collection inbetween.
+    keepalive_until_here(initial_transaction)
+    #
+    # Tell the C code to run all transactions.
+    callback = llhelper(_CALLBACK, _run_transaction)
+    ptr = _cast_transaction_to_voidp(initial_transaction)
+    StmOperations.run_all_transactions(callback, ptr, num_threads)
+    #
+    # Tell the GC we are leaving transactional mode.
+    llop.stm_leave_transactional_mode(lltype.Void)
+    #
+    # If an exception was raised, re-raise it here.
+    _transactionalstate.close_exceptions()
+
+
+_CALLBACK = lltype.Ptr(lltype.FuncType([rffi.VOIDP, lltype.Signed],
+                                       rffi.VOIDP))
+
+def _cast_transaction_to_voidp(transaction):
+    ptr = cast_instance_to_base_ptr(transaction)
+    return lltype.cast_pointer(rffi.VOIDP, ptr)
+
+def _cast_voidp_to_transaction(transactionptr):
+    ptr = lltype.cast_pointer(base_ptr_lltype(), transactionptr)
+    return cast_base_ptr_to_instance(Transaction, ptr)
+
+
+class _TransactionalState(object):
+    def initialize(self):
+        self._reraise_exception = None
+
+    def has_exception(self):
+        return self._reraise_exception is not None
+
+    def must_reraise_exception(self, got_exception):
+        self._got_exception = got_exception
+        self._reraise_exception = self.reraise_exception_callback
+
+    def close_exceptions(self):
+        if self._reraise_exception is not None:
+            self._reraise_exception()
+
+    @staticmethod
+    def reraise_exception_callback():
+        exc = _transactionalstate._got_exception
+        self._got_exception = None
+        raise exc
+
+_transactionalstate = _TransactionalState()
+
+
+def _run_transaction(transactionptr, retry_counter):
+    #
+    # Tell the GC we are starting a transaction
+    llop.stm_start_transaction(lltype.Void)
+    #
+    # Now we can use the GC
+    next = None
+    try:
+        if _transactionalstate.has_exception():
+            # a previously committed transaction raised: don't do anything
+            # more in this transaction
+            pass
         else:
-            arg = lltype.TLS.stm_callback_arg
-        try:
-            res = func(arg, retry_counter)
-            ll_assert(res is None, "stm_callback should return None")
-        finally:
-            llop.stm_commit_transaction(lltype.Void)
-        return lltype.nullptr(rffi.VOIDP.TO)
-    return _stm_callback
+            # run!
+            next = _run_really(transactionptr, retry_counter)
+        #
+    except Exception, e:
+        _transactionalstate.must_reraise_exception(e)
+    #
+    # Stop using the GC.  This will make 'next' and all transactions linked
+    # from there GLOBAL objects.
+    llop.stm_stop_transaction(lltype.Void)
+    #
+    # Mark 'next' as kept-alive-until-here.  In truth we would like to
+    # keep it alive after the return, for the C code.  This should be
+    # equivalent because there is no possibility of having a GC collection
+    # inbetween.
+    keepalive_until_here(next)
+    return _cast_transaction_to_voidp(next)
 
- at specialize.arg(0, 1)
-def perform_transaction(func, argcls, arg):
-    ll_assert(arg is None or isinstance(arg, argcls),
-              "perform_transaction: wrong class")
-    if we_are_translated():
-        llarg = cast_instance_to_base_ptr(arg)
-        llarg = rffi.cast(rffi.VOIDP, llarg)
-        adr_of_top = llop.gc_adr_of_root_stack_top(llmemory.Address)
-    else:
-        # only for tests: we want (1) to test the calls to the C library,
-        # but also (2) to work with multiple Python threads, so we acquire
-        # and release some custom GIL here --- even though it doesn't make
-        # sense from an STM point of view :-/
-        _global_lock.acquire()
-        lltype.TLS.stm_callback_arg = arg
-        llarg = lltype.nullptr(rffi.VOIDP.TO)
-        adr_of_top = llmemory.NULL
-    #
-    callback = _get_stm_callback(func, argcls)
-    llcallback = llhelper(StmOperations.CALLBACK_TX, callback)
-    StmOperations.perform_transaction(llcallback, llarg, adr_of_top)
-    keepalive_until_here(arg)
-    if not we_are_translated():
-        _global_lock.release()
 
-def enter_transactional_mode():
-    llop.stm_enter_transactional_mode(lltype.Void)
+def _run_really(transactionptr, retry_counter):
+    # Call the RPython method run() on the Transaction instance.
+    # This logic is in a sub-function because we want to catch
+    # the MemoryErrors that could occur.
+    transaction = _cast_voidp_to_transaction(transactionptr)
+    ll_assert(transaction._next_transaction is None,
+              "_next_transaction should be cleared by C code")
+    transaction.retry_counter = retry_counter
+    new_transactions = transaction.run()
+    return _link_new_transactions(new_transactions)
+_run_really._dont_inline_ = True
 
-def leave_transactional_mode():
-    llop.stm_leave_transactional_mode(lltype.Void)
-
-def descriptor_init():
-    if not we_are_translated(): _global_lock.acquire()
-    llop.stm_descriptor_init(lltype.Void)
-    if not we_are_translated(): _global_lock.release()
-
-def descriptor_done():
-    if not we_are_translated(): _global_lock.acquire()
-    llop.stm_descriptor_done(lltype.Void)
-    if not we_are_translated(): _global_lock.release()
-
-def _debug_get_state():
-    if not we_are_translated(): _global_lock.acquire()
-    res = StmOperations._debug_get_state()
-    if not we_are_translated(): _global_lock.release()
-    return res
-
-def thread_id():
-    return StmOperations.thread_id()
+def _link_new_transactions(new_transactions):
+    # in order to schedule the new transactions, we have to return a
+    # raw pointer to the first one, with their field '_next_transaction'
+    # making a linked list.  The C code reads directly from this
+    # field '_next_transaction'.
+    if new_transactions is None:
+        return None
+    n = len(new_transactions) - 1
+    next = None
+    while n >= 0:
+        new_transactions[n]._next_transaction = next
+        next = new_transactions[n]
+        n -= 1
+    return next
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -1,8 +1,6 @@
-from pypy.rpython.lltypesystem import lltype, rffi
-from pypy.module.thread import ll_thread
-from pypy.rlib import rstm, rgc
-from pypy.rlib.debug import debug_print
-from pypy.rpython.annlowlevel import llhelper
+from pypy.rpython.lltypesystem import rffi
+from pypy.rlib import rstm
+from pypy.rlib.debug import debug_print, ll_assert
 
 
 class Node:
@@ -21,10 +19,7 @@
     pass
 
 
-def add_at_end_of_chained_list(arg, retry_counter):
-    assert arg.foobar == 42
-    node = arg.anchor
-    value = arg.value
+def add_at_end_of_chained_list(node, value):
     x = Node(value)
     while node.next:
         node = node.next
@@ -59,65 +54,40 @@
     print "check ok!"
 
 
-def increment_done(arg, retry_counter):
-    print "thread done."
-    glob.done += 1
-
 def _check_pointer(arg1):
     arg1.foobar = 40    # now 'arg1' is local
     return arg1
 
-def check_pointer_equality(arg, retry_counter):
-    res = _check_pointer(arg)
-    if res is not arg:
-        debug_print("ERROR: bogus pointer equality")
-        raise AssertionError
-    raw1 = rffi.cast(rffi.CCHARP, retry_counter)
-    raw2 = rffi.cast(rffi.CCHARP, -1)
-    if raw1 == raw2:
-        debug_print("ERROR: retry_counter == -1")
-        raise AssertionError
+class CheckPointerEquality(rstm.Transaction):
+    def __init__(self, arg):
+        self.arg = arg
+    def run(self):
+        res = _check_pointer(self.arg)    # 'self.arg' reads a GLOBAL object
+        ll_assert(res is self.arg, "ERROR: bogus pointer equality")
+        raw1 = rffi.cast(rffi.CCHARP, self.retry_counter)
+        raw2 = rffi.cast(rffi.CCHARP, -1)
+        ll_assert(raw1 == raw2, "ERROR: retry_counter == -1")
 
-def run_me():
-    rstm.descriptor_init()
-    try:
-        debug_print("thread starting...")
-        arg = glob._arg
-        ll_thread.release_NOAUTO(glob.lock)
-        arg.foobar = 41
-        rstm.perform_transaction(check_pointer_equality, Arg, arg)
-        i = 0
-        while i < glob.LENGTH:
-            arg.anchor = glob.anchor
-            arg.value = i
-            arg.foobar = 42
-            rstm.perform_transaction(add_at_end_of_chained_list, Arg, arg)
-            i += 1
-        rstm.perform_transaction(increment_done, Arg, arg)
-    finally:
-        rstm.descriptor_done()
+class MakeChain(rstm.Transaction):
+    def __init__(self, anchor, value):
+        self.anchor = anchor
+        self.value = value
+    def run(self):
+        add_at_end_of_chained_list(self.anchor, self.value)
+        self.value += 1
+        if self.value < glob.LENGTH:
+            return [self]       # re-schedule the same Transaction object
 
-
- at rgc.no_collect     # don't use the gc as long as other threads are running
-def _run():
-    i = 0
-    while i < glob.NUM_THREADS:
-        glob._arg = glob._arglist[i]
-        ll_run_me = llhelper(ll_thread.CALLBACK, run_me)
-        ll_thread.c_thread_start_NOGIL(ll_run_me)
-        ll_thread.acquire_NOAUTO(glob.lock, True)
-        i += 1
-    debug_print("sleeping...")
-    while glob.done < glob.NUM_THREADS:    # poor man's lock
-        _sleep(rffi.cast(rffi.ULONG, 1))
-    debug_print("done sleeping.")
-
-
-# Posix only
-_sleep = rffi.llexternal('sleep', [rffi.ULONG], rffi.ULONG,
-                         _nowrapper=True,
-                         random_effects_on_gcobjs=False)
-
+class InitialTransaction(rstm.Transaction):
+    def run(self):
+        ll_assert(self.retry_counter == 0, "no reason to abort-and-retry here")
+        scheduled = []
+        for i in range(glob.NUM_THREADS):
+            arg = Arg()
+            arg.foobar = 41
+            scheduled.append(CheckPointerEquality(arg))
+            scheduled.append(MakeChain(glob.anchor, 0))
+        return scheduled
 
 # __________  Entry point  __________
 
@@ -129,14 +99,9 @@
             glob.LENGTH = int(argv[2])
             if len(argv) > 3:
                 glob.USE_MEMORY = bool(int(argv[3]))
-    glob.done = 0
-    glob.lock = ll_thread.allocate_ll_lock()
-    ll_thread.acquire_NOAUTO(glob.lock, True)
-    glob._arglist = [Arg() for i in range(glob.NUM_THREADS)]
     #
-    rstm.enter_transactional_mode()
-    _run()
-    rstm.leave_transactional_mode()
+    rstm.run_all_transactions(InitialTransaction(),
+                              num_threads=glob.NUM_THREADS)
     #
     check_chained_list(glob.anchor.next)
     return 0
@@ -145,3 +110,7 @@
 
 def target(*args):
     return entry_point, None
+
+if __name__ == '__main__':
+    import sys
+    entry_point(sys.argv)

From noreply at buildbot.pypy.org  Sun Apr 22 12:32:04 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 22 Apr 2012 12:32:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: add where to compile
Message-ID: <20120422103204.2F0CD8203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54614:ee17da202bea
Date: 2012-04-22 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/ee17da202bea/

Log:	add where to compile

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -10,6 +10,7 @@
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
      scalar_w, W_NDimArray, array)
+from pypy.module.micronumpy.interp_arrayops import where
 from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize, instantiate
 
@@ -35,6 +36,7 @@
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring"]
 TWO_ARG_FUNCTIONS = ["dot", 'take']
+THREE_ARG_FUNCTIONS = ['where']
 
 class FakeSpace(object):
     w_ValueError = None
@@ -445,14 +447,25 @@
             arg = self.args[1].execute(interp)
             if not isinstance(arg, BaseArray):
                 raise ArgumentNotAnArray
-            if not isinstance(arg, BaseArray):
-                raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
             else:
                 assert False # unreachable code
+        elif self.name in THREE_ARG_FUNCTIONS:
+            if len(self.args) != 3:
+                raise ArgumentMismatch
+            arg1 = self.args[1].execute(interp)
+            arg2 = self.args[2].execute(interp)
+            if not isinstance(arg1, BaseArray):
+                raise ArgumentNotAnArray
+            if not isinstance(arg2, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "where":
+                w_res = where(interp.space, arr, arg1, arg2)
+            else:
+                assert False
         else:
             raise WrongFunctionName
         if isinstance(w_res, BaseArray):
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -270,3 +270,13 @@
         b -> 2
         """)
         assert interp.results[0].value == 3
+
+    def test_where(self):
+        interp = self.run('''
+        a = [1, 0, 3, 0]
+        b = [1, 1, 1, 1]
+        c = [0, 0, 0, 0]
+        d = where(a, b, c)
+        d -> 1
+        ''')
+        assert interp.results[0].value == 0

From noreply at buildbot.pypy.org  Sun Apr 22 12:56:41 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 22 Apr 2012 12:56:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: a passing test
Message-ID: <20120422105641.16AF98203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54615:034ec45e7799
Date: 2012-04-22 12:56 +0200
http://bitbucket.org/pypy/pypy/changeset/034ec45e7799/

Log:	a passing test

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1830,6 +1830,12 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
+    def test_array_indexing_bool_setitem_multidim(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])

From noreply at buildbot.pypy.org  Sun Apr 22 15:05:31 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 15:05:31 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: In-progress
Message-ID: <20120422130531.B4AC68203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54616:3b5630b98667
Date: 2012-04-22 15:05 +0200
http://bitbucket.org/pypy/pypy/changeset/3b5630b98667/

Log:	In-progress

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -2,7 +2,8 @@
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.annlowlevel import llhelper, cast_instance_to_base_ptr
 from pypy.rpython.annlowlevel import base_ptr_lltype, cast_base_ptr_to_instance
-from pypy.rlib.objectmodel import keepalive_until_here
+from pypy.rlib.objectmodel import keepalive_until_here, we_are_translated
+from pypy.rlib.debug import ll_assert
 from pypy.translator.stm.stmgcintf import StmOperations
 
 
@@ -21,15 +22,28 @@
         raise NotImplementedError
 
 
+def stm_operations():
+    if we_are_translated():
+        return StmOperations
+    else:
+        from pypy.rlib.test.test_rstm import fake_stm_operations
+        return fake_stm_operations
+
+
+def in_transaction():
+    return bool(stm_operations().in_transaction())
+
+
 def run_all_transactions(initial_transaction,
                          num_threads = NUM_THREADS_DEFAULT):
-    if StmOperations.in_transaction():
+    if in_transaction():
         raise TransactionError("nested call to rstm.run_all_transactions()")
     #
     _transactionalstate.initialize()
     #
     # Tell the GC we are entering transactional mode.  This makes
     # sure that 'initial_transaction' is flagged as GLOBAL.
+    # (Actually it flags all surviving objects as GLOBAL.)
     # No more GC operation afterwards!
     llop.stm_enter_transactional_mode(lltype.Void)
     #
@@ -39,31 +53,53 @@
     # is no possibility of having a GC collection inbetween.
     keepalive_until_here(initial_transaction)
     #
+    # The following line causes the _run_transaction() function to be
+    # generated in the C source with a specific signature, where it
+    # can be called by the C code.
+    llhelper(StmOperations.RUN_TRANSACTION, _run_transaction)
+    #
     # Tell the C code to run all transactions.
-    callback = llhelper(_CALLBACK, _run_transaction)
     ptr = _cast_transaction_to_voidp(initial_transaction)
-    StmOperations.run_all_transactions(callback, ptr, num_threads)
+    stm_operations().run_all_transactions(ptr, num_threads)
     #
     # Tell the GC we are leaving transactional mode.
     llop.stm_leave_transactional_mode(lltype.Void)
     #
+    # Hack
+    if not we_are_translated():
+        stm_operations().leaving()
+    #
     # If an exception was raised, re-raise it here.
     _transactionalstate.close_exceptions()
 
 
-_CALLBACK = lltype.Ptr(lltype.FuncType([rffi.VOIDP, lltype.Signed],
-                                       rffi.VOIDP))
-
 def _cast_transaction_to_voidp(transaction):
-    ptr = cast_instance_to_base_ptr(transaction)
-    return lltype.cast_pointer(rffi.VOIDP, ptr)
+    if we_are_translated():
+        ptr = cast_instance_to_base_ptr(transaction)
+        return rffi.cast(rffi.VOIDP, ptr)
+    else:
+        return stm_operations().cast_transaction_to_voidp(transaction)
 
 def _cast_voidp_to_transaction(transactionptr):
-    ptr = lltype.cast_pointer(base_ptr_lltype(), transactionptr)
-    return cast_base_ptr_to_instance(Transaction, ptr)
+    if we_are_translated():
+        ptr = rffi.cast(base_ptr_lltype(), transactionptr)
+        return cast_base_ptr_to_instance(Transaction, ptr)
+    else:
+        return stm_operations().cast_voidp_to_transaction(transactionptr)
 
 
 class _TransactionalState(object):
+    """This is the class of a global singleton, seen by every transaction.
+    Used for cross-transaction synchronization.  Of course writing to it
+    will likely cause conflicts.  Reserved for now for storing the
+    exception that must be re-raised by run_all_transactions().
+    """
+    # The logic ensures that once a transaction calls must_reraise_exception()
+    # and commits, all uncommitted transactions will abort (because they have
+    # read '_reraise_exception' when they started) and then, when they retry,
+    # do nothing.  This makes the transaction committing an exception the last
+    # one to commit, and it cleanly shuts down all other pending transactions.
+
     def initialize(self):
         self._reraise_exception = None
 
@@ -73,6 +109,8 @@
     def must_reraise_exception(self, got_exception):
         self._got_exception = got_exception
         self._reraise_exception = self.reraise_exception_callback
+        if not we_are_translated():
+            import sys; self._got_tb = sys.exc_info()[2]
 
     def close_exceptions(self):
         if self._reraise_exception is not None:
@@ -80,8 +118,11 @@
 
     @staticmethod
     def reraise_exception_callback():
-        exc = _transactionalstate._got_exception
+        self = _transactionalstate
+        exc = self._got_exception
         self._got_exception = None
+        if not we_are_translated() and hasattr(self, '_got_tb'):
+            raise exc.__class__, exc, self._got_tb
         raise exc
 
 _transactionalstate = _TransactionalState()
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -1,101 +1,90 @@
-import os, thread, time
-from pypy.rlib.debug import debug_print, ll_assert, fatalerror
+import random
+from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib import rstm
-from pypy.rpython.annlowlevel import llhelper
-from pypy.translator.stm.test.support import CompiledSTMTests
-from pypy.module.thread import ll_thread
 
 
-class Arg(object):
-    pass
-arg_list = [Arg() for i in range(10)]
+class FakeStmOperations:
+    _in_transaction = 0
+    _mapping = {}
 
-def setx(arg, retry_counter):
-    debug_print(arg.x)
-    assert rstm._debug_get_state() == 1
-    if arg.x == 303:
-        # this will trigger stm_become_inevitable()
-        os.write(1, "hello\n")
-        assert rstm._debug_get_state() == 2
-    arg.x = 42
+    def in_transaction(self):
+        return self._in_transaction
 
-def stm_perform_transaction(done=None, i=0):
-    ll_assert(rstm._debug_get_state() == -2, "bad debug_get_state (1)")
-    rstm.descriptor_init()
-    arg = arg_list[i]
-    if done is None:
-        arg.x = 202
-    else:
-        arg.x = done.initial_x
-    ll_assert(rstm._debug_get_state() == 0, "bad debug_get_state (2)")
-    rstm.perform_transaction(setx, Arg, arg)
-    ll_assert(rstm._debug_get_state() == 0, "bad debug_get_state (3)")
-    ll_assert(arg.x == 42, "bad arg.x")
-    if done is not None:
-        ll_thread.release_NOAUTO(done.finished_lock)
-    rstm.descriptor_done()
-    ll_assert(rstm._debug_get_state() == -2, "bad debug_get_state (4)")
+    def _add(self, transactionptr):
+        r = random.random()
+        assert r not in self._pending    # very bad luck if it is
+        self._pending[r] = transactionptr
 
-def test_stm_multiple_threads():
-    ok = []
-    def f(i):
-        stm_perform_transaction(i=i)
-        ok.append(i)
-    rstm.enter_transactional_mode()
-    for i in range(10):
-        thread.start_new_thread(f, (i,))
-    timeout = 10
-    while len(ok) < 10:
-        time.sleep(0.1)
-        timeout -= 0.1
-        assert timeout >= 0.0, "timeout!"
-    rstm.leave_transactional_mode()
-    assert sorted(ok) == range(10)
+    def run_all_transactions(self, initial_transaction_ptr, num_threads=4):
+        self._pending = {}
+        self._add(initial_transaction_ptr)
+        while self._pending:
+            r, transactionptr = self._pending.popitem()
+            transaction = self.cast_voidp_to_transaction(transactionptr)
+            transaction._next_transaction = None
+            nextptr = rstm._run_transaction(transactionptr, 0)
+            next = self.cast_voidp_to_transaction(nextptr)
+            while next is not None:
+                self._add(self.cast_transaction_to_voidp(next))
+                next = next._next_transaction
+        del self._pending
 
+    def cast_transaction_to_voidp(self, transaction):
+        if transaction is None:
+            return lltype.nullptr(rffi.VOIDP.TO)
+        assert isinstance(transaction, rstm.Transaction)
+        num = 10000 + len(self._mapping)
+        self._mapping[num] = transaction
+        return rffi.cast(rffi.VOIDP, num)
 
-class TestTransformSingleThread(CompiledSTMTests):
+    def cast_voidp_to_transaction(self, transactionptr):
+        if not transactionptr:
+            return None
+        num = rffi.cast(lltype.Signed, transactionptr)
+        return self._mapping[num]
 
-    def test_no_pointer_operations(self):
-        def simplefunc(argv):
-            i = 0
-            while i < 100:
-                i += 3
-            debug_print(i)
-            return 0
-        t, cbuilder = self.compile(simplefunc)
-        dataout, dataerr = cbuilder.cmdexec('', err=True)
-        assert dataout == ''
-        assert '102' in dataerr.splitlines()
+    def leaving(self):
+        self._mapping.clear()
 
-    def build_perform_transaction(self):
-        class Done: done = False
-        done = Done()
-        def g():
-            stm_perform_transaction(done)
-        def f(argv):
-            done.initial_x = int(argv[1])
-            assert rstm._debug_get_state() == -1    # main thread
-            done.finished_lock = ll_thread.allocate_ll_lock()
-            ll_thread.acquire_NOAUTO(done.finished_lock, True)
-            #
-            rstm.enter_transactional_mode()
-            #
-            llcallback = llhelper(ll_thread.CALLBACK, g)
-            ident = ll_thread.c_thread_start_NOGIL(llcallback)
-            ll_thread.acquire_NOAUTO(done.finished_lock, True)
-            #
-            rstm.leave_transactional_mode()
-            return 0
-        t, cbuilder = self.compile(f)
-        return cbuilder
+fake_stm_operations = FakeStmOperations()
 
-    def test_perform_transaction(self):
-        cbuilder = self.build_perform_transaction()
-        #
-        dataout, dataerr = cbuilder.cmdexec('202', err=True)
-        assert dataout == ''
-        assert '202' in dataerr.splitlines()
-        #
-        dataout, dataerr = cbuilder.cmdexec('303', err=True)
-        assert 'hello' in dataout.splitlines()
-        assert '303' in dataerr.splitlines()
+
+def test_in_transaction():
+    res = rstm.in_transaction()
+    assert res is False
+
+def test_run_all_transactions_minimal():
+    seen = []
+    class Empty(rstm.Transaction):
+        def run(self):
+            seen.append(42)
+    rstm.run_all_transactions(Empty())
+    assert seen == [42]
+
+def test_run_all_transactions_recursive():
+    seen = []
+    class DoInOrder(rstm.Transaction):
+        def run(self):
+            assert self._next_transaction is None
+            if len(seen) < 10:
+                seen.append(len(seen))
+                return [self]
+    rstm.run_all_transactions(DoInOrder())
+    assert seen == range(10)
+
+def test_run_all_transactions_random_order():
+    seen = []
+    class AddToSeen(rstm.Transaction):
+        def run(self):
+            seen.append(self.value)
+    class DoInOrder(rstm.Transaction):
+        count = 0
+        def run(self):
+            assert self._next_transaction is None
+            if self.count < 50:
+                other = AddToSeen()
+                other.value = self.count
+                self.count += 1
+                return [self, other]
+    rstm.run_all_transactions(DoInOrder())
+    assert seen != range(50) and sorted(seen) == range(50)
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -403,14 +403,12 @@
     'stm_getarrayitem':       LLOp(sideeffects=False, canrun=True),
     'stm_getinteriorfield':   LLOp(sideeffects=False, canrun=True),
     'stm_become_inevitable':  LLOp(),
-    'stm_descriptor_init':    LLOp(canrun=True),
-    'stm_descriptor_done':    LLOp(canrun=True),
     'stm_enter_transactional_mode': LLOp(canrun=True, canmallocgc=True),
     'stm_leave_transactional_mode': LLOp(canrun=True, canmallocgc=True),
-    'stm_writebarrier':       LLOp(sideeffects=False),
+    'stm_writebarrier':       LLOp(),
     'stm_normalize_global':   LLOp(),
     'stm_start_transaction':  LLOp(canrun=True, canmallocgc=True),
-    'stm_commit_transaction': LLOp(canrun=True, canmallocgc=True),
+    'stm_stop_transaction':   LLOp(canrun=True, canmallocgc=True),
 
     # __________ address operations __________
 
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -625,17 +625,19 @@
 
 def op_stm_descriptor_init():
     # for direct testing only
+    xxx
     from pypy.translator.stm import stmgcintf
     stmgcintf.StmOperations.set_tls(llmemory.NULL, 0)
 
 def op_stm_descriptor_done():
+    xxx
     from pypy.translator.stm import stmgcintf
     stmgcintf.StmOperations.del_tls()
 
 def op_stm_start_transaction():
     pass
 
-def op_stm_commit_transaction():
+def op_stm_stop_transaction():
     pass
 
 def op_stm_enter_transactional_mode():
diff --git a/pypy/translator/stm/src_stm/core.c b/pypy/translator/stm/src_stm/core.c
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/src_stm/core.c
@@ -0,0 +1,668 @@
+/* -*- c-basic-offset: 2 -*- */
+
+/************************************************************/
+
+#define ABORT_REASONS 8
+#define SPINLOOP_REASONS 10
+
+struct tx_descriptor {
+  void *rpython_tls_object;
+  jmp_buf *setjmp_buf;
+  owner_version_t start_time;
+  owner_version_t end_time;
+  /*unsigned long last_known_global_timestamp;*/
+  owner_version_t my_lock_word;
+  struct OrecList reads;
+  unsigned num_commits;
+  unsigned num_aborts[ABORT_REASONS];
+  unsigned num_spinloops[SPINLOOP_REASONS];
+  /*unsigned int spinloop_counter;*/
+  struct RedoLog redolog;   /* last item, because it's the biggest one */
+};
+
+/* global_timestamp contains in its lowest bit a flag equal to 1
+   if there is an inevitable transaction running */
+static volatile unsigned long global_timestamp = 2;
+static __thread struct tx_descriptor *thread_descriptor = NULL;
+static __thread struct tx_descriptor *active_thread_descriptor = NULL;
+static long (*rpython_get_size)(void*);
+
+/************************************************************/
+
+static unsigned long get_global_timestamp(struct tx_descriptor *d)
+{
+  return (/*d->last_known_global_timestamp =*/ global_timestamp);
+}
+
+static _Bool change_global_timestamp(struct tx_descriptor *d,
+                                     unsigned long old,
+                                     unsigned long new)
+{
+  if (bool_cas(&global_timestamp, old, new))
+    {
+      /*d->last_known_global_timestamp = new;*/
+      return 1;
+    }
+  return 0;
+}
+
+static void set_global_timestamp(struct tx_descriptor *d, unsigned long new)
+{
+  global_timestamp = new;
+  /*d->last_known_global_timestamp = new;*/
+}
+
+static void tx_abort(int);
+
+static void tx_spinloop(int num)
+{
+  unsigned int c;
+  int i;
+  struct tx_descriptor *d = active_thread_descriptor;
+  d->num_spinloops[num]++;
+
+  //printf("tx_spinloop(%d)\n", num);
+
+#if 0
+  c = d->spinloop_counter;
+  d->spinloop_counter = c * 9;
+  i = c & 0xff0000;
+  while (i >= 0) {
+    spinloop();
+    i -= 0x10000;
+  }
+#else
+  spinloop();
+#endif
+}
+
+static _Bool is_inevitable(struct tx_descriptor *d)
+{
+  return d->setjmp_buf == NULL;
+}
+
+/*** run the redo log to commit a transaction, and release the locks */
+static void tx_redo(struct tx_descriptor *d)
+{
+  owner_version_t newver = d->end_time;
+  wlog_t *item;
+  REDOLOG_LOOP_FORWARD(d->redolog, item)
+    {
+      void *globalobj = item->addr;
+      void *localobj = item->val;
+      long size = rpython_get_size(localobj);
+      memcpy(((char *)globalobj) + sizeof(orec_t),
+             ((char *)localobj) + sizeof(orec_t),
+             size - sizeof(orec_t));
+      /* unlock the orec */
+      volatile orec_t* o = get_orec(globalobj);
+      CFENCE;
+      o->version = newver;
+    } REDOLOG_LOOP_END;
+}
+
+/*** on abort, release locks and restore the old version number. */
+static void releaseAndRevertLocks(struct tx_descriptor *d)
+{
+  wlog_t *item;
+  REDOLOG_LOOP_FORWARD(d->redolog, item)
+    {
+      if (item->p != -1)
+        {
+          volatile orec_t* o = get_orec(item->addr);
+          o->version = item->p;
+        }
+    } REDOLOG_LOOP_END;
+}
+
+/*** release locks and restore the old version number, ready to retry later */
+static void releaseLocksForRetry(struct tx_descriptor *d)
+{
+  wlog_t *item;
+  REDOLOG_LOOP_FORWARD(d->redolog, item)
+    {
+      volatile orec_t* o = get_orec(item->addr);
+      assert(item->p != -1);
+      o->version = item->p;
+      item->p = -1;
+    } REDOLOG_LOOP_END;
+}
+
+/*** lock all locations */
+static void acquireLocks(struct tx_descriptor *d)
+{
+  wlog_t *item;
+  // try to lock every location in the write set
+  REDOLOG_LOOP_BACKWARD(d->redolog, item)
+    {
+      // get orec, read its version#
+      volatile orec_t* o = get_orec(item->addr);
+      owner_version_t ovt;
+
+    retry:
+      ovt = o->version;
+
+      // if orec not locked, lock it
+      //
+      // NB: if ovt > start time, we may introduce inconsistent
+      // reads.  Since most writes are also reads, we'll just abort under this
+      // condition.  This can introduce false conflicts
+      if (!IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
+        if (!bool_cas(&o->version, ovt, d->my_lock_word))
+          goto retry;
+        // save old version to item->p.  Now we hold the lock.
+        item->p = ovt;
+      }
+      // else if the location is too recent...
+      else if (!IS_LOCKED(ovt))
+        tx_abort(0);
+      // else it is locked: check it's not by me
+      else {
+        assert(ovt != d->my_lock_word);
+        // we can either abort or spinloop.  Because we are at the end of
+        // the transaction we might try to spinloop, even though after the
+        // lock is released the ovt will be very recent, possibly
+        // > d->start_time.  It is necessary to spinloop in case we are
+        // inevitable, so use that as a criteria.  Another solution to avoid
+        // deadlocks would be to sort the order in which we take the locks.
+        if (is_inevitable(d))
+          tx_spinloop(8);
+        else
+          tx_abort(6);
+        goto retry;
+      }
+    } REDOLOG_LOOP_END;
+}
+
+static void common_cleanup(struct tx_descriptor *d)
+{
+  d->reads.size = 0;
+  redolog_clear(&d->redolog);
+}
+
+static void tx_cleanup(struct tx_descriptor *d)
+{
+  // release the locks and restore version numbers
+  releaseAndRevertLocks(d);
+  // reset all lists
+  common_cleanup(d);
+}
+
+static void tx_restart(struct tx_descriptor *d)
+{
+  tx_cleanup(d);
+  tx_spinloop(0);
+  longjmp(*d->setjmp_buf, 1);
+}
+
+/*** increase the abort count and restart the transaction */
+static void tx_abort(int reason)
+{
+  struct tx_descriptor *d = active_thread_descriptor;
+  assert(!is_inevitable(d));
+  d->num_aborts[reason]++;
+#ifdef RPY_STM_DEBUG_PRINT
+  PYPY_DEBUG_START("stm-abort");
+  if (PYPY_HAVE_DEBUG_PRINTS)
+      fprintf(PYPY_DEBUG_FILE, "thread %lx aborting %d\n",
+                               (long)pthread_self(), reason);
+  PYPY_DEBUG_STOP("stm-abort");
+#endif
+  tx_restart(d);
+}
+
+/**
+ * fast-path validation, assuming that I don't hold locks.
+ */
+static void validate_fast(struct tx_descriptor *d, int lognum)
+{
+  int i;
+  owner_version_t ovt;
+  assert(!is_inevitable(d));
+  for (i=0; i<d->reads.size; i++)
+    {
+    retry:
+      ovt = d->reads.items[i]->version;
+      if (IS_LOCKED_OR_NEWER(ovt, d->start_time))
+        {
+          // If locked, we wait until it becomes unlocked.  The chances are
+          // that it will then have a very recent start_time, likely
+          // > d->start_time, but it might still be better than always aborting
+          if (IS_LOCKED(ovt))
+            {
+              tx_spinloop(lognum);  /* tx_spinloop(1), tx_spinloop(2),
+                                       tx_spinloop(3) */
+              goto retry;
+            }
+          else
+            // abort if the timestamp is newer than my start time.  
+            tx_abort(lognum);  /* tx_abort(1), tx_abort(2), tx_abort(3) */
+        }
+    }
+}
+
+/**
+ * validate the read set by making sure that all orecs that we've read have
+ * timestamps at least as old as our start time, unless we locked those orecs.
+ */
+static void validate(struct tx_descriptor *d)
+{
+  int i;
+  owner_version_t ovt;
+  assert(!is_inevitable(d));
+  for (i=0; i<d->reads.size; i++)
+    {
+      ovt = d->reads.items[i]->version;      // read this orec
+      if (IS_LOCKED_OR_NEWER(ovt, d->start_time))
+        {
+          if (!IS_LOCKED(ovt))
+            // if unlocked and newer than start time, abort
+            tx_abort(4);
+          else
+            {
+              // if locked and not by me, abort
+              if (ovt != d->my_lock_word)
+                tx_abort(5);
+            }
+        }
+    }
+}
+
+#ifdef USE_PTHREAD_MUTEX
+/* mutex: only to avoid busy-looping too much in tx_spinloop() below */
+static pthread_mutex_t mutex_inevitable = PTHREAD_MUTEX_INITIALIZER;
+# ifdef RPY_STM_ASSERT
+static unsigned long locked_by = 0;
+static void mutex_lock(void)
+{
+  unsigned long pself = (unsigned long)pthread_self();
+#ifdef RPY_STM_DEBUG_PRINT
+  if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE,
+                                      "%lx: mutex inev locking...\n", pself);
+#endif
+  assert(locked_by != pself);
+  pthread_mutex_lock(&mutex_inevitable);
+  locked_by = pself;
+#ifdef RPY_STM_DEBUG_PRINT
+  if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE,
+                                      "%lx: mutex inev locked\n", pself);
+#endif
+}
+static void mutex_unlock(void)
+{
+  unsigned long pself = (unsigned long)pthread_self();
+  locked_by = 0;
+#ifdef RPY_STM_DEBUG_PRINT
+  if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE,
+                                      "%lx: mutex inev unlocked\n", pself);
+#endif
+  pthread_mutex_unlock(&mutex_inevitable);
+}
+# else
+#  define mutex_lock()    pthread_mutex_lock(&mutex_inevitable)
+#  define mutex_unlock()  pthread_mutex_unlock(&mutex_inevitable)
+# endif
+#else
+# define mutex_lock()     /* nothing */
+# define mutex_unlock()   /* nothing */
+#endif
+
+static void wait_end_inevitability(struct tx_descriptor *d)
+{
+  unsigned long curts;
+  releaseLocksForRetry(d);
+
+  // We are going to wait until the other inevitable transaction
+  // finishes.  XXX We could do better here: we could check if
+  // committing 'd' would create a conflict for the other inevitable
+  // thread 'd_inev' or not.  It requires peeking in 'd_inev' from this
+  // thread (which we never do so far) in order to do something like
+  // 'validate_fast(d_inev); d_inev->start_time = updated;'
+
+  while ((curts = get_global_timestamp(d)) & 1)
+    {
+      // while we're about to wait anyway, we can do a validate_fast
+      if (d->start_time < curts - 1)
+        {
+          validate_fast(d, 3);
+          d->start_time = curts - 1;
+        }
+      tx_spinloop(4);
+      mutex_lock();
+      mutex_unlock();
+    }
+  acquireLocks(d);
+}
+
+static void commitInevitableTransaction(struct tx_descriptor *d)
+{
+  unsigned long ts;
+  _Bool ok;
+
+  // no-one else can modify global_timestamp if I'm inevitable
+  // and d_inev_checking is 0
+  ts = get_global_timestamp(d);
+  assert(ts & 1);
+  set_global_timestamp(d, ts + 1);
+  d->end_time = ts + 1;
+  assert(d->end_time == (d->start_time + 2));
+
+  // run the redo log, and release the locks
+  tx_redo(d);
+
+  mutex_unlock();
+}
+
+/* lazy/lazy read instrumentation */
+#define STM_DO_READ(READ_OPERATION)                                     \
+  if (is_inevitable(d)) {                                               \
+    /* if is_inevitable(), then we don't need to do the checking of  */ \
+    /* o->version done below --- but more importantly, we don't need */ \
+    /* to insert o in the OrecList                                   */ \
+    READ_OPERATION;                                                     \
+  }                                                                     \
+  else {                                                                \
+ retry:                                                                 \
+  /* read the orec BEFORE we read anything else */                      \
+  ovt = o->version;                                                     \
+  CFENCE;                                                               \
+                                                                        \
+  /* this tx doesn't hold any locks, so if the lock for this addr is */ \
+  /* held, there is contention.  A lock is never hold for too long,  */ \
+  /* so spinloop until it is released.                               */ \
+  if (IS_LOCKED_OR_NEWER(ovt, d->start_time))                           \
+    {                                                                   \
+      if (IS_LOCKED(ovt)) {                                             \
+        tx_spinloop(7);                                                 \
+        goto retry;                                                     \
+      }                                                                 \
+      /* else this location is too new, scale forward */                \
+      owner_version_t newts = get_global_timestamp(d) & ~1;             \
+      validate_fast(d, 1);                                              \
+      d->start_time = newts;                                            \
+    }                                                                   \
+                                                                        \
+  /* orec is unlocked, with ts <= start_time.  read the location */     \
+  READ_OPERATION;                                                       \
+                                                                        \
+  /* postvalidate AFTER reading addr: */                                \
+  CFENCE;                                                               \
+  if (__builtin_expect(o->version != ovt, 0))                           \
+    goto retry;       /* oups, try again */                             \
+                                                                        \
+  oreclist_insert(&d->reads, (orec_t*)o);                               \
+  }
+
+
+#define STM_READ_WORD(SIZE, SUFFIX, TYPE)                               \
+TYPE stm_read_int##SIZE##SUFFIX(void* addr, long offset)                \
+{                                                                       \
+  struct tx_descriptor *d = active_thread_descriptor;                   \
+  volatile orec_t *o = get_orec(addr);                                  \
+  owner_version_t ovt;                                                  \
+                                                                        \
+  assert(sizeof(TYPE) == SIZE);                                         \
+  /* XXX try to remove this check from the main path:           */      \
+  /*     d is NULL only when in non-main threads but            */      \
+  /*     outside a transaction.                                 */      \
+  if (d == NULL)                                                        \
+    return *(TYPE *)(((char *)addr) + offset);                          \
+                                                                        \
+  if ((o->tid & GCFLAG_WAS_COPIED) != 0)                                \
+    {                                                                   \
+      /* Look up in the thread-local dictionary. */                     \
+      wlog_t *found;                                                    \
+      REDOLOG_FIND(d->redolog, addr, found, goto not_found);            \
+      orec_t *localobj = (orec_t *)found->val;                          \
+      assert((localobj->tid & GCFLAG_GLOBAL) == 0);                     \
+      return *(TYPE *)(((char *)localobj) + offset);                    \
+                                                                        \
+    not_found:;                                                         \
+    }                                                                   \
+                                                                        \
+  TYPE tmp;                                                             \
+  STM_DO_READ(tmp = *(TYPE *)(((char *)addr) + offset));                \
+  return tmp;                                                           \
+}
+
+STM_READ_WORD(1, , char)
+STM_READ_WORD(2, , short)
+STM_READ_WORD(4, , int)
+STM_READ_WORD(8, , long long)
+STM_READ_WORD(8,f, double)
+STM_READ_WORD(4,f, float)
+
+void stm_copy_transactional_to_raw(void *src, void *dst, long size)
+{
+  struct tx_descriptor *d = active_thread_descriptor;
+  volatile orec_t *o = get_orec(src);
+  owner_version_t ovt;
+
+  assert(d != NULL);
+
+  /* don't copy the header */
+  src = ((char *)src) + sizeof(orec_t);
+  dst = ((char *)dst) + sizeof(orec_t);
+  size -= sizeof(orec_t);
+
+  STM_DO_READ(memcpy(dst, src, size));
+}
+
+static struct tx_descriptor *descriptor_init(long in_main_thread)
+{
+  assert(thread_descriptor == NULL);
+  assert(active_thread_descriptor == NULL);
+  if (1)  /* for hg diff */
+    {
+      struct tx_descriptor *d = malloc(sizeof(struct tx_descriptor));
+      memset(d, 0, sizeof(struct tx_descriptor));
+
+#ifdef RPY_STM_DEBUG_PRINT
+      PYPY_DEBUG_START("stm-init");
+#endif
+
+      if (in_main_thread)
+        {
+          d->my_lock_word = 0;   /* special value for the main thread */
+        }
+      else
+        {
+          /* initialize 'my_lock_word' to be a unique negative number */
+          d->my_lock_word = (owner_version_t)d;
+          if (!IS_LOCKED(d->my_lock_word))
+            d->my_lock_word = ~d->my_lock_word;
+          assert(IS_LOCKED(d->my_lock_word));
+        }
+      /*d->spinloop_counter = (unsigned int)(d->my_lock_word | 1);*/
+
+      thread_descriptor = d;
+      if (in_main_thread)
+        ; //stm_leave_transactional_mode();
+      else
+        ;   /* active_thread_descriptor stays NULL */
+
+#ifdef RPY_STM_DEBUG_PRINT
+      if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE, "thread %lx starting\n",
+                                          (long)pthread_self());
+      PYPY_DEBUG_STOP("stm-init");
+#endif
+      return d;
+    }
+}
+
+static void descriptor_done(void)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  assert(d != NULL);
+  assert(active_thread_descriptor == NULL);
+
+  thread_descriptor = NULL;
+
+#ifdef RPY_STM_DEBUG_PRINT
+  PYPY_DEBUG_START("stm-done");
+  if (PYPY_HAVE_DEBUG_PRINTS) {
+    int num_aborts = 0, num_spinloops = 0;
+    int i, prevchar;
+    char line[256], *p = line;
+
+    for (i=0; i<ABORT_REASONS; i++)
+      num_aborts += d->num_aborts[i];
+    for (i=0; i<SPINLOOP_REASONS; i++)
+      num_spinloops += d->num_spinloops[i];
+
+    p += sprintf(p, "thread %lx: %d commits, %d aborts\n",
+                 (long)pthread_self(),
+                 d->num_commits,
+                 num_aborts);
+
+    for (i=0; i<ABORT_REASONS; i++)
+      p += sprintf(p, "%c%d", i == 0 ? '[' : ',',
+                   d->num_aborts[i]);
+
+    for (i=1; i<SPINLOOP_REASONS; i++)  /* num_spinloops[0] == num_aborts */
+      p += sprintf(p, "%c%d", i == 1 ? '|' : ',',
+                   d->num_spinloops[i]);
+
+    p += sprintf(p, "]\n");
+    fwrite(line, 1, p - line, PYPY_DEBUG_FILE);
+  }
+  PYPY_DEBUG_STOP("stm-done");
+#endif
+
+  free(d);
+}
+
+static void begin_transaction(jmp_buf* buf)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  /* you need to call descriptor_init() before calling
+     stm_perform_transaction() */
+  assert(d != NULL);
+  d->setjmp_buf = buf;
+  d->start_time = (/*d->last_known_global_timestamp*/ global_timestamp) & ~1;
+  active_thread_descriptor = d;
+}
+
+static long commit_transaction(void)
+{
+  struct tx_descriptor *d = active_thread_descriptor;
+  assert(d != NULL);
+
+  // if I don't have writes, I'm committed
+  if (!redolog_any_entry(&d->redolog))
+    {
+      if (is_inevitable(d))
+        {
+          unsigned long ts = get_global_timestamp(d);
+          assert(ts & 1);
+          set_global_timestamp(d, ts - 1);
+          mutex_unlock();
+        }
+      d->num_commits++;
+      common_cleanup(d);
+      active_thread_descriptor = NULL;
+      return d->start_time;
+    }
+
+  // bring that variable over to this CPU core (optimization, maybe)
+  global_timestamp;
+
+  // acquire locks
+  acquireLocks(d);
+
+  if (is_inevitable(d))
+    {
+      commitInevitableTransaction(d);
+    }
+  else
+    {
+      while (1)
+        {
+          unsigned long expected = get_global_timestamp(d);
+          if (expected & 1)
+            {
+              // wait until it is done.  hopefully we can then proceed
+              // without conflicts.
+              wait_end_inevitability(d);
+              continue;
+            }
+          if (change_global_timestamp(d, expected, expected + 2))
+            {
+              d->end_time = expected + 2;
+              break;
+            }
+        }
+
+      // validate (but skip validation if nobody else committed)
+      if (d->end_time != (d->start_time + 2))
+        validate(d);
+
+      // run the redo log, and release the locks
+      tx_redo(d);
+    }
+
+  // remember that this was a commit
+  d->num_commits++;
+
+  // reset all lists
+  common_cleanup(d);
+  active_thread_descriptor = NULL;
+  return d->end_time;
+}
+
+void stm_try_inevitable(STM_CCHARP1(why))
+{
+  /* when a transaction is inevitable, its start_time is equal to
+     global_timestamp and global_timestamp cannot be incremented
+     by another thread.  We set the lowest bit in global_timestamp
+     to 1. */
+  struct tx_descriptor *d = active_thread_descriptor;
+  if (d == NULL)
+    return;
+
+  if (is_inevitable(d))   /* also when the transaction is inactive */
+    {
+      return;  /* I am already inevitable */
+    }
+
+#ifdef RPY_STM_DEBUG_PRINT
+  PYPY_DEBUG_START("stm-inevitable");
+  if (PYPY_HAVE_DEBUG_PRINTS)
+    {
+      fprintf(PYPY_DEBUG_FILE, "%s\n", why);
+    }
+#endif
+
+  while (1)
+    {
+      unsigned long curtime = get_global_timestamp(d);
+      if (d->start_time != (curtime & ~1))
+        {                             /* scale forward */
+          validate_fast(d, 2);
+          d->start_time = curtime & ~1;
+        }
+      mutex_lock();
+      if (curtime & 1)   /* there is, or was, already an inevitable thread */
+        {
+          /* should we spinloop here, or abort (and likely come back
+             in try_inevitable() very soon)?  unclear.  For now
+             let's try to spinloop, after the waiting done by
+             acquiring the mutex */
+          mutex_unlock();
+          tx_spinloop(6);
+          continue;
+        }
+      if (change_global_timestamp(d, curtime, curtime + 1))
+        break;
+      mutex_unlock();
+    }
+  d->setjmp_buf = NULL;   /* inevitable from now on */
+#ifdef RPY_STM_DEBUG_PRINT
+  PYPY_DEBUG_STOP("stm-inevitable");
+#endif
+}
+
+void stm_abort_and_retry(void)
+{
+  tx_abort(7);     /* manual abort */
+}
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -13,7 +13,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#define USE_PTHREAD_MUTEX    /* optional */
+#define USE_PTHREAD_MUTEX    /* can be made optional */
 #ifdef USE_PTHREAD_MUTEX
 # include <pthread.h>
 #endif
@@ -55,821 +55,12 @@
 
 #define get_orec(addr)  ((volatile orec_t *)(addr))
 
+/************************************************************/
+
 #include "src_stm/lists.c"
+#include "src_stm/core.c"
+#include "src_stm/rpyintf.c"
 
 /************************************************************/
 
-#define ABORT_REASONS 8
-#define SPINLOOP_REASONS 10
-
-struct tx_descriptor {
-  void *rpython_tls_object;
-  jmp_buf *setjmp_buf;
-  owner_version_t start_time;
-  owner_version_t end_time;
-  /*unsigned long last_known_global_timestamp;*/
-  owner_version_t my_lock_word;
-  struct OrecList reads;
-  unsigned num_commits;
-  unsigned num_aborts[ABORT_REASONS];
-  unsigned num_spinloops[SPINLOOP_REASONS];
-  /*unsigned int spinloop_counter;*/
-  struct RedoLog redolog;   /* last item, because it's the biggest one */
-};
-
-/* global_timestamp contains in its lowest bit a flag equal to 1
-   if there is an inevitable transaction running */
-static volatile unsigned long global_timestamp = 2;
-static __thread struct tx_descriptor *thread_descriptor = NULL;
-static __thread struct tx_descriptor *active_thread_descriptor = NULL;
-static long (*rpython_get_size)(void*);
-
-/************************************************************/
-
-static unsigned long get_global_timestamp(struct tx_descriptor *d)
-{
-  return (/*d->last_known_global_timestamp =*/ global_timestamp);
-}
-
-static _Bool change_global_timestamp(struct tx_descriptor *d,
-                                     unsigned long old,
-                                     unsigned long new)
-{
-  if (bool_cas(&global_timestamp, old, new))
-    {
-      /*d->last_known_global_timestamp = new;*/
-      return 1;
-    }
-  return 0;
-}
-
-static void set_global_timestamp(struct tx_descriptor *d, unsigned long new)
-{
-  global_timestamp = new;
-  /*d->last_known_global_timestamp = new;*/
-}
-
-static void tx_abort(int);
-
-static void tx_spinloop(int num)
-{
-  unsigned int c;
-  int i;
-  struct tx_descriptor *d = active_thread_descriptor;
-  d->num_spinloops[num]++;
-
-  //printf("tx_spinloop(%d)\n", num);
-
-#if 0
-  c = d->spinloop_counter;
-  d->spinloop_counter = c * 9;
-  i = c & 0xff0000;
-  while (i >= 0) {
-    spinloop();
-    i -= 0x10000;
-  }
-#else
-  spinloop();
-#endif
-}
-
-static _Bool is_inevitable(struct tx_descriptor *d)
-{
-  return d->setjmp_buf == NULL;
-}
-
-/*** run the redo log to commit a transaction, and release the locks */
-static void tx_redo(struct tx_descriptor *d)
-{
-  owner_version_t newver = d->end_time;
-  wlog_t *item;
-  REDOLOG_LOOP_FORWARD(d->redolog, item)
-    {
-      void *globalobj = item->addr;
-      void *localobj = item->val;
-      long size = rpython_get_size(localobj);
-      memcpy(((char *)globalobj) + sizeof(orec_t),
-             ((char *)localobj) + sizeof(orec_t),
-             size - sizeof(orec_t));
-      /* unlock the orec */
-      volatile orec_t* o = get_orec(globalobj);
-      CFENCE;
-      o->version = newver;
-    } REDOLOG_LOOP_END;
-}
-
-/*** on abort, release locks and restore the old version number. */
-static void releaseAndRevertLocks(struct tx_descriptor *d)
-{
-  wlog_t *item;
-  REDOLOG_LOOP_FORWARD(d->redolog, item)
-    {
-      if (item->p != -1)
-        {
-          volatile orec_t* o = get_orec(item->addr);
-          o->version = item->p;
-        }
-    } REDOLOG_LOOP_END;
-}
-
-/*** release locks and restore the old version number, ready to retry later */
-static void releaseLocksForRetry(struct tx_descriptor *d)
-{
-  wlog_t *item;
-  REDOLOG_LOOP_FORWARD(d->redolog, item)
-    {
-      volatile orec_t* o = get_orec(item->addr);
-      assert(item->p != -1);
-      o->version = item->p;
-      item->p = -1;
-    } REDOLOG_LOOP_END;
-}
-
-/*** lock all locations */
-static void acquireLocks(struct tx_descriptor *d)
-{
-  wlog_t *item;
-  // try to lock every location in the write set
-  REDOLOG_LOOP_BACKWARD(d->redolog, item)
-    {
-      // get orec, read its version#
-      volatile orec_t* o = get_orec(item->addr);
-      owner_version_t ovt;
-
-    retry:
-      ovt = o->version;
-
-      // if orec not locked, lock it
-      //
-      // NB: if ovt > start time, we may introduce inconsistent
-      // reads.  Since most writes are also reads, we'll just abort under this
-      // condition.  This can introduce false conflicts
-      if (!IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
-        if (!bool_cas(&o->version, ovt, d->my_lock_word))
-          goto retry;
-        // save old version to item->p.  Now we hold the lock.
-        item->p = ovt;
-      }
-      // else if the location is too recent...
-      else if (!IS_LOCKED(ovt))
-        tx_abort(0);
-      // else it is locked: check it's not by me
-      else {
-        assert(ovt != d->my_lock_word);
-        // we can either abort or spinloop.  Because we are at the end of
-        // the transaction we might try to spinloop, even though after the
-        // lock is released the ovt will be very recent, possibly
-        // > d->start_time.  It is necessary to spinloop in case we are
-        // inevitable, so use that as a criteria.  Another solution to avoid
-        // deadlocks would be to sort the order in which we take the locks.
-        if (is_inevitable(d))
-          tx_spinloop(8);
-        else
-          tx_abort(6);
-        goto retry;
-      }
-    } REDOLOG_LOOP_END;
-}
-
-static void common_cleanup(struct tx_descriptor *d)
-{
-  d->reads.size = 0;
-  redolog_clear(&d->redolog);
-}
-
-static void tx_cleanup(struct tx_descriptor *d)
-{
-  // release the locks and restore version numbers
-  releaseAndRevertLocks(d);
-  // reset all lists
-  common_cleanup(d);
-}
-
-static void tx_restart(struct tx_descriptor *d)
-{
-  tx_cleanup(d);
-  tx_spinloop(0);
-  longjmp(*d->setjmp_buf, 1);
-}
-
-/*** increase the abort count and restart the transaction */
-static void tx_abort(int reason)
-{
-  struct tx_descriptor *d = active_thread_descriptor;
-  assert(!is_inevitable(d));
-  d->num_aborts[reason]++;
-#ifdef RPY_STM_DEBUG_PRINT
-  PYPY_DEBUG_START("stm-abort");
-  if (PYPY_HAVE_DEBUG_PRINTS)
-      fprintf(PYPY_DEBUG_FILE, "thread %lx aborting %d\n",
-                               (long)pthread_self(), reason);
-  PYPY_DEBUG_STOP("stm-abort");
-#endif
-  tx_restart(d);
-}
-
-/**
- * fast-path validation, assuming that I don't hold locks.
- */
-static void validate_fast(struct tx_descriptor *d, int lognum)
-{
-  int i;
-  owner_version_t ovt;
-  assert(!is_inevitable(d));
-  for (i=0; i<d->reads.size; i++)
-    {
-    retry:
-      ovt = d->reads.items[i]->version;
-      if (IS_LOCKED_OR_NEWER(ovt, d->start_time))
-        {
-          // If locked, we wait until it becomes unlocked.  The chances are
-          // that it will then have a very recent start_time, likely
-          // > d->start_time, but it might still be better than always aborting
-          if (IS_LOCKED(ovt))
-            {
-              tx_spinloop(lognum);  /* tx_spinloop(1), tx_spinloop(2),
-                                       tx_spinloop(3) */
-              goto retry;
-            }
-          else
-            // abort if the timestamp is newer than my start time.  
-            tx_abort(lognum);  /* tx_abort(1), tx_abort(2), tx_abort(3) */
-        }
-    }
-}
-
-/**
- * validate the read set by making sure that all orecs that we've read have
- * timestamps at least as old as our start time, unless we locked those orecs.
- */
-static void validate(struct tx_descriptor *d)
-{
-  int i;
-  owner_version_t ovt;
-  assert(!is_inevitable(d));
-  for (i=0; i<d->reads.size; i++)
-    {
-      ovt = d->reads.items[i]->version;      // read this orec
-      if (IS_LOCKED_OR_NEWER(ovt, d->start_time))
-        {
-          if (!IS_LOCKED(ovt))
-            // if unlocked and newer than start time, abort
-            tx_abort(4);
-          else
-            {
-              // if locked and not by me, abort
-              if (ovt != d->my_lock_word)
-                tx_abort(5);
-            }
-        }
-    }
-}
-
-#ifdef USE_PTHREAD_MUTEX
-/* mutex: only to avoid busy-looping too much in tx_spinloop() below */
-static pthread_mutex_t mutex_inevitable = PTHREAD_MUTEX_INITIALIZER;
-# ifdef RPY_STM_ASSERT
-unsigned long locked_by = 0;
-static void mutex_lock(void)
-{
-  unsigned long pself = (unsigned long)pthread_self();
-#ifdef RPY_STM_DEBUG_PRINT
-  if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE,
-                                      "%lx: mutex inev locking...\n", pself);
-#endif
-  assert(locked_by != pself);
-  pthread_mutex_lock(&mutex_inevitable);
-  locked_by = pself;
-#ifdef RPY_STM_DEBUG_PRINT
-  if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE,
-                                      "%lx: mutex inev locked\n", pself);
-#endif
-}
-static void mutex_unlock(void)
-{
-  unsigned long pself = (unsigned long)pthread_self();
-  locked_by = 0;
-#ifdef RPY_STM_DEBUG_PRINT
-  if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE,
-                                      "%lx: mutex inev unlocked\n", pself);
-#endif
-  pthread_mutex_unlock(&mutex_inevitable);
-}
-# else
-#  define mutex_lock()    pthread_mutex_lock(&mutex_inevitable)
-#  define mutex_unlock()  pthread_mutex_unlock(&mutex_inevitable)
-# endif
-#else
-# define mutex_lock()     /* nothing */
-# define mutex_unlock()   /* nothing */
-#endif
-
-static void wait_end_inevitability(struct tx_descriptor *d)
-{
-  unsigned long curts;
-  releaseLocksForRetry(d);
-
-  // We are going to wait until the other inevitable transaction
-  // finishes.  XXX We could do better here: we could check if
-  // committing 'd' would create a conflict for the other inevitable
-  // thread 'd_inev' or not.  It requires peeking in 'd_inev' from this
-  // thread (which we never do so far) in order to do something like
-  // 'validate_fast(d_inev); d_inev->start_time = updated;'
-
-  while ((curts = get_global_timestamp(d)) & 1)
-    {
-      // while we're about to wait anyway, we can do a validate_fast
-      if (d->start_time < curts - 1)
-        {
-          validate_fast(d, 3);
-          d->start_time = curts - 1;
-        }
-      tx_spinloop(4);
-      mutex_lock();
-      mutex_unlock();
-    }
-  acquireLocks(d);
-}
-
-static void commitInevitableTransaction(struct tx_descriptor *d)
-{
-  unsigned long ts;
-  _Bool ok;
-
-  // no-one else can modify global_timestamp if I'm inevitable
-  // and d_inev_checking is 0
-  ts = get_global_timestamp(d);
-  assert(ts & 1);
-  set_global_timestamp(d, ts + 1);
-  d->end_time = ts + 1;
-  assert(d->end_time == (d->start_time + 2));
-
-  // run the redo log, and release the locks
-  tx_redo(d);
-
-  mutex_unlock();
-}
-
-/* lazy/lazy read instrumentation */
-#define STM_DO_READ(READ_OPERATION)                                     \
-  if (is_inevitable(d)) {                                               \
-    /* if is_inevitable(), then we don't need to do the checking of  */ \
-    /* o->version done below --- but more importantly, we don't need */ \
-    /* to insert o in the OrecList                                   */ \
-    READ_OPERATION;                                                     \
-  }                                                                     \
-  else {                                                                \
- retry:                                                                 \
-  /* read the orec BEFORE we read anything else */                      \
-  ovt = o->version;                                                     \
-  CFENCE;                                                               \
-                                                                        \
-  /* this tx doesn't hold any locks, so if the lock for this addr is */ \
-  /* held, there is contention.  A lock is never hold for too long,  */ \
-  /* so spinloop until it is released.                               */ \
-  if (IS_LOCKED_OR_NEWER(ovt, d->start_time))                           \
-    {                                                                   \
-      if (IS_LOCKED(ovt)) {                                             \
-        tx_spinloop(7);                                                 \
-        goto retry;                                                     \
-      }                                                                 \
-      /* else this location is too new, scale forward */                \
-      owner_version_t newts = get_global_timestamp(d) & ~1;             \
-      validate_fast(d, 1);                                              \
-      d->start_time = newts;                                            \
-    }                                                                   \
-                                                                        \
-  /* orec is unlocked, with ts <= start_time.  read the location */     \
-  READ_OPERATION;                                                       \
-                                                                        \
-  /* postvalidate AFTER reading addr: */                                \
-  CFENCE;                                                               \
-  if (__builtin_expect(o->version != ovt, 0))                           \
-    goto retry;       /* oups, try again */                             \
-                                                                        \
-  oreclist_insert(&d->reads, (orec_t*)o);                               \
-  }
-
-
-#define STM_READ_WORD(SIZE, SUFFIX, TYPE)                               \
-TYPE stm_read_int##SIZE##SUFFIX(void* addr, long offset)                \
-{                                                                       \
-  struct tx_descriptor *d = active_thread_descriptor;                   \
-  volatile orec_t *o = get_orec(addr);                                  \
-  owner_version_t ovt;                                                  \
-                                                                        \
-  assert(sizeof(TYPE) == SIZE);                                         \
-  /* XXX try to remove this check from the main path:           */      \
-  /*     d is NULL only when in non-main threads but            */      \
-  /*     outside a transaction.                                 */      \
-  if (d == NULL)                                                        \
-    return *(TYPE *)(((char *)addr) + offset);                          \
-                                                                        \
-  if ((o->tid & GCFLAG_WAS_COPIED) != 0)                                \
-    {                                                                   \
-      /* Look up in the thread-local dictionary. */                     \
-      wlog_t *found;                                                    \
-      REDOLOG_FIND(d->redolog, addr, found, goto not_found);            \
-      orec_t *localobj = (orec_t *)found->val;                          \
-      assert((localobj->tid & GCFLAG_GLOBAL) == 0);                     \
-      return *(TYPE *)(((char *)localobj) + offset);                    \
-                                                                        \
-    not_found:;                                                         \
-    }                                                                   \
-                                                                        \
-  TYPE tmp;                                                             \
-  STM_DO_READ(tmp = *(TYPE *)(((char *)addr) + offset));                \
-  return tmp;                                                           \
-}
-
-STM_READ_WORD(1, , char)
-STM_READ_WORD(2, , short)
-STM_READ_WORD(4, , int)
-STM_READ_WORD(8, , long long)
-STM_READ_WORD(8,f, double)
-STM_READ_WORD(4,f, float)
-
-void stm_copy_transactional_to_raw(void *src, void *dst, long size)
-{
-  struct tx_descriptor *d = active_thread_descriptor;
-  volatile orec_t *o = get_orec(src);
-  owner_version_t ovt;
-
-  assert(d != NULL);
-
-  /* don't copy the header */
-  src = ((char *)src) + sizeof(orec_t);
-  dst = ((char *)dst) + sizeof(orec_t);
-  size -= sizeof(orec_t);
-
-  STM_DO_READ(memcpy(dst, src, size));
-}
-
-
-static struct tx_descriptor *descriptor_init(long in_main_thread)
-{
-  assert(thread_descriptor == NULL);
-  assert(active_thread_descriptor == NULL);
-  if (1)  /* for hg diff */
-    {
-      struct tx_descriptor *d = malloc(sizeof(struct tx_descriptor));
-      memset(d, 0, sizeof(struct tx_descriptor));
-
-#ifdef RPY_STM_DEBUG_PRINT
-      PYPY_DEBUG_START("stm-init");
-#endif
-
-      if (in_main_thread)
-        {
-          d->my_lock_word = 0;   /* special value for the main thread */
-        }
-      else
-        {
-          /* initialize 'my_lock_word' to be a unique negative number */
-          d->my_lock_word = (owner_version_t)d;
-          if (!IS_LOCKED(d->my_lock_word))
-            d->my_lock_word = ~d->my_lock_word;
-          assert(IS_LOCKED(d->my_lock_word));
-        }
-      /*d->spinloop_counter = (unsigned int)(d->my_lock_word | 1);*/
-
-      thread_descriptor = d;
-      if (in_main_thread)
-        stm_leave_transactional_mode();
-      else
-        ;   /* active_thread_descriptor stays NULL */
-
-#ifdef RPY_STM_DEBUG_PRINT
-      if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE, "thread %lx starting\n",
-                                          (long)pthread_self());
-      PYPY_DEBUG_STOP("stm-init");
-#endif
-      return d;
-    }
-}
-
-static void descriptor_done(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  assert(d != NULL);
-  assert(active_thread_descriptor == NULL);
-
-  thread_descriptor = NULL;
-
-#ifdef RPY_STM_DEBUG_PRINT
-  PYPY_DEBUG_START("stm-done");
-  if (PYPY_HAVE_DEBUG_PRINTS) {
-    int num_aborts = 0, num_spinloops = 0;
-    int i, prevchar;
-    char line[256], *p = line;
-
-    for (i=0; i<ABORT_REASONS; i++)
-      num_aborts += d->num_aborts[i];
-    for (i=0; i<SPINLOOP_REASONS; i++)
-      num_spinloops += d->num_spinloops[i];
-
-    p += sprintf(p, "thread %lx: %d commits, %d aborts\n",
-                 (long)pthread_self(),
-                 d->num_commits,
-                 num_aborts);
-
-    for (i=0; i<ABORT_REASONS; i++)
-      p += sprintf(p, "%c%d", i == 0 ? '[' : ',',
-                   d->num_aborts[i]);
-
-    for (i=1; i<SPINLOOP_REASONS; i++)  /* num_spinloops[0] == num_aborts */
-      p += sprintf(p, "%c%d", i == 1 ? '|' : ',',
-                   d->num_spinloops[i]);
-
-    p += sprintf(p, "]\n");
-    fwrite(line, 1, p - line, PYPY_DEBUG_FILE);
-  }
-  PYPY_DEBUG_STOP("stm-done");
-#endif
-
-  free(d);
-}
-
-static void begin_transaction(jmp_buf* buf)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  /* you need to call descriptor_init() before calling
-     stm_perform_transaction() */
-  assert(d != NULL);
-  d->setjmp_buf = buf;
-  d->start_time = (/*d->last_known_global_timestamp*/ global_timestamp) & ~1;
-  active_thread_descriptor = d;
-}
-
-static long commit_transaction(void)
-{
-  struct tx_descriptor *d = active_thread_descriptor;
-  assert(d != NULL);
-
-  // if I don't have writes, I'm committed
-  if (!redolog_any_entry(&d->redolog))
-    {
-      if (is_inevitable(d))
-        {
-          unsigned long ts = get_global_timestamp(d);
-          assert(ts & 1);
-          set_global_timestamp(d, ts - 1);
-          mutex_unlock();
-        }
-      d->num_commits++;
-      common_cleanup(d);
-      active_thread_descriptor = NULL;
-      return d->start_time;
-    }
-
-  // bring that variable over to this CPU core (optimization, maybe)
-  global_timestamp;
-
-  // acquire locks
-  acquireLocks(d);
-
-  if (is_inevitable(d))
-    {
-      commitInevitableTransaction(d);
-    }
-  else
-    {
-      while (1)
-        {
-          unsigned long expected = get_global_timestamp(d);
-          if (expected & 1)
-            {
-              // wait until it is done.  hopefully we can then proceed
-              // without conflicts.
-              wait_end_inevitability(d);
-              continue;
-            }
-          if (change_global_timestamp(d, expected, expected + 2))
-            {
-              d->end_time = expected + 2;
-              break;
-            }
-        }
-
-      // validate (but skip validation if nobody else committed)
-      if (d->end_time != (d->start_time + 2))
-        validate(d);
-
-      // run the redo log, and release the locks
-      tx_redo(d);
-    }
-
-  // remember that this was a commit
-  d->num_commits++;
-
-  // reset all lists
-  common_cleanup(d);
-  active_thread_descriptor = NULL;
-  return d->end_time;
-}
-
-void* stm_perform_transaction(void*(*callback)(void*, long), void *arg,
-                              void *save_and_restore)
-{
-  void *result;
-  jmp_buf _jmpbuf;
-  volatile long v_counter = 0;
-  long counter;
-  void *volatile saved_value;
-  assert(active_thread_descriptor == NULL);
-  if (save_and_restore)
-    saved_value = *(void**)save_and_restore;
-  setjmp(_jmpbuf);
-  if (save_and_restore)
-    *(void**)save_and_restore = saved_value;
-  begin_transaction(&_jmpbuf);
-  counter = v_counter;
-  v_counter = counter + 1;
-  result = callback(arg, counter);
-  commit_transaction();
-  return result;
-}
-
-void stm_enter_transactional_mode(void)
-{
-  struct tx_descriptor *d = active_thread_descriptor;
-  assert(d != NULL);
-  assert(is_inevitable(d));
-  /* we only need a subset of a full commit */
-  acquireLocks(d);
-  commitInevitableTransaction(d);
-  common_cleanup(d);
-  active_thread_descriptor = NULL;
-}
-
-void stm_leave_transactional_mode(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  assert(active_thread_descriptor == NULL);
-
-  mutex_lock();
-  d->setjmp_buf = NULL;
-  d->start_time = get_global_timestamp(d);
-  assert(!(d->start_time & 1));
-  set_global_timestamp(d, d->start_time | 1);
-
-  assert(is_inevitable(d));
-  active_thread_descriptor = d;
-}
-
-void stm_try_inevitable(STM_CCHARP1(why))
-{
-  /* when a transaction is inevitable, its start_time is equal to
-     global_timestamp and global_timestamp cannot be incremented
-     by another thread.  We set the lowest bit in global_timestamp
-     to 1. */
-  struct tx_descriptor *d = active_thread_descriptor;
-  if (d == NULL)
-    return;
-
-  if (is_inevitable(d))   /* also when the transaction is inactive */
-    {
-      return;  /* I am already inevitable */
-    }
-
-#ifdef RPY_STM_DEBUG_PRINT
-  PYPY_DEBUG_START("stm-inevitable");
-  if (PYPY_HAVE_DEBUG_PRINTS)
-    {
-      fprintf(PYPY_DEBUG_FILE, "%s\n", why);
-    }
-#endif
-
-  while (1)
-    {
-      unsigned long curtime = get_global_timestamp(d);
-      if (d->start_time != (curtime & ~1))
-        {                             /* scale forward */
-          validate_fast(d, 2);
-          d->start_time = curtime & ~1;
-        }
-      mutex_lock();
-      if (curtime & 1)   /* there is, or was, already an inevitable thread */
-        {
-          /* should we spinloop here, or abort (and likely come back
-             in try_inevitable() very soon)?  unclear.  For now
-             let's try to spinloop, after the waiting done by
-             acquiring the mutex */
-          mutex_unlock();
-          tx_spinloop(6);
-          continue;
-        }
-      if (change_global_timestamp(d, curtime, curtime + 1))
-        break;
-      mutex_unlock();
-    }
-  d->setjmp_buf = NULL;   /* inevitable from now on */
-#ifdef RPY_STM_DEBUG_PRINT
-  PYPY_DEBUG_STOP("stm-inevitable");
-#endif
-}
-
-void stm_abort_and_retry(void)
-{
-  tx_abort(7);     /* manual abort */
-}
-
-long stm_debug_get_state(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  if (d == NULL)
-    return -2;
-  if (active_thread_descriptor == NULL)
-    {
-      assert(d->my_lock_word != 0);
-      return 0;
-    }
-  assert(d == active_thread_descriptor);
-  if (d->my_lock_word == 0)
-    return -1;
-  if (!is_inevitable(d))
-    return 1;
-  else
-    return 2;
-}
-
-long stm_thread_id(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  if (d == NULL)
-    return 0;    /* no thread_descriptor yet, assume it's the main thread */
-  return d->my_lock_word;
-}
-
-
-void stm_set_tls(void *newtls, long in_main_thread)
-{
-  struct tx_descriptor *d = descriptor_init(in_main_thread);
-  d->rpython_tls_object = newtls;
-}
-
-void *stm_get_tls(void)
-{
-  return thread_descriptor->rpython_tls_object;
-}
-
-void stm_del_tls(void)
-{
-  descriptor_done();
-}
-
-void *stm_tldict_lookup(void *key)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  wlog_t* found;
-  REDOLOG_FIND(d->redolog, key, found, goto not_found);
-  return found->val;
-
- not_found:
-  return NULL;
-}
-
-void stm_tldict_add(void *key, void *value)
-{
-  struct tx_descriptor *d = active_thread_descriptor;
-  assert(d != NULL);
-  redolog_insert(&d->redolog, key, value);
-}
-
-void stm_tldict_enum(void(*callback)(void*, void*, void*))
-{
-  struct tx_descriptor *d = thread_descriptor;
-  wlog_t *item;
-  void *tls = stm_get_tls();
-
-  REDOLOG_LOOP_FORWARD(d->redolog, item)
-    {
-      callback(tls, item->addr, item->val);
-    } REDOLOG_LOOP_END;
-}
-
-void stm_setup_size_getter(long(*getsize_fn)(void*))
-{
-  rpython_get_size = getsize_fn;
-}
-
-long stm_in_transaction(void)
-{
-  struct tx_descriptor *d = active_thread_descriptor;
-  return d != NULL;
-}
-
-void _stm_activate_transaction(long activate)
-{
-  assert(thread_descriptor != NULL);
-  if (activate)
-    {
-      active_thread_descriptor = thread_descriptor;
-    }
-  else
-    {
-      active_thread_descriptor = NULL;
-    }
-}
-
 #endif  /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -12,6 +12,11 @@
 #include "src/commondefs.h"
 
 
+long stm_in_transaction(void);
+void stm_run_all_transactions(void*(*)(void*, long), void*, long);
+
+
+
 void stm_setup_size_getter(long(*)(void*));
 
 void stm_set_tls(void *, long);
@@ -40,8 +45,6 @@
 
 
 void* stm_perform_transaction(void*(*)(void*, long), void*, void*);
-void stm_enter_transactional_mode(void);
-void stm_leave_transactional_mode(void);
 void stm_try_inevitable(STM_CCHARP1(why));
 void stm_abort_and_retry(void);
 long stm_debug_get_state(void);  /* -1: descriptor_init() was not called
@@ -50,7 +53,6 @@
                                      2: in an inevitable transaction */
 long stm_thread_id(void);  /* returns a unique thread id,
                               or 0 if descriptor_init() was not called */
-long stm_in_transaction(void);
 void _stm_activate_transaction(long);
 
 void stm_copy_transactional_to_raw(void *src, void *dst, long size);
diff --git a/pypy/translator/stm/src_stm/rpyintf.c b/pypy/translator/stm/src_stm/rpyintf.c
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/src_stm/rpyintf.c
@@ -0,0 +1,139 @@
+/* -*- c-basic-offset: 2 -*- */
+
+long stm_thread_id(void)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  if (d == NULL)
+    return 0;    /* no thread_descriptor yet, assume it's the main thread */
+  return d->my_lock_word;
+}
+
+void stm_set_tls(void *newtls, long in_main_thread)
+{
+  struct tx_descriptor *d = descriptor_init(in_main_thread);
+  d->rpython_tls_object = newtls;
+}
+
+void *stm_get_tls(void)
+{
+  return thread_descriptor->rpython_tls_object;
+}
+
+void stm_del_tls(void)
+{
+  descriptor_done();
+}
+
+void *stm_tldict_lookup(void *key)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  wlog_t* found;
+  REDOLOG_FIND(d->redolog, key, found, goto not_found);
+  return found->val;
+
+ not_found:
+  return NULL;
+}
+
+void stm_tldict_add(void *key, void *value)
+{
+  struct tx_descriptor *d = active_thread_descriptor;
+  assert(d != NULL);
+  redolog_insert(&d->redolog, key, value);
+}
+
+void stm_tldict_enum(void(*callback)(void*, void*, void*))
+{
+  struct tx_descriptor *d = thread_descriptor;
+  wlog_t *item;
+  void *tls = stm_get_tls();
+
+  REDOLOG_LOOP_FORWARD(d->redolog, item)
+    {
+      callback(tls, item->addr, item->val);
+    } REDOLOG_LOOP_END;
+}
+
+void stm_setup_size_getter(long(*getsize_fn)(void*))
+{
+  rpython_get_size = getsize_fn;
+}
+
+long stm_in_transaction(void)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  return d != NULL;
+}
+
+void _stm_activate_transaction(long activate)
+{
+  assert(thread_descriptor != NULL);
+  if (activate)
+    {
+      active_thread_descriptor = thread_descriptor;
+    }
+  else
+    {
+      active_thread_descriptor = NULL;
+    }
+}
+
+
+/* a helper to directly read the field '_next_transaction' on
+   RPython instances of pypy.rlib.rstm.Transaction */
+static void *next_transaction(void *
+
+
+
+/* this lock is acquired when we start running transactions, and
+   released only when we are finished. */
+static pthread_mutex_t mutex_unfinished = PTHREAD_MUTEX_INITIALIZER;
+
+/* this mutex is used to ensure non-conflicting accesses to global
+   data in run_thread(). */
+static pthread_mutex_t mutex_global = PTHREAD_MUTEX_INITIALIZER;
+
+/* some global data put there by run_all_transactions(). */
+typedef void *(*run_transaction_t)(void *, long);
+static run_transaction_t g_run_transaction;
+static void *g_first_transaction, *g_last_transaction;
+static int g_num_threads;
+static int g_num_waiting_threads;
+
+/* the main function running a thread */
+static void *run_thread(void *ignored)
+{
+  pthread_mutex_lock(&mutex_global);
+
+  g_num_waiting_threads++;
+  if (g_num_waiting_threads == g_num_threads)
+    pthread_mutex_unlock(&mutex_unfinished);
+
+  pthread_mutex_unlock(&mutex_global);
+  return NULL;
+}
+
+void stm_run_all_transactions(run_transaction_t run_transaction,
+                              void *initial_transaction,
+                              long num_threads)
+{
+  int i;
+  g_run_transaction = run_transaction;
+  g_first_transaction = initial_transaction;
+  g_last_transaction = initial_transaction;
+  g_num_threads = (int)num_threads;
+  g_num_waiting_threads = 0;
+
+  pthread_mutex_lock(&mutex_unfinished);
+
+  for (i=0; i<(int)num_threads; i++)
+    {
+      pthread_t th;
+      int status = pthread_create(&th, NULL, run_thread, NULL);
+      assert(status == 0);
+      pthread_detach(th);
+    }
+
+  pthread_mutex_lock(&mutex_unfinished);
+  pthread_mutex_unlock(&mutex_unfinished);
+}
diff --git a/pypy/translator/stm/stmgcintf.py b/pypy/translator/stm/stmgcintf.py
--- a/pypy/translator/stm/stmgcintf.py
+++ b/pypy/translator/stm/stmgcintf.py
@@ -35,19 +35,22 @@
                          '8f': rffi.DOUBLE,
                          '4f': rffi.FLOAT}
 
-    CALLBACK_TX   = lltype.Ptr(lltype.FuncType([rffi.VOIDP, lltype.Signed],
-                                               rffi.VOIDP))
-    CALLBACK_ENUM = lltype.Ptr(lltype.FuncType([llmemory.Address] * 3,
-                                               lltype.Void))
-    GETSIZE  = lltype.Ptr(lltype.FuncType([llmemory.Address], lltype.Signed))
+    RUN_TRANSACTION = lltype.Ptr(lltype.FuncType([rffi.VOIDP, lltype.Signed],
+                                                 rffi.VOIDP))
 
     def _freeze_(self):
         return True
 
-    setup_size_getter = smexternal('stm_setup_size_getter', [GETSIZE],
-                                   lltype.Void)
+    in_transaction = smexternal('stm_in_transaction', [], lltype.Signed)
+    run_all_transactions = smexternal('stm_run_all_transactions',
+                                      [rffi.VOIDP, lltype.Signed],
+                                      lltype.Void)
 
-    in_transaction = smexternal('stm_in_transaction', [], lltype.Signed)
+
+
+    # xxx review and delete xxx
+    CALLBACK_ENUM = lltype.Ptr(lltype.FuncType([llmemory.Address] * 3,
+                                               lltype.Void))
     _activate_transaction = smexternal('_stm_activate_transaction',
                                        [lltype.Signed], lltype.Void)
 
@@ -56,11 +59,6 @@
     get_tls = smexternal('stm_get_tls', [], llmemory.Address)
     del_tls = smexternal('stm_del_tls', [], lltype.Void)
 
-    enter_transactional_mode = smexternal('stm_enter_transactional_mode',
-                                          [], lltype.Void)
-    leave_transactional_mode = smexternal('stm_leave_transactional_mode',
-                                          [], lltype.Void)
-
     tldict_lookup = smexternal('stm_tldict_lookup', [llmemory.Address],
                                llmemory.Address)
     tldict_add = smexternal('stm_tldict_add', [llmemory.Address] * 2,
@@ -79,15 +77,5 @@
                                                lltype.Void)
 
     try_inevitable = smexternal('stm_try_inevitable', [], lltype.Void)
-    perform_transaction = smexternal('stm_perform_transaction',
-                                     [CALLBACK_TX, rffi.VOIDP,
-                                      llmemory.Address], rffi.VOIDP)
     thread_id        = smexternal('stm_thread_id',       [], lltype.Signed)
     abort_and_retry  = smexternal('stm_abort_and_retry', [], lltype.Void)
-
-    _debug_get_state = smexternal('stm_debug_get_state', [], lltype.Signed)
-    STATE_NOT_INITIALIZED   = -2
-    STATE_MAIN_THREAD       = -1
-    STATE_INACTIVE          = 0
-    STATE_ACTIVE            = 1
-    STATE_ACTIVE_INEVITABLE = 2

From noreply at buildbot.pypy.org  Sun Apr 22 18:17:40 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Apr 2012 18:17:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: cpyext: implement PyList_GetSlice
Message-ID: <20120422161740.93C808203C@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54617:866e71a3a373
Date: 2012-04-19 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/866e71a3a373/

Log:	cpyext: implement PyList_GetSlice

diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
     space.call_method(w_list, "reverse")
     return 0
 
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+    """Return a list of the objects in list containing the objects between low
+    and high.  Return NULL and set an exception if unsuccessful.  Analogous
+    to list[low:high].  Negative indices, as when slicing from Python, are not
+    supported."""
+    w_start = space.wrap(low)
+    w_stop = space.wrap(high)
+    return space.getslice(w_list, w_start, w_stop)
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyList_SetSlice(space, w_list, low, high, w_sequence):
     """Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
-    """Return a list of the objects in list containing the objects between low
-    and high.  Return NULL and set an exception if unsuccessful.  Analogous
-    to list[low:high].  Negative indices, as when slicing from Python, are not
-    supported.
-
-    This function used an int for low and high. This might
-    require changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([Py_ssize_t], PyObject)
 def PyLong_FromSsize_t(space, v):
     """Return a new PyLongObject object from a C Py_ssize_t, or
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
         w_t = api.PyList_AsTuple(w_l)
         assert space.unwrap(w_t) == (3, 2, 1)
 
+    def test_list_getslice(self, space, api):
+        w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+        w_s = api.PyList_GetSlice(w_l, 1, 5)
+        assert space.unwrap(w_s) == [2, 1]
+
 class AppTestListObject(AppTestCpythonExtensionBase):
     def test_listobject(self):
         import sys

From noreply at buildbot.pypy.org  Sun Apr 22 18:17:41 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Apr 2012 18:17:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: cpyext: add PyObject_HashNotImplemented
Message-ID: <20120422161741.DC6098204C@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54618:2829e1f34ab8
Date: 2012-04-21 22:18 +0200
http://bitbucket.org/pypy/pypy/changeset/2829e1f34ab8/

Log:	cpyext: add PyObject_HashNotImplemented

diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
     This is the equivalent of the Python expression hash(o)."""
     return space.int_w(space.hash(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+    """Set a TypeError indicating that type(o) is not hashable and return -1.
+    This function receives special treatment when stored in a tp_hash slot,
+    allowing a type to explicitly indicate to the interpreter that it is not
+    hashable.
+    """
+    raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Dir(space, w_o):
     """This is equivalent to the Python expression dir(o), returning a (possibly

From noreply at buildbot.pypy.org  Sun Apr 22 18:17:43 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Apr 2012 18:17:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: cpyext: add PyString_AsDecodedObject
Message-ID: <20120422161743.252F18203C@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54619:696f2ce3a963
Date: 2012-04-22 17:27 +0200
http://bitbucket.org/pypy/pypy/changeset/696f2ce3a963/

Log:	cpyext: add PyString_AsDecodedObject

diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
         w_errors = space.wrap(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+    """Decode a string object by passing it to the codec registered
+    for encoding and return the result as Python object. encoding and
+    errors have the same meaning as the parameters of the same name in
+    the string encode() method.  The codec to be used is looked up
+    using the Python codec registry. Return NULL if an exception was
+    raised by the codec.
+
+    This function is not available in 3.x and does not have a PyBytes alias."""
+    if not PyString_Check(space, w_str):
+        PyErr_BadArgument(space)
+
+    w_encoding = w_errors = space.w_None
+    if encoding:
+        w_encoding = space.wrap(rffi.charp2str(encoding))
+    if errors:
+        w_errors = space.wrap(rffi.charp2str(errors))
+    return space.call_method(w_str, "decode", w_encoding, w_errors)
+
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
+    def test_AsDecodedObject(self, space, api):
+        w_str = space.wrap('caf\xe9')
+        encoding = rffi.str2charp("latin-1")
+        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        rffi.free_charp(encoding)
+        assert space.unwrap(w_res) == u"caf\xe9"
+
     def test_eq(self, space, api):
         assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
         assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))

From noreply at buildbot.pypy.org  Sun Apr 22 18:17:44 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Apr 2012 18:17:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: cleanup stubs.py
Message-ID: <20120422161744.5F9738203C@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54620:6ed8c6cc3d8a
Date: 2012-04-22 17:28 +0200
http://bitbucket.org/pypy/pypy/changeset/6ed8c6cc3d8a/

Log:	cleanup stubs.py

diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1595,15 +1595,6 @@
     for PyObject_Str()."""
     raise NotImplementedError
 
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
-    """Set a TypeError indicating that type(o) is not hashable and return -1.
-    This function receives special treatment when stored in a tp_hash slot,
-    allowing a type to explicitly indicate to the interpreter that it is not
-    hashable.
-    """
-    raise NotImplementedError
-
 @cpython_api([], PyFrameObject)
 def PyEval_GetFrame(space):
     """Return the current thread state's frame, which is NULL if no frame is
@@ -1726,17 +1717,6 @@
     changes in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
-    """Decode a string object by passing it to the codec registered for encoding and
-    return the result as Python object. encoding and errors have the same
-    meaning as the parameters of the same name in the string encode() method.
-    The codec to be used is looked up using the Python codec registry. Return NULL
-    if an exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Encode(space, s, size, encoding, errors):
     """Encode the char buffer of the given size by passing it to the codec

From noreply at buildbot.pypy.org  Sun Apr 22 18:17:45 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sun, 22 Apr 2012 18:17:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: cpyext: implement wrap_inquirypred() to
 respect the __nonzero__ slot.
Message-ID: <20120422161745.9CACA8203C@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54621:4e5b2e1949ae
Date: 2012-04-22 18:16 +0200
http://bitbucket.org/pypy/pypy/changeset/4e5b2e1949ae/

Log:	cpyext: implement wrap_inquirypred() to respect the __nonzero__
	slot.

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
-    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc)
@@ -60,6 +60,15 @@
     args_w = space.fixedview(w_args)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
 
+def wrap_inquirypred(space, w_self, w_args, func):
+    func_inquiry = rffi.cast(inquiry, func)
+    check_num_args(space, w_args, 0)
+    args_w = space.fixedview(w_args)
+    res = generic_cpy_call(space, func_inquiry, w_self)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
+    return space.wrap(bool(res))
+
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
     check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
+        
+    def test_bool(self):
+        module = self.import_extension('foo', [
+            ("newInt", "METH_VARARGS",
+             """
+                IntLikeObject *intObj;
+                long intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "i", &intval))
+                    return NULL;
+
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+                intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+                if (!intObj) {
+                    return NULL;
+                }
+
+                intObj->value = intval;
+                return (PyObject *)intObj;
+             """)],
+            """
+            typedef struct
+            {
+                PyObject_HEAD
+                int value;
+            } IntLikeObject;
+
+            static int
+            intlike_nb_nonzero(IntLikeObject *v)
+            {
+                if (v->value == -42) {
+                    PyErr_SetNone(PyExc_ValueError);
+                    return -1;
+                }
+                return v->value;
+            }
+
+            PyTypeObject IntLike_Type = {
+                PyObject_HEAD_INIT(0)
+                /*ob_size*/             0,
+                /*tp_name*/             "IntLike",
+                /*tp_basicsize*/        sizeof(IntLikeObject),
+            };
+            static PyNumberMethods intlike_as_number;
+            """)
+        assert not bool(module.newInt(0))
+        assert bool(module.newInt(1))
+        assert bool(module.newInt(-1))
+        raises(ValueError, bool, module.newInt(-42))

From noreply at buildbot.pypy.org  Sun Apr 22 19:28:14 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 19:28:14 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Extra test.
Message-ID: <20120422172814.1E19D8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54622:c9a25e2c4e18
Date: 2012-04-22 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/c9a25e2c4e18/

Log:	Extra test.

diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -1,4 +1,5 @@
 import random
+import py
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib import rstm
 
@@ -88,3 +89,14 @@
                 return [self, other]
     rstm.run_all_transactions(DoInOrder())
     assert seen != range(50) and sorted(seen) == range(50)
+
+def test_raise():
+    class MyException(Exception):
+        pass
+    class FooBar(rstm.Transaction):
+        def run(self):
+            raise MyException
+    class DoInOrder(rstm.Transaction):
+        def run(self):
+            return [FooBar() for i in range(10)]
+    py.test.raises(MyException, rstm.run_all_transactions, DoInOrder())

From noreply at buildbot.pypy.org  Sun Apr 22 19:28:15 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 19:28:15 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Progress
Message-ID: <20120422172815.62BAE8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54623:c3cbaae90b9e
Date: 2012-04-22 17:29 +0200
http://bitbucket.org/pypy/pypy/changeset/c3cbaae90b9e/

Log:	Progress

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -53,10 +53,17 @@
     # is no possibility of having a GC collection inbetween.
     keepalive_until_here(initial_transaction)
     #
-    # The following line causes the _run_transaction() function to be
+    # The following line causes the _stm_run_transaction() function to be
     # generated in the C source with a specific signature, where it
     # can be called by the C code.
-    llhelper(StmOperations.RUN_TRANSACTION, _run_transaction)
+    llop.nop(lltype.Void, llhelper(StmOperations.RUN_TRANSACTION,
+                                   _stm_run_transaction))
+    #
+    # The same about the _stm_thread_starting() and _stm_thread_stopping()
+    llop.nop(lltype.Void, llhelper(StmOperations.INIT_DONE,
+                                   _stm_thread_starting))
+    llop.nop(lltype.Void, llhelper(StmOperations.INIT_DONE,
+                                   _stm_thread_stopping))
     #
     # Tell the C code to run all transactions.
     ptr = _cast_transaction_to_voidp(initial_transaction)
@@ -128,7 +135,7 @@
 _transactionalstate = _TransactionalState()
 
 
-def _run_transaction(transactionptr, retry_counter):
+def _stm_run_transaction(transactionptr, retry_counter):
     #
     # Tell the GC we are starting a transaction
     llop.stm_start_transaction(lltype.Void)
@@ -185,3 +192,10 @@
         next = new_transactions[n]
         n -= 1
     return next
+
+
+def _stm_thread_starting():
+    llop.stm_thread_starting(lltype.Void)
+
+def _stm_thread_stopping():
+    llop.stm_thread_stopping(lltype.Void)
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -23,7 +23,7 @@
             r, transactionptr = self._pending.popitem()
             transaction = self.cast_voidp_to_transaction(transactionptr)
             transaction._next_transaction = None
-            nextptr = rstm._run_transaction(transactionptr, 0)
+            nextptr = rstm._stm_run_transaction(transactionptr, 0)
             next = self.cast_voidp_to_transaction(nextptr)
             while next is not None:
                 self._add(self.cast_transaction_to_voidp(next))
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -409,6 +409,8 @@
     'stm_normalize_global':   LLOp(),
     'stm_start_transaction':  LLOp(canrun=True, canmallocgc=True),
     'stm_stop_transaction':   LLOp(canrun=True, canmallocgc=True),
+    'stm_thread_starting':    LLOp(),
+    'stm_thread_stopping':    LLOp(),
 
     # __________ address operations __________
 
@@ -540,6 +542,7 @@
 
     'stack_current':        LLOp(sideeffects=False),
     'keepalive':            LLOp(),
+    'nop':                  LLOp(canrun=True),
     'same_as':              LLOp(canfold=True),
     'hint':                 LLOp(),
     'check_no_more_arg':    LLOp(canraise=(Exception,)),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -646,6 +646,9 @@
 def op_stm_leave_transactional_mode():
     pass
 
+def op_nop(x):
+    pass
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -98,9 +98,9 @@
         self.sharedarea = stmshared.StmGCSharedArea(self)
         self.transactional_mode = False
         #
-        def _get_size(obj):     # indirection to hide 'self'
+        def _stm_getsize(obj):     # indirection to hide 'self'
             return self.get_size(obj)
-        self._getsize_fn = _get_size
+        self._stm_getsize = _stm_getsize
         #
         ##for size, TYPE in PRIMITIVE_SIZES.items():
         ##    self.declare_reader(size, TYPE)
@@ -113,8 +113,11 @@
         # we implement differently anyway.  So directly call GCBase.setup().
         GCBase.setup(self)
         #
-        self.stm_operations.setup_size_getter(
-                llhelper(self.stm_operations.GETSIZE, self._getsize_fn))
+        # The following line causes the _stm_getsize() function to be
+        # generated in the C source with a specific signature, where it
+        # can be called by the C code.
+        llop.nop(lltype.Void, llhelper(self.stm_operations.GETSIZE,
+                                       self._stm_getsize))
         #
         self.sharedarea.setup()
         #
@@ -197,7 +200,7 @@
     def start_transaction(self):
         self.get_tls().start_transaction()
 
-    def commit_transaction(self):
+    def stop_transaction(self):
         self.get_tls().stop_transaction()
 
 
@@ -275,10 +278,9 @@
         @dont_inline
         def _stm_write_barrier_global(obj):
             tls = self.get_tls()
-            if not tls.in_transaction():
-                return obj # not in transaction: only when running the code
-                           # in _run_thread(), i.e. in sub-threads outside
-                           # transactions.  xxx statically detect this case?
+            if tls is self.main_thread_tls:
+                return obj        # not in transaction
+            #
             # we need to find or make a local copy
             hdr = self.header(obj)
             if hdr.tid & GCFLAG_WAS_COPIED == 0:
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -1,4 +1,5 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.annlowlevel import cast_instance_to_base_ptr, llhelper
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, base_ptr_lltype
 from pypy.rlib.objectmodel import we_are_translated, free_non_gc_object
@@ -104,7 +105,6 @@
         """Called on the main thread, just before spawning the other
         threads."""
         self.stop_transaction()
-        self.stm_operations.enter_transactional_mode()
 
     def leave_transactional_mode(self):
         """Restart using the main thread for mallocs."""
@@ -112,7 +112,6 @@
             for key, value in StmGCTLS.nontranslated_dict.items():
                 if value is not self:
                     del StmGCTLS.nontranslated_dict[key]
-        self.stm_operations.leave_transactional_mode()
         self.start_transaction()
 
     def start_transaction(self):
@@ -149,9 +148,6 @@
                   "local_nursery_is_empty: gc not running")
         return self.nursery_free == self.nursery_start
 
-    def in_transaction(self):
-        return bool(self.nursery_free)
-
     # ------------------------------------------------------------
 
     def local_collection(self, end_of_transaction=0):
@@ -190,7 +186,8 @@
         #
         # Also find the roots that are the local copy of GCFLAG_WAS_COPIED
         # objects.
-        self.collect_roots_from_tldict()
+        if not self.in_main_thread:
+            self.collect_roots_from_tldict()
         #
         # Now repeatedly follow objects until 'pending' is empty.
         self.collect_flush_pending()
@@ -427,11 +424,14 @@
             if not hasattr(self.stm_operations, 'tldict_enum'):
                 return
         CALLBACK = self.stm_operations.CALLBACK_ENUM
-        callback = llhelper(CALLBACK, StmGCTLS._enum_entries)
-        self.stm_operations.tldict_enum(callback)
+        llop.nop(lltype.Void, llhelper(CALLBACK, StmGCTLS._stm_enum_callback))
+        # The previous line causes the _stm_enum_callback() function to be
+        # generated in the C source with a specific signature, where it
+        # can be called by the C code.
+        self.stm_operations.tldict_enum()
 
     @staticmethod
-    def _enum_entries(tlsaddr, globalobj, localobj):
+    def _stm_enum_callback(tlsaddr, globalobj, localobj):
         self = StmGCTLS.cast_address_to_tls_object(tlsaddr)
         localhdr = self.gc.header(localobj)
         ll_assert(localhdr.version == globalobj,
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -13,21 +13,21 @@
 
     def _declare_functions(self, GCClass, getfn, s_gc, *args):
         #
-        def setup_thread(gc):
+        def thread_starting(gc):
             self.root_walker.allocate_shadow_stack()
             gc.setup_thread()
         #
-        def teardown_thread(gc):
+        def thread_stopping(gc):
             gc.teardown_thread()
             self.root_walker.free_shadow_stack()
         #
         super(StmFrameworkGCTransformer, self)._declare_functions(
             GCClass, getfn, s_gc, *args)
-        self.setup_secondary_thread_ptr = getfn(
-            setup_thread,
+        self.thread_starting_ptr = getfn(
+            thread_starting,
             [s_gc], annmodel.s_None)
-        self.teardown_thread_ptr = getfn(
-            teardown_thread,
+        self.thread_stopping_ptr = getfn(
+            thread_stopping,
             [s_gc], annmodel.s_None)
         self.stm_writebarrier_ptr = getfn(
             self.gcdata.gc.stm_writebarrier,
@@ -44,8 +44,8 @@
         self.stm_start_ptr = getfn(
             self.gcdata.gc.start_transaction.im_func,
             [s_gc], annmodel.s_None)
-        self.stm_commit_ptr = getfn(
-            self.gcdata.gc.commit_transaction.im_func,
+        self.stm_stop_ptr = getfn(
+            self.gcdata.gc.stop_transaction.im_func,
             [s_gc], annmodel.s_None)
 
     def build_root_walker(self):
@@ -62,12 +62,11 @@
                                  resulttype=llmemory.Address)
         hop.genop('adr_add', [v_gcdata_adr, c_ofs], resultvar=op.result)
 
-    def gct_stm_descriptor_init(self, hop):
-        hop.genop("direct_call", [self.setup_secondary_thread_ptr,
-                                  self.c_const_gc])
+    def gct_stm_thread_starting(self, hop):
+        hop.genop("direct_call", [self.thread_starting_ptr, self.c_const_gc])
 
-    def gct_stm_descriptor_done(self, hop):
-        hop.genop("direct_call", [self.teardown_thread_ptr, self.c_const_gc])
+    def gct_stm_thread_stopping(self, hop):
+        hop.genop("direct_call", [self.thread_stopping_ptr, self.c_const_gc])
 
     def gct_stm_enter_transactional_mode(self, hop):
         livevars = self.push_roots(hop)
@@ -104,9 +103,9 @@
         hop.genop("direct_call", [self.stm_start_ptr, self.c_const_gc])
         self.pop_roots(hop, livevars)
 
-    def gct_stm_commit_transaction(self, hop):
+    def gct_stm_stop_transaction(self, hop):
         livevars = self.push_roots(hop)
-        hop.genop("direct_call", [self.stm_commit_ptr, self.c_const_gc])
+        hop.genop("direct_call", [self.stm_stop_ptr, self.c_const_gc])
         self.pop_roots(hop, livevars)
 
 
diff --git a/pypy/translator/c/src/mem.h b/pypy/translator/c/src/mem.h
--- a/pypy/translator/c/src/mem.h
+++ b/pypy/translator/c/src/mem.h
@@ -91,6 +91,9 @@
 #endif
 
 
+#define OP_NOP(r, x)         /* nothing */
+
+
 /* used by pypy.rlib.rstack, but also by asmgcc */
 #define OP_STACK_CURRENT(r)  r = (Signed)&r
 
diff --git a/pypy/translator/stm/src_stm/core.c b/pypy/translator/stm/src_stm/core.c
--- a/pypy/translator/stm/src_stm/core.c
+++ b/pypy/translator/stm/src_stm/core.c
@@ -6,7 +6,6 @@
 #define SPINLOOP_REASONS 10
 
 struct tx_descriptor {
-  void *rpython_tls_object;
   jmp_buf *setjmp_buf;
   owner_version_t start_time;
   owner_version_t end_time;
@@ -24,8 +23,6 @@
    if there is an inevitable transaction running */
 static volatile unsigned long global_timestamp = 2;
 static __thread struct tx_descriptor *thread_descriptor = NULL;
-static __thread struct tx_descriptor *active_thread_descriptor = NULL;
-static long (*rpython_get_size)(void*);
 
 /************************************************************/
 
@@ -58,7 +55,7 @@
 {
   unsigned int c;
   int i;
-  struct tx_descriptor *d = active_thread_descriptor;
+  struct tx_descriptor *d = thread_descriptor;
   d->num_spinloops[num]++;
 
   //printf("tx_spinloop(%d)\n", num);
@@ -90,7 +87,7 @@
     {
       void *globalobj = item->addr;
       void *localobj = item->val;
-      long size = rpython_get_size(localobj);
+      long size = pypy_g__stm_getsize(localobj);
       memcpy(((char *)globalobj) + sizeof(orec_t),
              ((char *)localobj) + sizeof(orec_t),
              size - sizeof(orec_t));
@@ -198,7 +195,7 @@
 /*** increase the abort count and restart the transaction */
 static void tx_abort(int reason)
 {
-  struct tx_descriptor *d = active_thread_descriptor;
+  struct tx_descriptor *d = thread_descriptor;
   assert(!is_inevitable(d));
   d->num_aborts[reason]++;
 #ifdef RPY_STM_DEBUG_PRINT
@@ -397,14 +394,13 @@
 #define STM_READ_WORD(SIZE, SUFFIX, TYPE)                               \
 TYPE stm_read_int##SIZE##SUFFIX(void* addr, long offset)                \
 {                                                                       \
-  struct tx_descriptor *d = active_thread_descriptor;                   \
+  struct tx_descriptor *d = thread_descriptor;                          \
   volatile orec_t *o = get_orec(addr);                                  \
   owner_version_t ovt;                                                  \
                                                                         \
   assert(sizeof(TYPE) == SIZE);                                         \
   /* XXX try to remove this check from the main path:           */      \
-  /*     d is NULL only when in non-main threads but            */      \
-  /*     outside a transaction.                                 */      \
+  /*     d is NULL when running in the main thread              */      \
   if (d == NULL)                                                        \
     return *(TYPE *)(((char *)addr) + offset);                          \
                                                                         \
@@ -434,7 +430,7 @@
 
 void stm_copy_transactional_to_raw(void *src, void *dst, long size)
 {
-  struct tx_descriptor *d = active_thread_descriptor;
+  struct tx_descriptor *d = thread_descriptor;
   volatile orec_t *o = get_orec(src);
   owner_version_t ovt;
 
@@ -448,11 +444,14 @@
   STM_DO_READ(memcpy(dst, src, size));
 }
 
-static struct tx_descriptor *descriptor_init(long in_main_thread)
+static void descriptor_init(long in_main_thread)
 {
   assert(thread_descriptor == NULL);
-  assert(active_thread_descriptor == NULL);
-  if (1)  /* for hg diff */
+  if (in_main_thread)
+    {
+      /* the main thread doesn't have a thread_descriptor at all */
+    }
+  else
     {
       struct tx_descriptor *d = malloc(sizeof(struct tx_descriptor));
       memset(d, 0, sizeof(struct tx_descriptor));
@@ -461,40 +460,28 @@
       PYPY_DEBUG_START("stm-init");
 #endif
 
-      if (in_main_thread)
-        {
-          d->my_lock_word = 0;   /* special value for the main thread */
-        }
-      else
-        {
-          /* initialize 'my_lock_word' to be a unique negative number */
-          d->my_lock_word = (owner_version_t)d;
-          if (!IS_LOCKED(d->my_lock_word))
-            d->my_lock_word = ~d->my_lock_word;
-          assert(IS_LOCKED(d->my_lock_word));
-        }
-      /*d->spinloop_counter = (unsigned int)(d->my_lock_word | 1);*/
+      /* initialize 'my_lock_word' to be a unique negative number */
+      d->my_lock_word = (owner_version_t)d;
+      if (!IS_LOCKED(d->my_lock_word))
+        d->my_lock_word = ~d->my_lock_word;
+      assert(IS_LOCKED(d->my_lock_word));
 
       thread_descriptor = d;
-      if (in_main_thread)
-        ; //stm_leave_transactional_mode();
-      else
-        ;   /* active_thread_descriptor stays NULL */
 
 #ifdef RPY_STM_DEBUG_PRINT
-      if (PYPY_HAVE_DEBUG_PRINTS) fprintf(PYPY_DEBUG_FILE, "thread %lx starting\n",
-                                          (long)pthread_self());
+      if (PYPY_HAVE_DEBUG_PRINTS)
+        fprintf(PYPY_DEBUG_FILE, "thread %lx starting with id %lx\n",
+                (long)pthread_self(), (long)d->my_lock_word);
       PYPY_DEBUG_STOP("stm-init");
 #endif
-      return d;
     }
 }
 
 static void descriptor_done(void)
 {
   struct tx_descriptor *d = thread_descriptor;
-  assert(d != NULL);
-  assert(active_thread_descriptor == NULL);
+  if (d == NULL)
+    return;
 
   thread_descriptor = NULL;
 
@@ -535,17 +522,15 @@
 static void begin_transaction(jmp_buf* buf)
 {
   struct tx_descriptor *d = thread_descriptor;
-  /* you need to call descriptor_init() before calling
-     stm_perform_transaction() */
+  /* you need to call descriptor_init() before calling this */
   assert(d != NULL);
   d->setjmp_buf = buf;
   d->start_time = (/*d->last_known_global_timestamp*/ global_timestamp) & ~1;
-  active_thread_descriptor = d;
 }
 
-static long commit_transaction(void)
+static void commit_transaction(void)
 {
-  struct tx_descriptor *d = active_thread_descriptor;
+  struct tx_descriptor *d = thread_descriptor;
   assert(d != NULL);
 
   // if I don't have writes, I'm committed
@@ -560,12 +545,10 @@
         }
       d->num_commits++;
       common_cleanup(d);
-      active_thread_descriptor = NULL;
-      return d->start_time;
     }
 
   // bring that variable over to this CPU core (optimization, maybe)
-  global_timestamp;
+  /* global_timestamp; */
 
   // acquire locks
   acquireLocks(d);
@@ -606,8 +589,6 @@
 
   // reset all lists
   common_cleanup(d);
-  active_thread_descriptor = NULL;
-  return d->end_time;
 }
 
 void stm_try_inevitable(STM_CCHARP1(why))
@@ -616,14 +597,11 @@
      global_timestamp and global_timestamp cannot be incremented
      by another thread.  We set the lowest bit in global_timestamp
      to 1. */
-  struct tx_descriptor *d = active_thread_descriptor;
+  struct tx_descriptor *d = thread_descriptor;
   if (d == NULL)
-    return;
-
-  if (is_inevitable(d))   /* also when the transaction is inactive */
-    {
-      return;  /* I am already inevitable */
-    }
+    return;  /* I am running in the main thread */
+  if (is_inevitable(d))
+    return;  /* I am already inevitable */
 
 #ifdef RPY_STM_DEBUG_PRINT
   PYPY_DEBUG_START("stm-inevitable");
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -1,6 +1,6 @@
 /*** Extendable Timestamps
  *
- * This is a C version of rstm_r5/stm/et.hpp.
+ * This is a (heavily modified) C version of rstm_r5/stm/et.hpp.
  * See http://www.cs.rochester.edu/research/synchronization/rstm/api.shtml
  *
  */
@@ -12,20 +12,22 @@
 #include "src/commondefs.h"
 
 
+/* see comments in ../stmgcintf.py */
 long stm_in_transaction(void);
-void stm_run_all_transactions(void*(*)(void*, long), void*, long);
-
-
-
-void stm_setup_size_getter(long(*)(void*));
+void stm_run_all_transactions(void*, long);
 
 void stm_set_tls(void *, long);
 void *stm_get_tls(void);
 void stm_del_tls(void);
+long stm_thread_id(void);
 
 void *stm_tldict_lookup(void *);
 void stm_tldict_add(void *, void *);
-void stm_tldict_enum(void(*)(void*, void*, void*));
+void stm_tldict_enum(void);
+
+/* these functions are declared by generated C code from the GC */
+extern long pypy_g__stm_getsize(void *);
+extern void pypy_g__stm_enum_callback(void *, void *, void *);
 
 char      stm_read_int1(void *, long);
 short     stm_read_int2(void *, long);
@@ -51,8 +53,6 @@
                                      0: not in a transaction
                                      1: in a regular transaction
                                      2: in an inevitable transaction */
-long stm_thread_id(void);  /* returns a unique thread id,
-                              or 0 if descriptor_init() was not called */
 void _stm_activate_transaction(long);
 
 void stm_copy_transactional_to_raw(void *src, void *dst, long size);
diff --git a/pypy/translator/stm/src_stm/rpyintf.c b/pypy/translator/stm/src_stm/rpyintf.c
--- a/pypy/translator/stm/src_stm/rpyintf.c
+++ b/pypy/translator/stm/src_stm/rpyintf.c
@@ -4,19 +4,21 @@
 {
   struct tx_descriptor *d = thread_descriptor;
   if (d == NULL)
-    return 0;    /* no thread_descriptor yet, assume it's the main thread */
+    return 0;    /* no thread_descriptor: it's the main thread */
   return d->my_lock_word;
 }
 
+static __thread void *rpython_tls_object;
+
 void stm_set_tls(void *newtls, long in_main_thread)
 {
-  struct tx_descriptor *d = descriptor_init(in_main_thread);
-  d->rpython_tls_object = newtls;
+  descriptor_init(in_main_thread);
+  rpython_tls_object = newtls;
 }
 
 void *stm_get_tls(void)
 {
-  return thread_descriptor->rpython_tls_object;
+  return rpython_tls_object;
 }
 
 void stm_del_tls(void)
@@ -37,12 +39,12 @@
 
 void stm_tldict_add(void *key, void *value)
 {
-  struct tx_descriptor *d = active_thread_descriptor;
+  struct tx_descriptor *d = thread_descriptor;
   assert(d != NULL);
   redolog_insert(&d->redolog, key, value);
 }
 
-void stm_tldict_enum(void(*callback)(void*, void*, void*))
+void stm_tldict_enum(void)
 {
   struct tx_descriptor *d = thread_descriptor;
   wlog_t *item;
@@ -50,40 +52,17 @@
 
   REDOLOG_LOOP_FORWARD(d->redolog, item)
     {
-      callback(tls, item->addr, item->val);
+      pypy_g__stm_enum_callback(tls, item->addr, item->val);
     } REDOLOG_LOOP_END;
 }
 
-void stm_setup_size_getter(long(*getsize_fn)(void*))
-{
-  rpython_get_size = getsize_fn;
-}
-
 long stm_in_transaction(void)
 {
   struct tx_descriptor *d = thread_descriptor;
   return d != NULL;
 }
 
-void _stm_activate_transaction(long activate)
-{
-  assert(thread_descriptor != NULL);
-  if (activate)
-    {
-      active_thread_descriptor = thread_descriptor;
-    }
-  else
-    {
-      active_thread_descriptor = NULL;
-    }
-}
-
-
-/* a helper to directly read the field '_next_transaction' on
-   RPython instances of pypy.rlib.rstm.Transaction */
-static void *next_transaction(void *
-
-
+/************************************************************/
 
 /* this lock is acquired when we start running transactions, and
    released only when we are finished. */
@@ -94,11 +73,8 @@
 static pthread_mutex_t mutex_global = PTHREAD_MUTEX_INITIALIZER;
 
 /* some global data put there by run_all_transactions(). */
-typedef void *(*run_transaction_t)(void *, long);
-static run_transaction_t g_run_transaction;
 static void *g_first_transaction, *g_last_transaction;
-static int g_num_threads;
-static int g_num_waiting_threads;
+static int g_num_threads, g_num_waiting_threads;
 
 /* the main function running a thread */
 static void *run_thread(void *ignored)
@@ -113,12 +89,10 @@
   return NULL;
 }
 
-void stm_run_all_transactions(run_transaction_t run_transaction,
-                              void *initial_transaction,
+void stm_run_all_transactions(void *initial_transaction,
                               long num_threads)
 {
   int i;
-  g_run_transaction = run_transaction;
   g_first_transaction = initial_transaction;
   g_last_transaction = initial_transaction;
   g_num_threads = (int)num_threads;
diff --git a/pypy/translator/stm/stmgcintf.py b/pypy/translator/stm/stmgcintf.py
--- a/pypy/translator/stm/stmgcintf.py
+++ b/pypy/translator/stm/stmgcintf.py
@@ -35,47 +35,58 @@
                          '8f': rffi.DOUBLE,
                          '4f': rffi.FLOAT}
 
+    INIT_DONE       = lltype.Ptr(lltype.FuncType([], lltype.Void))
     RUN_TRANSACTION = lltype.Ptr(lltype.FuncType([rffi.VOIDP, lltype.Signed],
                                                  rffi.VOIDP))
+    GETSIZE         = lltype.Ptr(lltype.FuncType([llmemory.Address],
+                                                 lltype.Signed))
+    CALLBACK_ENUM   = lltype.Ptr(lltype.FuncType([llmemory.Address]*3,
+                                                 lltype.Void))
 
     def _freeze_(self):
         return True
 
+    # C part of the implementation of the pypy.rlib.rstm module
     in_transaction = smexternal('stm_in_transaction', [], lltype.Signed)
     run_all_transactions = smexternal('stm_run_all_transactions',
                                       [rffi.VOIDP, lltype.Signed],
                                       lltype.Void)
 
-
-
-    # xxx review and delete xxx
-    CALLBACK_ENUM = lltype.Ptr(lltype.FuncType([llmemory.Address] * 3,
-                                               lltype.Void))
-    _activate_transaction = smexternal('_stm_activate_transaction',
-                                       [lltype.Signed], lltype.Void)
-
+    # for the GC: store and read a thread-local-storage field, as well
+    # as initialize and shut down the internal thread_descriptor
     set_tls = smexternal('stm_set_tls', [llmemory.Address, lltype.Signed],
                          lltype.Void)
     get_tls = smexternal('stm_get_tls', [], llmemory.Address)
     del_tls = smexternal('stm_del_tls', [], lltype.Void)
 
+    # return the current thread id (a random non-null number, or 0 for
+    # the main thread)
+    thread_id = smexternal('stm_thread_id', [], lltype.Signed)
+
+    # lookup, add, and enumerate the content of the internal dictionary
+    # that maps GLOBAL objects to LOCAL objects
     tldict_lookup = smexternal('stm_tldict_lookup', [llmemory.Address],
                                llmemory.Address)
     tldict_add = smexternal('stm_tldict_add', [llmemory.Address] * 2,
                             lltype.Void)
-    tldict_enum = smexternal('stm_tldict_enum', [CALLBACK_ENUM], lltype.Void)
+    tldict_enum = smexternal('stm_tldict_enum', [], lltype.Void)
 
+    # reader functions, to call if the object is GLOBAL
     for _size, _TYPE in PRIMITIVE_SIZES.items():
         _name = 'stm_read_int%s' % _size
         locals()[_name] = smexternal(_name, [llmemory.Address, lltype.Signed],
                                      _TYPE)
 
+    # a special reader function that copies all the content of the object
+    # somewhere else
     stm_copy_transactional_to_raw = smexternal('stm_copy_transactional_to_raw',
                                                [llmemory.Address,
                                                 llmemory.Address,
                                                 lltype.Signed],
                                                lltype.Void)
 
+    # if running in a transaction, make it inevitable
     try_inevitable = smexternal('stm_try_inevitable', [], lltype.Void)
-    thread_id        = smexternal('stm_thread_id',       [], lltype.Signed)
+
+    # for testing
     abort_and_retry  = smexternal('stm_abort_and_retry', [], lltype.Void)

From noreply at buildbot.pypy.org  Sun Apr 22 19:28:16 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 19:28:16 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Progress...
Message-ID: <20120422172816.A17B78203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54624:0a022231bea0
Date: 2012-04-22 19:10 +0200
http://bitbucket.org/pypy/pypy/changeset/0a022231bea0/

Log:	Progress...

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -171,8 +171,7 @@
     # This logic is in a sub-function because we want to catch
     # the MemoryErrors that could occur.
     transaction = _cast_voidp_to_transaction(transactionptr)
-    ll_assert(transaction._next_transaction is None,
-              "_next_transaction should be cleared by C code")
+    transaction._next_transaction = None
     transaction.retry_counter = retry_counter
     new_transactions = transaction.run()
     return _link_new_transactions(new_transactions)
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -21,8 +21,6 @@
         self._add(initial_transaction_ptr)
         while self._pending:
             r, transactionptr = self._pending.popitem()
-            transaction = self.cast_voidp_to_transaction(transactionptr)
-            transaction._next_transaction = None
             nextptr = rstm._stm_run_transaction(transactionptr, 0)
             next = self.cast_voidp_to_transaction(nextptr)
             while next is not None:
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -15,6 +15,10 @@
 
 #ifndef PYPY_NOT_MAIN_FILE
 
+#ifdef RPY_STM
+#include "src_stm/et.c"
+#endif
+
 #ifndef PYPY_MAIN_FUNCTION
 #define PYPY_MAIN_FUNCTION main
 #endif
diff --git a/pypy/translator/stm/src_stm/core.c b/pypy/translator/stm/src_stm/core.c
--- a/pypy/translator/stm/src_stm/core.c
+++ b/pypy/translator/stm/src_stm/core.c
@@ -26,6 +26,11 @@
 
 /************************************************************/
 
+#define GETVERSION(o)     ((owner_version_t)((o)->h_version))
+#define GETVERSIONREF(o)  ((volatile owner_version_t *)(&(o)->h_version))
+#define SETVERSION(o, v)  (o)->h_version = (void *)(v)
+#define GETTID(o)         ((o)->h_tid)
+
 static unsigned long get_global_timestamp(struct tx_descriptor *d)
 {
   return (/*d->last_known_global_timestamp =*/ global_timestamp);
@@ -94,7 +99,7 @@
       /* unlock the orec */
       volatile orec_t* o = get_orec(globalobj);
       CFENCE;
-      o->version = newver;
+      SETVERSION(o, newver);
     } REDOLOG_LOOP_END;
 }
 
@@ -107,7 +112,7 @@
       if (item->p != -1)
         {
           volatile orec_t* o = get_orec(item->addr);
-          o->version = item->p;
+          SETVERSION(o, item->p);
         }
     } REDOLOG_LOOP_END;
 }
@@ -120,7 +125,7 @@
     {
       volatile orec_t* o = get_orec(item->addr);
       assert(item->p != -1);
-      o->version = item->p;
+      SETVERSION(o, item->p);
       item->p = -1;
     } REDOLOG_LOOP_END;
 }
@@ -137,7 +142,7 @@
       owner_version_t ovt;
 
     retry:
-      ovt = o->version;
+      ovt = GETVERSION(o);
 
       // if orec not locked, lock it
       //
@@ -145,7 +150,7 @@
       // reads.  Since most writes are also reads, we'll just abort under this
       // condition.  This can introduce false conflicts
       if (!IS_LOCKED_OR_NEWER(ovt, d->start_time)) {
-        if (!bool_cas(&o->version, ovt, d->my_lock_word))
+        if (!bool_cas(GETVERSIONREF(o), ovt, d->my_lock_word))
           goto retry;
         // save old version to item->p.  Now we hold the lock.
         item->p = ovt;
@@ -219,7 +224,7 @@
   for (i=0; i<d->reads.size; i++)
     {
     retry:
-      ovt = d->reads.items[i]->version;
+      ovt = GETVERSION(d->reads.items[i]);
       if (IS_LOCKED_OR_NEWER(ovt, d->start_time))
         {
           // If locked, we wait until it becomes unlocked.  The chances are
@@ -249,7 +254,7 @@
   assert(!is_inevitable(d));
   for (i=0; i<d->reads.size; i++)
     {
-      ovt = d->reads.items[i]->version;      // read this orec
+      ovt = GETVERSION(d->reads.items[i]);      // read this orec
       if (IS_LOCKED_OR_NEWER(ovt, d->start_time))
         {
           if (!IS_LOCKED(ovt))
@@ -361,7 +366,7 @@
   else {                                                                \
  retry:                                                                 \
   /* read the orec BEFORE we read anything else */                      \
-  ovt = o->version;                                                     \
+  ovt = GETVERSION(o);                                                  \
   CFENCE;                                                               \
                                                                         \
   /* this tx doesn't hold any locks, so if the lock for this addr is */ \
@@ -384,7 +389,7 @@
                                                                         \
   /* postvalidate AFTER reading addr: */                                \
   CFENCE;                                                               \
-  if (__builtin_expect(o->version != ovt, 0))                           \
+  if (__builtin_expect(GETVERSION(o) != ovt, 0))                        \
     goto retry;       /* oups, try again */                             \
                                                                         \
   oreclist_insert(&d->reads, (orec_t*)o);                               \
@@ -404,13 +409,13 @@
   if (d == NULL)                                                        \
     return *(TYPE *)(((char *)addr) + offset);                          \
                                                                         \
-  if ((o->tid & GCFLAG_WAS_COPIED) != 0)                                \
+  if ((GETTID(o) & GCFLAG_WAS_COPIED) != 0)                             \
     {                                                                   \
       /* Look up in the thread-local dictionary. */                     \
       wlog_t *found;                                                    \
       REDOLOG_FIND(d->redolog, addr, found, goto not_found);            \
       orec_t *localobj = (orec_t *)found->val;                          \
-      assert((localobj->tid & GCFLAG_GLOBAL) == 0);                     \
+      assert((GETTID(localobj) & GCFLAG_GLOBAL) == 0);                  \
       return *(TYPE *)(((char *)localobj) + offset);                    \
                                                                         \
     not_found:;                                                         \
@@ -644,3 +649,72 @@
 {
   tx_abort(7);     /* manual abort */
 }
+
+/************************************************************/
+
+long stm_thread_id(void)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  if (d == NULL)
+    return 0;    /* no thread_descriptor: it's the main thread */
+  return d->my_lock_word;
+}
+
+static __thread void *rpython_tls_object;
+
+void stm_set_tls(void *newtls, long in_main_thread)
+{
+  descriptor_init(in_main_thread);
+  rpython_tls_object = newtls;
+}
+
+void *stm_get_tls(void)
+{
+  return rpython_tls_object;
+}
+
+void stm_del_tls(void)
+{
+  descriptor_done();
+}
+
+void *stm_tldict_lookup(void *key)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  wlog_t* found;
+  REDOLOG_FIND(d->redolog, key, found, goto not_found);
+  return found->val;
+
+ not_found:
+  return NULL;
+}
+
+void stm_tldict_add(void *key, void *value)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  assert(d != NULL);
+  redolog_insert(&d->redolog, key, value);
+}
+
+void stm_tldict_enum(void)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  wlog_t *item;
+  void *tls = stm_get_tls();
+
+  REDOLOG_LOOP_FORWARD(d->redolog, item)
+    {
+      pypy_g__stm_enum_callback(tls, item->addr, item->val);
+    } REDOLOG_LOOP_END;
+}
+
+long stm_in_transaction(void)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  return d != NULL;
+}
+
+#undef GETVERSION
+#undef GETVERSIONREF
+#undef SETVERSION
+#undef GETTID
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -1,7 +1,5 @@
 /* -*- c-basic-offset: 2 -*- */
 
-#ifndef PYPY_NOT_MAIN_FILE
-
 /* XXX assumes that time never wraps around (in a 'long'), which may be
  * correct on 64-bit machines but not on 32-bit machines if the process
  * runs for long enough.
@@ -40,10 +38,7 @@
 /* This is the same as the object header structure HDR
  * declared in stmgc.py */
 
-typedef struct {
-  long tid;
-  long version;
-} orec_t;
+typedef struct pypy_header0 orec_t;
 
 /************************************************************/
 
@@ -62,5 +57,3 @@
 #include "src_stm/rpyintf.c"
 
 /************************************************************/
-
-#endif  /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -25,7 +25,11 @@
 void stm_tldict_add(void *, void *);
 void stm_tldict_enum(void);
 
-/* these functions are declared by generated C code from the GC */
+/* these functions are declared by generated C code from pypy.rlib.rstm
+   and from the GC (see llop.nop(...)) */
+extern void pypy_g__stm_thread_starting(void);
+extern void pypy_g__stm_thread_stopping(void);
+extern void *_stm_run_transaction(void *, long);
 extern long pypy_g__stm_getsize(void *);
 extern void pypy_g__stm_enum_callback(void *, void *, void *);
 
diff --git a/pypy/translator/stm/src_stm/fifo.c b/pypy/translator/stm/src_stm/fifo.c
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/src_stm/fifo.c
@@ -0,0 +1,70 @@
+/* -*- c-basic-offset: 2 -*- */
+
+
+/* xxx Direct access to this field.  Relies on genc producing always the
+   same names, but that should be ok. */
+#define NEXT(item)  (((struct pypy_pypy_rlib_rstm_Transaction0 *)(item)) \
+                     ->t_inst__next_transaction)
+
+
+typedef struct {
+    void *first;
+    void *last;
+} stm_fifo_t;
+
+
+static void fifo_init(stm_fifo_t *fifo)
+{
+  fifo->first = NULL;
+  fifo->last = NULL;
+}
+
+static void *fifo_next(void *item)
+{
+  return NEXT(item);
+}
+
+static void fifo_append(stm_fifo_t *fifo, void *newitem)
+{
+  NEXT(newitem) = NULL;
+  if (fifo->last == NULL)
+    fifo->first = newitem;
+  else
+    NEXT(fifo->last) = newitem;
+  fifo->last = newitem;
+}
+
+static bool_t fifo_is_empty(stm_fifo_t *fifo)
+{
+  assert((fifo->first == NULL) == (fifo->last == NULL));
+  return (fifo->first == NULL);
+}
+
+/* static bool_t fifo_is_of_length_1(stm_fifo_t *fifo) */
+/* { */
+/*   return (fifo->first != NULL && fifo->first == fifo->last); */
+/* } */
+
+static void *fifo_popleft(stm_fifo_t *fifo)
+{
+  void *item = fifo->first;
+  fifo->first = NEXT(item);
+  if (fifo->first == NULL)
+    fifo->last = NULL;
+  return item;
+}
+
+static void fifo_extend(stm_fifo_t *fifo, void *newitems)
+{
+  if (fifo->last == NULL)
+    fifo->first = newitems;
+  else
+    NEXT(fifo->last) = newitems;
+
+  while (NEXT(newitems) != NULL)
+    newitems = NEXT(newitems);
+
+  fifo->last = newitems;
+}
+
+#undef NEXT
diff --git a/pypy/translator/stm/src_stm/rpyintf.c b/pypy/translator/stm/src_stm/rpyintf.c
--- a/pypy/translator/stm/src_stm/rpyintf.c
+++ b/pypy/translator/stm/src_stm/rpyintf.c
@@ -1,68 +1,7 @@
 /* -*- c-basic-offset: 2 -*- */
 
-long stm_thread_id(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  if (d == NULL)
-    return 0;    /* no thread_descriptor: it's the main thread */
-  return d->my_lock_word;
-}
+#include "src_stm/fifo.c"
 
-static __thread void *rpython_tls_object;
-
-void stm_set_tls(void *newtls, long in_main_thread)
-{
-  descriptor_init(in_main_thread);
-  rpython_tls_object = newtls;
-}
-
-void *stm_get_tls(void)
-{
-  return rpython_tls_object;
-}
-
-void stm_del_tls(void)
-{
-  descriptor_done();
-}
-
-void *stm_tldict_lookup(void *key)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  wlog_t* found;
-  REDOLOG_FIND(d->redolog, key, found, goto not_found);
-  return found->val;
-
- not_found:
-  return NULL;
-}
-
-void stm_tldict_add(void *key, void *value)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  assert(d != NULL);
-  redolog_insert(&d->redolog, key, value);
-}
-
-void stm_tldict_enum(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  wlog_t *item;
-  void *tls = stm_get_tls();
-
-  REDOLOG_LOOP_FORWARD(d->redolog, item)
-    {
-      pypy_g__stm_enum_callback(tls, item->addr, item->val);
-    } REDOLOG_LOOP_END;
-}
-
-long stm_in_transaction(void)
-{
-  struct tx_descriptor *d = thread_descriptor;
-  return d != NULL;
-}
-
-/************************************************************/
 
 /* this lock is acquired when we start running transactions, and
    released only when we are finished. */
@@ -72,19 +11,107 @@
    data in run_thread(). */
 static pthread_mutex_t mutex_global = PTHREAD_MUTEX_INITIALIZER;
 
+/* this lock is acquired if and only if there are no tasks pending,
+   i.e. the linked list stm_g_first_transaction ... stm_g_last_transaction is
+   empty and both pointers are NULL. */
+static pthread_mutex_t mutex_no_tasks_pending = PTHREAD_MUTEX_INITIALIZER;
+
 /* some global data put there by run_all_transactions(). */
-static void *g_first_transaction, *g_last_transaction;
-static int g_num_threads, g_num_waiting_threads;
+static stm_fifo_t stm_g_pending;
+static int stm_g_num_threads, stm_g_num_waiting_threads, stm_g_finished;
+
+
+static void* perform_transaction(void *transaction)
+{
+  void *new_transaction_list;
+  jmp_buf _jmpbuf;
+  long counter;
+  volatile long v_counter = 0;
+
+  setjmp(_jmpbuf);
+
+  begin_transaction(&_jmpbuf);
+
+  counter = v_counter;
+  v_counter = counter + 1;
+
+  new_transaction_list = pypy_g__stm_run_transaction(transaction, counter);
+
+  commit_transaction();
+
+  return new_transaction_list;
+}
+
+static void add_list(void *new_transaction_list)
+{
+  bool_t was_empty;
+
+  if (new_transaction_list == NULL)
+    return;
+
+  was_empty = fifo_is_empty(&stm_g_pending);
+  fifo_extend(&stm_g_pending, new_transaction_list);
+  if (was_empty)
+    pthread_mutex_unlock(&mutex_no_tasks_pending);
+}
+
 
 /* the main function running a thread */
 static void *run_thread(void *ignored)
 {
   pthread_mutex_lock(&mutex_global);
+  pypy_g__stm_thread_starting();
 
-  g_num_waiting_threads++;
-  if (g_num_waiting_threads == g_num_threads)
+  while (1)
+    {
+      if (fifo_is_empty(&stm_g_pending))
+        {
+          stm_g_num_waiting_threads += 1;
+          if (stm_g_num_waiting_threads == stm_g_num_threads)
+            {
+              stm_g_finished = 1;
+              pthread_mutex_unlock(&mutex_no_tasks_pending);
+            }
+          pthread_mutex_unlock(&mutex_global);
+
+          pthread_mutex_lock(&mutex_no_tasks_pending);
+          pthread_mutex_unlock(&mutex_no_tasks_pending);
+
+          pthread_mutex_lock(&mutex_global);
+          stm_g_num_waiting_threads -= 1;
+          if (stm_g_finished)
+            break;
+        }
+      else
+        {
+          void *new_transaction_list;
+          void *transaction = fifo_popleft(&stm_g_pending);
+          if (fifo_is_empty(&stm_g_pending))
+            pthread_mutex_lock(&mutex_no_tasks_pending);
+          pthread_mutex_unlock(&mutex_global);
+
+          while (1)
+            {
+              new_transaction_list = perform_transaction(transaction);
+              
+              /* for now, always break out of this loop,
+                 unless 'new_transaction_list' contains precisely one item */
+              if (new_transaction_list == NULL)
+                break;
+              if (fifo_next(new_transaction_list) != NULL)
+                break;
+
+              transaction = new_transaction_list;   /* single element */
+            }
+
+          pthread_mutex_lock(&mutex_global);
+          add_list(new_transaction_list);
+        }
+    }
+
+  pypy_g__stm_thread_stopping();
+  if (stm_g_num_waiting_threads == 0)   /* only the last thread to leave */
     pthread_mutex_unlock(&mutex_unfinished);
-
   pthread_mutex_unlock(&mutex_global);
   return NULL;
 }
@@ -93,10 +120,11 @@
                               long num_threads)
 {
   int i;
-  g_first_transaction = initial_transaction;
-  g_last_transaction = initial_transaction;
-  g_num_threads = (int)num_threads;
-  g_num_waiting_threads = 0;
+  fifo_init(&stm_g_pending);
+  fifo_append(&stm_g_pending, initial_transaction);
+  stm_g_num_threads = (int)num_threads;
+  stm_g_num_waiting_threads = 0;
+  stm_g_finished = 0;
 
   pthread_mutex_lock(&mutex_unfinished);
 
@@ -104,7 +132,12 @@
     {
       pthread_t th;
       int status = pthread_create(&th, NULL, run_thread, NULL);
-      assert(status == 0);
+      if (status != 0)
+        {
+          /* XXX turn into a nice exception */
+          fprintf(stderr, "fatal error: cannot create threads\n");
+          exit(1);
+        }
       pthread_detach(th);
     }
 
diff --git a/pypy/translator/stm/stmgcintf.py b/pypy/translator/stm/stmgcintf.py
--- a/pypy/translator/stm/stmgcintf.py
+++ b/pypy/translator/stm/stmgcintf.py
@@ -10,7 +10,7 @@
 
 eci = ExternalCompilationInfo(
     include_dirs = [cdir, cdir2],
-    includes = ['src_stm/et.h', 'src_stm/et.c'],
+    includes = ['src_stm/et.h'],
     pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT,
                         '#define RPY_STM 1'],
     separate_module_sources = ['\n'],    # hack for test_rffi_stm
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -66,7 +66,7 @@
         ll_assert(res is self.arg, "ERROR: bogus pointer equality")
         raw1 = rffi.cast(rffi.CCHARP, self.retry_counter)
         raw2 = rffi.cast(rffi.CCHARP, -1)
-        ll_assert(raw1 == raw2, "ERROR: retry_counter == -1")
+        ll_assert(raw1 != raw2, "ERROR: retry_counter == -1")
 
 class MakeChain(rstm.Transaction):
     def __init__(self, anchor, value):

From noreply at buildbot.pypy.org  Sun Apr 22 19:28:17 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 19:28:17 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix.
Message-ID: <20120422172817.DA2488203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54625:45ec64b5f8c8
Date: 2012-04-22 19:27 +0200
http://bitbucket.org/pypy/pypy/changeset/45ec64b5f8c8/

Log:	Fix.

diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -21,6 +21,10 @@
             gc.teardown_thread()
             self.root_walker.free_shadow_stack()
         #
+        def start_transaction(gc):
+            self.root_walker.start_transaction()
+            gc.start_transaction()
+        #
         super(StmFrameworkGCTransformer, self)._declare_functions(
             GCClass, getfn, s_gc, *args)
         self.thread_starting_ptr = getfn(
@@ -42,7 +46,7 @@
             self.gcdata.gc.leave_transactional_mode.im_func,
             [s_gc], annmodel.s_None)
         self.stm_start_ptr = getfn(
-            self.gcdata.gc.start_transaction.im_func,
+            start_transaction,
             [s_gc], annmodel.s_None)
         self.stm_stop_ptr = getfn(
             self.gcdata.gc.stop_transaction.im_func,
@@ -172,6 +176,12 @@
         base = self.stackgcdata.root_stack_base
         llmemory.raw_free(base)
 
+    def start_transaction(self):
+        # When a transaction is aborted, it leaves behind its shadow
+        # stack content.  We have to clear it here.
+        stackgcdata = self.stackgcdata
+        stackgcdata.root_stack_top = stackgcdata.root_stack_base
+
     def walk_stack_roots(self, collect_stack_root):
         raise NotImplementedError
         # XXX only to walk the main thread's shadow stack, so far
diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py
--- a/pypy/translator/stm/test/test_ztranslated.py
+++ b/pypy/translator/stm/test/test_ztranslated.py
@@ -8,7 +8,6 @@
     def test_hello_world(self):
         t, cbuilder = self.compile(targetdemo.entry_point)
         data, dataerr = cbuilder.cmdexec('4 5000', err=True)
-        assert 'done sleeping.' in dataerr
         assert 'check ok!' in data
 
     def test_bug1(self):

From noreply at buildbot.pypy.org  Sun Apr 22 22:32:45 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 22 Apr 2012 22:32:45 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: be more careful about closing open
	files
Message-ID: <20120422203245.73D118203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54626:36849667fd71
Date: 2012-04-22 23:27 +0300
http://bitbucket.org/pypy/pypy/changeset/36849667fd71/

Log:	be more careful about closing open files

diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -425,10 +425,16 @@
                 raise CompressionError("zlib module is not available")
             self.zlib = zlib
             self.crc = zlib.crc32("") & 0xffffffffL
-            if mode == "r":
-                self._init_read_gz()
-            else:
-                self._init_write_gz()
+            try:
+                if mode == "r":
+                    self._init_read_gz()
+                else:
+                    self._init_write_gz()
+            except:
+                if not self._extfileobj:
+                    fileobj.close()
+                    self.closed = True
+                raise    
 
         if comptype == "bz2":
             try:
@@ -1682,13 +1688,14 @@
 
             if filemode not in "rw":
                 raise ValueError("mode must be 'r' or 'w'")
-
-            t = cls(name, filemode,
-                    _Stream(name, filemode, comptype, fileobj, bufsize),
-                    **kwargs)
-            t._extfileobj = False
-            return t
-
+            fid = _Stream(name, filemode, comptype, fileobj, bufsize)
+            try:
+                t = cls(name, filemode, fid, **kwargs)
+                t._extfileobj = False
+                return t
+            except:
+                fid.close()
+                raise
         elif mode in "aw":
             return cls.taropen(name, mode, fileobj, **kwargs)
 
@@ -1715,13 +1722,18 @@
             gzip.GzipFile
         except (ImportError, AttributeError):
             raise CompressionError("gzip module is not available")
-
+        gz_fid = None
         try:
-            t = cls.taropen(name, mode,
-                gzip.GzipFile(name, mode, compresslevel, fileobj),
-                **kwargs)
+            gz_fid = gzip.GzipFile(name, mode, compresslevel, fileobj)
+            t = cls.taropen(name, mode, gz_fid, **kwargs)
         except IOError:
+            if gz_fid:
+                gz_fid.close()
             raise ReadError("not a gzip file")
+        except:
+            if gz_fid:
+                gz_fid.close()
+            raise 
         t._extfileobj = False
         return t
 
@@ -1738,15 +1750,21 @@
         except ImportError:
             raise CompressionError("bz2 module is not available")
 
-        if fileobj is not None:
-            fileobj = _BZ2Proxy(fileobj, mode)
-        else:
-            fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+        try:
+            if fileobj is not None:
+                bzfileobj = _BZ2Proxy(fileobj, mode)
+            else:
+                bzfileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+            t = cls.taropen(name, mode, bzfileobj, **kwargs)
 
-        try:
-            t = cls.taropen(name, mode, fileobj, **kwargs)
         except (IOError, EOFError):
+            if fileobj is None:
+                bzfileobj.close()
             raise ReadError("not a bzip2 file")
+        except:
+            if fileobj is None:
+                bzfileobj.close()
+            raise 
         t._extfileobj = False
         return t
 
diff --git a/lib-python/modified-2.7/test/test_tarfile.py b/lib-python/modified-2.7/test/test_tarfile.py
--- a/lib-python/modified-2.7/test/test_tarfile.py
+++ b/lib-python/modified-2.7/test/test_tarfile.py
@@ -340,7 +340,8 @@
         # constructor in case of an error. For the test we rely on
         # the fact that opening an empty file raises a ReadError.
         empty = os.path.join(TEMPDIR, "empty")
-        open(empty, "wb").write("")
+        with open(empty, "wb") as fid:
+            fid.write("")
 
         try:
             tar = object.__new__(tarfile.TarFile)

From noreply at buildbot.pypy.org  Sun Apr 22 22:32:46 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 22 Apr 2012 22:32:46 +0200 (CEST)
Subject: [pypy-commit] pypy win32-stdlib: merge default into branch
Message-ID: <20120422203246.BE0478204C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-stdlib
Changeset: r54627:000d3f9fd903
Date: 2012-04-22 23:28 +0300
http://bitbucket.org/pypy/pypy/changeset/000d3f9fd903/

Log:	merge default into branch

diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
     space.call_method(w_list, "reverse")
     return 0
 
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+    """Return a list of the objects in list containing the objects between low
+    and high.  Return NULL and set an exception if unsuccessful.  Analogous
+    to list[low:high].  Negative indices, as when slicing from Python, are not
+    supported."""
+    w_start = space.wrap(low)
+    w_stop = space.wrap(high)
+    return space.getslice(w_list, w_start, w_stop)
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyList_SetSlice(space, w_list, low, high, w_sequence):
     """Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
     This is the equivalent of the Python expression hash(o)."""
     return space.int_w(space.hash(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+    """Set a TypeError indicating that type(o) is not hashable and return -1.
+    This function receives special treatment when stored in a tp_hash slot,
+    allowing a type to explicitly indicate to the interpreter that it is not
+    hashable.
+    """
+    raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Dir(space, w_o):
     """This is equivalent to the Python expression dir(o), returning a (possibly
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
-    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc)
@@ -60,6 +60,15 @@
     args_w = space.fixedview(w_args)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
 
+def wrap_inquirypred(space, w_self, w_args, func):
+    func_inquiry = rffi.cast(inquiry, func)
+    check_num_args(space, w_args, 0)
+    args_w = space.fixedview(w_args)
+    res = generic_cpy_call(space, func_inquiry, w_self)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
+    return space.wrap(bool(res))
+
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
     check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
         w_errors = space.wrap(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+    """Decode a string object by passing it to the codec registered
+    for encoding and return the result as Python object. encoding and
+    errors have the same meaning as the parameters of the same name in
+    the string encode() method.  The codec to be used is looked up
+    using the Python codec registry. Return NULL if an exception was
+    raised by the codec.
+
+    This function is not available in 3.x and does not have a PyBytes alias."""
+    if not PyString_Check(space, w_str):
+        PyErr_BadArgument(space)
+
+    w_encoding = w_errors = space.w_None
+    if encoding:
+        w_encoding = space.wrap(rffi.charp2str(encoding))
+    if errors:
+        w_errors = space.wrap(rffi.charp2str(errors))
+    return space.call_method(w_str, "decode", w_encoding, w_errors)
+
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
-    """Return a list of the objects in list containing the objects between low
-    and high.  Return NULL and set an exception if unsuccessful.  Analogous
-    to list[low:high].  Negative indices, as when slicing from Python, are not
-    supported.
-
-    This function used an int for low and high. This might
-    require changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([Py_ssize_t], PyObject)
 def PyLong_FromSsize_t(space, v):
     """Return a new PyLongObject object from a C Py_ssize_t, or
@@ -1606,15 +1595,6 @@
     for PyObject_Str()."""
     raise NotImplementedError
 
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
-    """Set a TypeError indicating that type(o) is not hashable and return -1.
-    This function receives special treatment when stored in a tp_hash slot,
-    allowing a type to explicitly indicate to the interpreter that it is not
-    hashable.
-    """
-    raise NotImplementedError
-
 @cpython_api([], PyFrameObject)
 def PyEval_GetFrame(space):
     """Return the current thread state's frame, which is NULL if no frame is
@@ -1737,17 +1717,6 @@
     changes in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
-    """Decode a string object by passing it to the codec registered for encoding and
-    return the result as Python object. encoding and errors have the same
-    meaning as the parameters of the same name in the string encode() method.
-    The codec to be used is looked up using the Python codec registry. Return NULL
-    if an exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Encode(space, s, size, encoding, errors):
     """Encode the char buffer of the given size by passing it to the codec
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
         w_t = api.PyList_AsTuple(w_l)
         assert space.unwrap(w_t) == (3, 2, 1)
 
+    def test_list_getslice(self, space, api):
+        w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+        w_s = api.PyList_GetSlice(w_l, 1, 5)
+        assert space.unwrap(w_s) == [2, 1]
+
 class AppTestListObject(AppTestCpythonExtensionBase):
     def test_listobject(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
+    def test_AsDecodedObject(self, space, api):
+        w_str = space.wrap('caf\xe9')
+        encoding = rffi.str2charp("latin-1")
+        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        rffi.free_charp(encoding)
+        assert space.unwrap(w_res) == u"caf\xe9"
+
     def test_eq(self, space, api):
         assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
         assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
+        
+    def test_bool(self):
+        module = self.import_extension('foo', [
+            ("newInt", "METH_VARARGS",
+             """
+                IntLikeObject *intObj;
+                long intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "i", &intval))
+                    return NULL;
+
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+                intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+                if (!intObj) {
+                    return NULL;
+                }
+
+                intObj->value = intval;
+                return (PyObject *)intObj;
+             """)],
+            """
+            typedef struct
+            {
+                PyObject_HEAD
+                int value;
+            } IntLikeObject;
+
+            static int
+            intlike_nb_nonzero(IntLikeObject *v)
+            {
+                if (v->value == -42) {
+                    PyErr_SetNone(PyExc_ValueError);
+                    return -1;
+                }
+                return v->value;
+            }
+
+            PyTypeObject IntLike_Type = {
+                PyObject_HEAD_INIT(0)
+                /*ob_size*/             0,
+                /*tp_name*/             "IntLike",
+                /*tp_basicsize*/        sizeof(IntLikeObject),
+            };
+            static PyNumberMethods intlike_as_number;
+            """)
+        assert not bool(module.newInt(0))
+        assert bool(module.newInt(1))
+        assert bool(module.newInt(-1))
+        raises(ValueError, bool, module.newInt(-42))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -10,6 +10,7 @@
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
      scalar_w, W_NDimArray, array)
+from pypy.module.micronumpy.interp_arrayops import where
 from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize, instantiate
 
@@ -35,6 +36,7 @@
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring"]
 TWO_ARG_FUNCTIONS = ["dot", 'take']
+THREE_ARG_FUNCTIONS = ['where']
 
 class FakeSpace(object):
     w_ValueError = None
@@ -445,14 +447,25 @@
             arg = self.args[1].execute(interp)
             if not isinstance(arg, BaseArray):
                 raise ArgumentNotAnArray
-            if not isinstance(arg, BaseArray):
-                raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
             else:
                 assert False # unreachable code
+        elif self.name in THREE_ARG_FUNCTIONS:
+            if len(self.args) != 3:
+                raise ArgumentMismatch
+            arg1 = self.args[1].execute(interp)
+            arg2 = self.args[2].execute(interp)
+            if not isinstance(arg1, BaseArray):
+                raise ArgumentNotAnArray
+            if not isinstance(arg2, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "where":
+                w_res = where(interp.space, arr, arg1, arg2)
+            else:
+                assert False
         else:
             raise WrongFunctionName
         if isinstance(w_res, BaseArray):
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -270,3 +270,13 @@
         b -> 2
         """)
         assert interp.results[0].value == 3
+
+    def test_where(self):
+        interp = self.run('''
+        a = [1, 0, 3, 0]
+        b = [1, 1, 1, 1]
+        c = [0, 0, 0, 0]
+        d = where(a, b, c)
+        d -> 1
+        ''')
+        assert interp.results[0].value == 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1830,6 +1830,12 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
+    def test_array_indexing_bool_setitem_multidim(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])

From noreply at buildbot.pypy.org  Sun Apr 22 22:32:49 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 22 Apr 2012 22:32:49 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge win32-stdlib branch into
	branch
Message-ID: <20120422203249.7351D8205C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54628:0550049021e6
Date: 2012-04-22 23:32 +0300
http://bitbucket.org/pypy/pypy/changeset/0550049021e6/

Log:	merge win32-stdlib branch into branch

diff --git a/lib-python/2.7/mailbox.py b/lib-python/2.7/mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/mailbox.py
@@ -0,0 +1,2171 @@
+#! /usr/bin/env python
+
+"""Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
+
+# Notes for authors of new mailbox subclasses:
+#
+# Remember to fsync() changes to disk before closing a modified file
+# or returning from a flush() method.  See functions _sync_flush() and
+# _sync_close().
+
+import sys
+import os
+import time
+import calendar
+import socket
+import errno
+import copy
+import email
+import email.message
+import email.generator
+import StringIO
+try:
+    if sys.platform == 'os2emx':
+        # OS/2 EMX fcntl() not adequate
+        raise ImportError
+    import fcntl
+except ImportError:
+    fcntl = None
+
+import warnings
+with warnings.catch_warnings():
+    if sys.py3kwarning:
+        warnings.filterwarnings("ignore", ".*rfc822 has been removed",
+                                DeprecationWarning)
+    import rfc822
+
+__all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
+            'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
+            'BabylMessage', 'MMDFMessage', 'UnixMailbox',
+            'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
+
+class Mailbox:
+    """A group of messages in a particular place."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a Mailbox instance."""
+        self._path = os.path.abspath(os.path.expanduser(path))
+        self._factory = factory
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def __delitem__(self, key):
+        self.remove(key)
+
+    def discard(self, key):
+        """If the keyed message exists, remove it."""
+        try:
+            self.remove(key)
+        except KeyError:
+            pass
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get(self, key, default=None):
+        """Return the keyed message, or default if it doesn't exist."""
+        try:
+            return self.__getitem__(key)
+        except KeyError:
+            return default
+
+    def __getitem__(self, key):
+        """Return the keyed message; raise KeyError if it doesn't exist."""
+        if not self._factory:
+            return self.get_message(key)
+        else:
+            return self._factory(self.get_file(key))
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def keys(self):
+        """Return a list of keys."""
+        return list(self.iterkeys())
+
+    def itervalues(self):
+        """Return an iterator over all messages."""
+        for key in self.iterkeys():
+            try:
+                value = self[key]
+            except KeyError:
+                continue
+            yield value
+
+    def __iter__(self):
+        return self.itervalues()
+
+    def values(self):
+        """Return a list of messages. Memory intensive."""
+        return list(self.itervalues())
+
+    def iteritems(self):
+        """Return an iterator over (key, message) tuples."""
+        for key in self.iterkeys():
+            try:
+                value = self[key]
+            except KeyError:
+                continue
+            yield (key, value)
+
+    def items(self):
+        """Return a list of (key, message) tuples. Memory intensive."""
+        return list(self.iteritems())
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def __contains__(self, key):
+        return self.has_key(key)
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def clear(self):
+        """Delete all messages."""
+        for key in self.iterkeys():
+            self.discard(key)
+
+    def pop(self, key, default=None):
+        """Delete the keyed message and return it, or default."""
+        try:
+            result = self[key]
+        except KeyError:
+            return default
+        self.discard(key)
+        return result
+
+    def popitem(self):
+        """Delete an arbitrary (key, message) pair and return it."""
+        for key in self.iterkeys():
+            return (key, self.pop(key))     # This is only run once.
+        else:
+            raise KeyError('No messages in mailbox')
+
+    def update(self, arg=None):
+        """Change the messages that correspond to certain keys."""
+        if hasattr(arg, 'iteritems'):
+            source = arg.iteritems()
+        elif hasattr(arg, 'items'):
+            source = arg.items()
+        else:
+            source = arg
+        bad_key = False
+        for key, message in source:
+            try:
+                self[key] = message
+            except KeyError:
+                bad_key = True
+        if bad_key:
+            raise KeyError('No message with key(s)')
+
+    def flush(self):
+        """Write any pending changes to the disk."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def lock(self):
+        """Lock the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def close(self):
+        """Flush and close the mailbox."""
+        raise NotImplementedError('Method must be implemented by subclass')
+
+    def _dump_message(self, message, target, mangle_from_=False):
+        # Most files are opened in binary mode to allow predictable seeking.
+        # To get native line endings on disk, the user-friendly \n line endings
+        # used in strings and by email.Message are translated here.
+        """Dump message contents to target file."""
+        if isinstance(message, email.message.Message):
+            buffer = StringIO.StringIO()
+            gen = email.generator.Generator(buffer, mangle_from_, 0)
+            gen.flatten(message)
+            buffer.seek(0)
+            target.write(buffer.read().replace('\n', os.linesep))
+        elif isinstance(message, str):
+            if mangle_from_:
+                message = message.replace('\nFrom ', '\n>From ')
+            message = message.replace('\n', os.linesep)
+            target.write(message)
+        elif hasattr(message, 'read'):
+            while True:
+                line = message.readline()
+                if line == '':
+                    break
+                if mangle_from_ and line.startswith('From '):
+                    line = '>From ' + line[5:]
+                line = line.replace('\n', os.linesep)
+                target.write(line)
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+
+
+class Maildir(Mailbox):
+    """A qmail-style Maildir mailbox."""
+
+    colon = ':'
+
+    def __init__(self, dirname, factory=rfc822.Message, create=True):
+        """Initialize a Maildir instance."""
+        Mailbox.__init__(self, dirname, factory, create)
+        self._paths = {
+            'tmp': os.path.join(self._path, 'tmp'),
+            'new': os.path.join(self._path, 'new'),
+            'cur': os.path.join(self._path, 'cur'),
+            }
+        if not os.path.exists(self._path):
+            if create:
+                os.mkdir(self._path, 0700)
+                for path in self._paths.values():
+                    os.mkdir(path, 0o700)
+            else:
+                raise NoSuchMailboxError(self._path)
+        self._toc = {}
+        self._toc_mtimes = {}
+        for subdir in ('cur', 'new'):
+            self._toc_mtimes[subdir] = os.path.getmtime(self._paths[subdir])
+        self._last_read = time.time()  # Records last time we read cur/new
+        self._skewfactor = 0.1         # Adjust if os/fs clocks are skewing
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        tmp_file = self._create_tmp()
+        try:
+            self._dump_message(message, tmp_file)
+        except BaseException:
+            tmp_file.close()
+            os.remove(tmp_file.name)
+            raise
+        _sync_close(tmp_file)
+        if isinstance(message, MaildirMessage):
+            subdir = message.get_subdir()
+            suffix = self.colon + message.get_info()
+            if suffix == self.colon:
+                suffix = ''
+        else:
+            subdir = 'new'
+            suffix = ''
+        uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
+        dest = os.path.join(self._path, subdir, uniq + suffix)
+        try:
+            if hasattr(os, 'link'):
+                os.link(tmp_file.name, dest)
+                os.remove(tmp_file.name)
+            else:
+                os.rename(tmp_file.name, dest)
+        except OSError, e:
+            os.remove(tmp_file.name)
+            if e.errno == errno.EEXIST:
+                raise ExternalClashError('Name clash with existing message: %s'
+                                         % dest)
+            else:
+                raise
+        if isinstance(message, MaildirMessage):
+            os.utime(dest, (os.path.getatime(dest), message.get_date()))
+        return uniq
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        os.remove(os.path.join(self._path, self._lookup(key)))
+
+    def discard(self, key):
+        """If the keyed message exists, remove it."""
+        # This overrides an inapplicable implementation in the superclass.
+        try:
+            self.remove(key)
+        except KeyError:
+            pass
+        except OSError, e:
+            if e.errno != errno.ENOENT:
+                raise
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        old_subpath = self._lookup(key)
+        temp_key = self.add(message)
+        temp_subpath = self._lookup(temp_key)
+        if isinstance(message, MaildirMessage):
+            # temp's subdir and suffix were specified by message.
+            dominant_subpath = temp_subpath
+        else:
+            # temp's subdir and suffix were defaults from add().
+            dominant_subpath = old_subpath
+        subdir = os.path.dirname(dominant_subpath)
+        if self.colon in dominant_subpath:
+            suffix = self.colon + dominant_subpath.split(self.colon)[-1]
+        else:
+            suffix = ''
+        self.discard(key)
+        new_path = os.path.join(self._path, subdir, key + suffix)
+        os.rename(os.path.join(self._path, temp_subpath), new_path)
+        if isinstance(message, MaildirMessage):
+            os.utime(new_path, (os.path.getatime(new_path),
+                                message.get_date()))
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        subpath = self._lookup(key)
+        f = open(os.path.join(self._path, subpath), 'r')
+        try:
+            if self._factory:
+                msg = self._factory(f)
+            else:
+                msg = MaildirMessage(f)
+        finally:
+            f.close()
+        subdir, name = os.path.split(subpath)
+        msg.set_subdir(subdir)
+        if self.colon in name:
+            msg.set_info(name.split(self.colon)[-1])
+        msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        f = open(os.path.join(self._path, self._lookup(key)), 'r')
+        try:
+            return f.read()
+        finally:
+            f.close()
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        f = open(os.path.join(self._path, self._lookup(key)), 'rb')
+        return _ProxyFile(f)
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        self._refresh()
+        for key in self._toc:
+            try:
+                self._lookup(key)
+            except KeyError:
+                continue
+            yield key
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        self._refresh()
+        return key in self._toc
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        self._refresh()
+        return len(self._toc)
+
+    def flush(self):
+        """Write any pending changes to disk."""
+        # Maildir changes are always written immediately, so there's nothing
+        # to do.
+        pass
+
+    def lock(self):
+        """Lock the mailbox."""
+        return
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        return
+
+    def list_folders(self):
+        """Return a list of folder names."""
+        result = []
+        for entry in os.listdir(self._path):
+            if len(entry) > 1 and entry[0] == '.' and \
+               os.path.isdir(os.path.join(self._path, entry)):
+                result.append(entry[1:])
+        return result
+
+    def get_folder(self, folder):
+        """Return a Maildir instance for the named folder."""
+        return Maildir(os.path.join(self._path, '.' + folder),
+                       factory=self._factory,
+                       create=False)
+
+    def add_folder(self, folder):
+        """Create a folder and return a Maildir instance representing it."""
+        path = os.path.join(self._path, '.' + folder)
+        result = Maildir(path, factory=self._factory)
+        maildirfolder_path = os.path.join(path, 'maildirfolder')
+        if not os.path.exists(maildirfolder_path):
+            os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
+                0666))
+        return result
+
+    def remove_folder(self, folder):
+        """Delete the named folder, which must be empty."""
+        path = os.path.join(self._path, '.' + folder)
+        for entry in os.listdir(os.path.join(path, 'new')) + \
+                     os.listdir(os.path.join(path, 'cur')):
+            if len(entry) < 1 or entry[0] != '.':
+                raise NotEmptyError('Folder contains message(s): %s' % folder)
+        for entry in os.listdir(path):
+            if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
+               os.path.isdir(os.path.join(path, entry)):
+                raise NotEmptyError("Folder contains subdirectory '%s': %s" %
+                                    (folder, entry))
+        for root, dirs, files in os.walk(path, topdown=False):
+            for entry in files:
+                os.remove(os.path.join(root, entry))
+            for entry in dirs:
+                os.rmdir(os.path.join(root, entry))
+        os.rmdir(path)
+
+    def clean(self):
+        """Delete old files in "tmp"."""
+        now = time.time()
+        for entry in os.listdir(os.path.join(self._path, 'tmp')):
+            path = os.path.join(self._path, 'tmp', entry)
+            if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
+                os.remove(path)
+
+    _count = 1  # This is used to generate unique file names.
+
+    def _create_tmp(self):
+        """Create a file in the tmp subdirectory and open and return it."""
+        now = time.time()
+        hostname = socket.gethostname()
+        if '/' in hostname:
+            hostname = hostname.replace('/', r'\057')
+        if ':' in hostname:
+            hostname = hostname.replace(':', r'\072')
+        uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
+                                    Maildir._count, hostname)
+        path = os.path.join(self._path, 'tmp', uniq)
+        try:
+            os.stat(path)
+        except OSError, e:
+            if e.errno == errno.ENOENT:
+                Maildir._count += 1
+                try:
+                    return _create_carefully(path)
+                except OSError, e:
+                    if e.errno != errno.EEXIST:
+                        raise
+            else:
+                raise
+
+        # Fall through to here if stat succeeded or open raised EEXIST.
+        raise ExternalClashError('Name clash prevented file creation: %s' %
+                                 path)
+
+    def _refresh(self):
+        """Update table of contents mapping."""
+        # If it has been less than two seconds since the last _refresh() call,
+        # we have to unconditionally re-read the mailbox just in case it has
+        # been modified, because os.path.mtime() has a 2 sec resolution in the
+        # most common worst case (FAT) and a 1 sec resolution typically.  This
+        # results in a few unnecessary re-reads when _refresh() is called
+        # multiple times in that interval, but once the clock ticks over, we
+        # will only re-read as needed.  Because the filesystem might be being
+        # served by an independent system with its own clock, we record and
+        # compare with the mtimes from the filesystem.  Because the other
+        # system's clock might be skewing relative to our clock, we add an
+        # extra delta to our wait.  The default is one tenth second, but is an
+        # instance variable and so can be adjusted if dealing with a
+        # particularly skewed or irregular system.
+        if time.time() - self._last_read > 2 + self._skewfactor:
+            refresh = False
+            for subdir in self._toc_mtimes:
+                mtime = os.path.getmtime(self._paths[subdir])
+                if mtime > self._toc_mtimes[subdir]:
+                    refresh = True
+                self._toc_mtimes[subdir] = mtime
+            if not refresh:
+                return
+        # Refresh toc
+        self._toc = {}
+        for subdir in self._toc_mtimes:
+            path = self._paths[subdir]
+            for entry in os.listdir(path):
+                p = os.path.join(path, entry)
+                if os.path.isdir(p):
+                    continue
+                uniq = entry.split(self.colon)[0]
+                self._toc[uniq] = os.path.join(subdir, entry)
+        self._last_read = time.time()
+
+    def _lookup(self, key):
+        """Use TOC to return subpath for given key, or raise a KeyError."""
+        try:
+            if os.path.exists(os.path.join(self._path, self._toc[key])):
+                return self._toc[key]
+        except KeyError:
+            pass
+        self._refresh()
+        try:
+            return self._toc[key]
+        except KeyError:
+            raise KeyError('No message with key: %s' % key)
+
+    # This method is for backward compatibility only.
+    def next(self):
+        """Return the next message in a one-time iteration."""
+        if not hasattr(self, '_onetime_keys'):
+            self._onetime_keys = self.iterkeys()
+        while True:
+            try:
+                return self[self._onetime_keys.next()]
+            except StopIteration:
+                return None
+            except KeyError:
+                continue
+
+
+class _singlefileMailbox(Mailbox):
+    """A single-file mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a single-file mailbox."""
+        Mailbox.__init__(self, path, factory, create)
+        try:
+            f = open(self._path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                if create:
+                    f = open(self._path, 'wb+')
+                else:
+                    raise NoSuchMailboxError(self._path)
+            elif e.errno in (errno.EACCES, errno.EROFS):
+                f = open(self._path, 'rb')
+            else:
+                raise
+        self._file = f
+        self._toc = None
+        self._next_key = 0
+        self._pending = False   # No changes require rewriting the file.
+        self._locked = False
+        self._file_length = None        # Used to record mailbox size
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        self._lookup()
+        self._toc[self._next_key] = self._append_message(message)
+        self._next_key += 1
+        self._pending = True
+        return self._next_key - 1
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        self._lookup(key)
+        del self._toc[key]
+        self._pending = True
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        self._lookup(key)
+        self._toc[key] = self._append_message(message)
+        self._pending = True
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        self._lookup()
+        for key in self._toc.keys():
+            yield key
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        self._lookup()
+        return key in self._toc
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        self._lookup()
+        return len(self._toc)
+
+    def lock(self):
+        """Lock the mailbox."""
+        if not self._locked:
+            _lock_file(self._file)
+            self._locked = True
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        if self._locked:
+            _unlock_file(self._file)
+            self._locked = False
+
+    def flush(self):
+        """Write any pending changes to disk."""
+        if not self._pending:
+            return
+
+        # In order to be writing anything out at all, self._toc must
+        # already have been generated (and presumably has been modified
+        # by adding or deleting an item).
+        assert self._toc is not None
+
+        # Check length of self._file; if it's changed, some other process
+        # has modified the mailbox since we scanned it.
+        self._file.seek(0, 2)
+        cur_len = self._file.tell()
+        if cur_len != self._file_length:
+            raise ExternalClashError('Size of mailbox file changed '
+                                     '(expected %i, found %i)' %
+                                     (self._file_length, cur_len))
+
+        new_file = _create_temporary(self._path)
+        try:
+            new_toc = {}
+            self._pre_mailbox_hook(new_file)
+            for key in sorted(self._toc.keys()):
+                start, stop = self._toc[key]
+                self._file.seek(start)
+                self._pre_message_hook(new_file)
+                new_start = new_file.tell()
+                while True:
+                    buffer = self._file.read(min(4096,
+                                                 stop - self._file.tell()))
+                    if buffer == '':
+                        break
+                    new_file.write(buffer)
+                new_toc[key] = (new_start, new_file.tell())
+                self._post_message_hook(new_file)
+        except:
+            new_file.close()
+            os.remove(new_file.name)
+            raise
+        _sync_close(new_file)
+        # self._file is about to get replaced, so no need to sync.
+        self._file.close()
+        try:
+            os.rename(new_file.name, self._path)
+        except OSError, e:
+            if e.errno == errno.EEXIST or \
+              (os.name == 'os2' and e.errno == errno.EACCES):
+                os.remove(self._path)
+                os.rename(new_file.name, self._path)
+            else:
+                raise
+        self._file = open(self._path, 'rb+')
+        self._toc = new_toc
+        self._pending = False
+        if self._locked:
+            _lock_file(self._file, dotlock=False)
+
+    def _pre_mailbox_hook(self, f):
+        """Called before writing the mailbox to file f."""
+        return
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        return
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        self.flush()
+        if self._locked:
+            self.unlock()
+        self._file.close()  # Sync has been done by self.flush() above.
+
+    def _lookup(self, key=None):
+        """Return (start, stop) or raise KeyError."""
+        if self._toc is None:
+            self._generate_toc()
+        if key is not None:
+            try:
+                return self._toc[key]
+            except KeyError:
+                raise KeyError('No message with key: %s' % key)
+
+    def _append_message(self, message):
+        """Append message to mailbox and return (start, stop) offsets."""
+        self._file.seek(0, 2)
+        before = self._file.tell()
+        try:
+            self._pre_message_hook(self._file)
+            offsets = self._install_message(message)
+            self._post_message_hook(self._file)
+        except BaseException:
+            self._file.truncate(before)
+            raise
+        self._file.flush()
+        self._file_length = self._file.tell()  # Record current length of mailbox
+        return offsets
+
+
+
+class _mboxMMDF(_singlefileMailbox):
+    """An mbox or MMDF mailbox."""
+
+    _mangle_from_ = True
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        from_line = self._file.readline().replace(os.linesep, '')
+        string = self._file.read(stop - self._file.tell())
+        msg = self._message_factory(string.replace(os.linesep, '\n'))
+        msg.set_from(from_line[5:])
+        return msg
+
+    def get_string(self, key, from_=False):
+        """Return a string representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        if not from_:
+            self._file.readline()
+        string = self._file.read(stop - self._file.tell())
+        return string.replace(os.linesep, '\n')
+
+    def get_file(self, key, from_=False):
+        """Return a file-like representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        if not from_:
+            self._file.readline()
+        return _PartialFile(self._file, self._file.tell(), stop)
+
+    def _install_message(self, message):
+        """Format a message and blindly write to self._file."""
+        from_line = None
+        if isinstance(message, str) and message.startswith('From '):
+            newline = message.find('\n')
+            if newline != -1:
+                from_line = message[:newline]
+                message = message[newline + 1:]
+            else:
+                from_line = message
+                message = ''
+        elif isinstance(message, _mboxMMDFMessage):
+            from_line = 'From ' + message.get_from()
+        elif isinstance(message, email.message.Message):
+            from_line = message.get_unixfrom()  # May be None.
+        if from_line is None:
+            from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
+        start = self._file.tell()
+        self._file.write(from_line + os.linesep)
+        self._dump_message(message, self._file, self._mangle_from_)
+        stop = self._file.tell()
+        return (start, stop)
+
+
+class mbox(_mboxMMDF):
+    """A classic mbox mailbox."""
+
+    _mangle_from_ = True
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an mbox mailbox."""
+        self._message_factory = mboxMessage
+        _mboxMMDF.__init__(self, path, factory, create)
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        if f.tell() != 0:
+            f.write(os.linesep)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        while True:
+            line_pos = self._file.tell()
+            line = self._file.readline()
+            if line.startswith('From '):
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+                starts.append(line_pos)
+            elif line == '':
+                stops.append(line_pos)
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._next_key = len(self._toc)
+        self._file_length = self._file.tell()
+
+
+class MMDF(_mboxMMDF):
+    """An MMDF mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an MMDF mailbox."""
+        self._message_factory = MMDFMessage
+        _mboxMMDF.__init__(self, path, factory, create)
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        f.write('\001\001\001\001' + os.linesep)
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        f.write(os.linesep + '\001\001\001\001' + os.linesep)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        next_pos = 0
+        while True:
+            line_pos = next_pos
+            line = self._file.readline()
+            next_pos = self._file.tell()
+            if line.startswith('\001\001\001\001' + os.linesep):
+                starts.append(next_pos)
+                while True:
+                    line_pos = next_pos
+                    line = self._file.readline()
+                    next_pos = self._file.tell()
+                    if line == '\001\001\001\001' + os.linesep:
+                        stops.append(line_pos - len(os.linesep))
+                        break
+                    elif line == '':
+                        stops.append(line_pos)
+                        break
+            elif line == '':
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._next_key = len(self._toc)
+        self._file.seek(0, 2)
+        self._file_length = self._file.tell()
+
+
+class MH(Mailbox):
+    """An MH mailbox."""
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize an MH instance."""
+        Mailbox.__init__(self, path, factory, create)
+        if not os.path.exists(self._path):
+            if create:
+                os.mkdir(self._path, 0700)
+                os.close(os.open(os.path.join(self._path, '.mh_sequences'),
+                                 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
+            else:
+                raise NoSuchMailboxError(self._path)
+        self._locked = False
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        keys = self.keys()
+        if len(keys) == 0:
+            new_key = 1
+        else:
+            new_key = max(keys) + 1
+        new_path = os.path.join(self._path, str(new_key))
+        f = _create_carefully(new_path)
+        closed = False
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                try:
+                    self._dump_message(message, f)
+                except BaseException:
+                    # Unlock and close so it can be deleted on Windows
+                    if self._locked:
+                        _unlock_file(f)
+                    _sync_close(f)
+                    closed = True
+                    os.remove(new_path)
+                    raise
+                if isinstance(message, MHMessage):
+                    self._dump_sequences(message, new_key)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            if not closed:
+                _sync_close(f)
+        return new_key
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        path = os.path.join(self._path, str(key))
+        try:
+            f = open(path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        else:
+            f.close()
+            os.remove(path)
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        path = os.path.join(self._path, str(key))
+        try:
+            f = open(path, 'rb+')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
+                self._dump_message(message, f)
+                if isinstance(message, MHMessage):
+                    self._dump_sequences(message, key)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            _sync_close(f)
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        try:
+            if self._locked:
+                f = open(os.path.join(self._path, str(key)), 'r+')
+            else:
+                f = open(os.path.join(self._path, str(key)), 'r')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                msg = MHMessage(f)
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            f.close()
+        for name, key_list in self.get_sequences().iteritems():
+            if key in key_list:
+                msg.add_sequence(name)
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        try:
+            if self._locked:
+                f = open(os.path.join(self._path, str(key)), 'r+')
+            else:
+                f = open(os.path.join(self._path, str(key)), 'r')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        try:
+            if self._locked:
+                _lock_file(f)
+            try:
+                return f.read()
+            finally:
+                if self._locked:
+                    _unlock_file(f)
+        finally:
+            f.close()
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        try:
+            f = open(os.path.join(self._path, str(key)), 'rb')
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                raise KeyError('No message with key: %s' % key)
+            else:
+                raise
+        return _ProxyFile(f)
+
+    def iterkeys(self):
+        """Return an iterator over keys."""
+        return iter(sorted(int(entry) for entry in os.listdir(self._path)
+                                      if entry.isdigit()))
+
+    def has_key(self, key):
+        """Return True if the keyed message exists, False otherwise."""
+        return os.path.exists(os.path.join(self._path, str(key)))
+
+    def __len__(self):
+        """Return a count of messages in the mailbox."""
+        return len(list(self.iterkeys()))
+
+    def lock(self):
+        """Lock the mailbox."""
+        if not self._locked:
+            self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
+            _lock_file(self._file)
+            self._locked = True
+
+    def unlock(self):
+        """Unlock the mailbox if it is locked."""
+        if self._locked:
+            _unlock_file(self._file)
+            _sync_close(self._file)
+            del self._file
+            self._locked = False
+
+    def flush(self):
+        """Write any pending changes to the disk."""
+        return
+
+    def close(self):
+        """Flush and close the mailbox."""
+        if self._locked:
+            self.unlock()
+
+    def list_folders(self):
+        """Return a list of folder names."""
+        result = []
+        for entry in os.listdir(self._path):
+            if os.path.isdir(os.path.join(self._path, entry)):
+                result.append(entry)
+        return result
+
+    def get_folder(self, folder):
+        """Return an MH instance for the named folder."""
+        return MH(os.path.join(self._path, folder),
+                  factory=self._factory, create=False)
+
+    def add_folder(self, folder):
+        """Create a folder and return an MH instance representing it."""
+        return MH(os.path.join(self._path, folder),
+                  factory=self._factory)
+
+    def remove_folder(self, folder):
+        """Delete the named folder, which must be empty."""
+        path = os.path.join(self._path, folder)
+        entries = os.listdir(path)
+        if entries == ['.mh_sequences']:
+            os.remove(os.path.join(path, '.mh_sequences'))
+        elif entries == []:
+            pass
+        else:
+            raise NotEmptyError('Folder not empty: %s' % self._path)
+        os.rmdir(path)
+
+    def get_sequences(self):
+        """Return a name-to-key-list dictionary to define each sequence."""
+        results = {}
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r')
+        try:
+            all_keys = set(self.keys())
+            for line in f:
+                try:
+                    name, contents = line.split(':')
+                    keys = set()
+                    for spec in contents.split():
+                        if spec.isdigit():
+                            keys.add(int(spec))
+                        else:
+                            start, stop = (int(x) for x in spec.split('-'))
+                            keys.update(range(start, stop + 1))
+                    results[name] = [key for key in sorted(keys) \
+                                         if key in all_keys]
+                    if len(results[name]) == 0:
+                        del results[name]
+                except ValueError:
+                    raise FormatError('Invalid sequence specification: %s' %
+                                      line.rstrip())
+        finally:
+            f.close()
+        return results
+
+    def set_sequences(self, sequences):
+        """Set sequences using the given name-to-key-list dictionary."""
+        f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
+        try:
+            os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
+            for name, keys in sequences.iteritems():
+                if len(keys) == 0:
+                    continue
+                f.write('%s:' % name)
+                prev = None
+                completing = False
+                for key in sorted(set(keys)):
+                    if key - 1 == prev:
+                        if not completing:
+                            completing = True
+                            f.write('-')
+                    elif completing:
+                        completing = False
+                        f.write('%s %s' % (prev, key))
+                    else:
+                        f.write(' %s' % key)
+                    prev = key
+                if completing:
+                    f.write(str(prev) + '\n')
+                else:
+                    f.write('\n')
+        finally:
+            _sync_close(f)
+
+    def pack(self):
+        """Re-name messages to eliminate numbering gaps. Invalidates keys."""
+        sequences = self.get_sequences()
+        prev = 0
+        changes = []
+        for key in self.iterkeys():
+            if key - 1 != prev:
+                changes.append((key, prev + 1))
+                if hasattr(os, 'link'):
+                    os.link(os.path.join(self._path, str(key)),
+                            os.path.join(self._path, str(prev + 1)))
+                    os.unlink(os.path.join(self._path, str(key)))
+                else:
+                    os.rename(os.path.join(self._path, str(key)),
+                              os.path.join(self._path, str(prev + 1)))
+            prev += 1
+        self._next_key = prev + 1
+        if len(changes) == 0:
+            return
+        for name, key_list in sequences.items():
+            for old, new in changes:
+                if old in key_list:
+                    key_list[key_list.index(old)] = new
+        self.set_sequences(sequences)
+
+    def _dump_sequences(self, message, key):
+        """Inspect a new MHMessage and update sequences appropriately."""
+        pending_sequences = message.get_sequences()
+        all_sequences = self.get_sequences()
+        for name, key_list in all_sequences.iteritems():
+            if name in pending_sequences:
+                key_list.append(key)
+            elif key in key_list:
+                del key_list[key_list.index(key)]
+        for sequence in pending_sequences:
+            if sequence not in all_sequences:
+                all_sequences[sequence] = [key]
+        self.set_sequences(all_sequences)
+
+
+class Babyl(_singlefileMailbox):
+    """An Rmail-style Babyl mailbox."""
+
+    _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
+                                 'forwarded', 'edited', 'resent'))
+
+    def __init__(self, path, factory=None, create=True):
+        """Initialize a Babyl mailbox."""
+        _singlefileMailbox.__init__(self, path, factory, create)
+        self._labels = {}
+
+    def add(self, message):
+        """Add message and return assigned key."""
+        key = _singlefileMailbox.add(self, message)
+        if isinstance(message, BabylMessage):
+            self._labels[key] = message.get_labels()
+        return key
+
+    def remove(self, key):
+        """Remove the keyed message; raise KeyError if it doesn't exist."""
+        _singlefileMailbox.remove(self, key)
+        if key in self._labels:
+            del self._labels[key]
+
+    def __setitem__(self, key, message):
+        """Replace the keyed message; raise KeyError if it doesn't exist."""
+        _singlefileMailbox.__setitem__(self, key, message)
+        if isinstance(message, BabylMessage):
+            self._labels[key] = message.get_labels()
+
+    def get_message(self, key):
+        """Return a Message representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        self._file.readline()   # Skip '1,' line specifying labels.
+        original_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == '*** EOOH ***' + os.linesep or line == '':
+                break
+            original_headers.write(line.replace(os.linesep, '\n'))
+        visible_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == os.linesep or line == '':
+                break
+            visible_headers.write(line.replace(os.linesep, '\n'))
+        body = self._file.read(stop - self._file.tell()).replace(os.linesep,
+                                                                 '\n')
+        msg = BabylMessage(original_headers.getvalue() + body)
+        msg.set_visible(visible_headers.getvalue())
+        if key in self._labels:
+            msg.set_labels(self._labels[key])
+        return msg
+
+    def get_string(self, key):
+        """Return a string representation or raise a KeyError."""
+        start, stop = self._lookup(key)
+        self._file.seek(start)
+        self._file.readline()   # Skip '1,' line specifying labels.
+        original_headers = StringIO.StringIO()
+        while True:
+            line = self._file.readline()
+            if line == '*** EOOH ***' + os.linesep or line == '':
+                break
+            original_headers.write(line.replace(os.linesep, '\n'))
+        while True:
+            line = self._file.readline()
+            if line == os.linesep or line == '':
+                break
+        return original_headers.getvalue() + \
+               self._file.read(stop - self._file.tell()).replace(os.linesep,
+                                                                 '\n')
+
+    def get_file(self, key):
+        """Return a file-like representation or raise a KeyError."""
+        return StringIO.StringIO(self.get_string(key).replace('\n',
+                                                              os.linesep))
+
+    def get_labels(self):
+        """Return a list of user-defined labels in the mailbox."""
+        self._lookup()
+        labels = set()
+        for label_list in self._labels.values():
+            labels.update(label_list)
+        labels.difference_update(self._special_labels)
+        return list(labels)
+
+    def _generate_toc(self):
+        """Generate key-to-(start, stop) table of contents."""
+        starts, stops = [], []
+        self._file.seek(0)
+        next_pos = 0
+        label_lists = []
+        while True:
+            line_pos = next_pos
+            line = self._file.readline()
+            next_pos = self._file.tell()
+            if line == '\037\014' + os.linesep:
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+                starts.append(next_pos)
+                labels = [label.strip() for label
+                                        in self._file.readline()[1:].split(',')
+                                        if label.strip() != '']
+                label_lists.append(labels)
+            elif line == '\037' or line == '\037' + os.linesep:
+                if len(stops) < len(starts):
+                    stops.append(line_pos - len(os.linesep))
+            elif line == '':
+                stops.append(line_pos - len(os.linesep))
+                break
+        self._toc = dict(enumerate(zip(starts, stops)))
+        self._labels = dict(enumerate(label_lists))
+        self._next_key = len(self._toc)
+        self._file.seek(0, 2)
+        self._file_length = self._file.tell()
+
+    def _pre_mailbox_hook(self, f):
+        """Called before writing the mailbox to file f."""
+        f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
+                (os.linesep, os.linesep, ','.join(self.get_labels()),
+                 os.linesep))
+
+    def _pre_message_hook(self, f):
+        """Called before writing each message to file f."""
+        f.write('\014' + os.linesep)
+
+    def _post_message_hook(self, f):
+        """Called after writing each message to file f."""
+        f.write(os.linesep + '\037')
+
+    def _install_message(self, message):
+        """Write message contents and return (start, stop)."""
+        start = self._file.tell()
+        if isinstance(message, BabylMessage):
+            special_labels = []
+            labels = []
+            for label in message.get_labels():
+                if label in self._special_labels:
+                    special_labels.append(label)
+                else:
+                    labels.append(label)
+            self._file.write('1')
+            for label in special_labels:
+                self._file.write(', ' + label)
+            self._file.write(',,')
+            for label in labels:
+                self._file.write(' ' + label + ',')
+            self._file.write(os.linesep)
+        else:
+            self._file.write('1,,' + os.linesep)
+        if isinstance(message, email.message.Message):
+            orig_buffer = StringIO.StringIO()
+            orig_generator = email.generator.Generator(orig_buffer, False, 0)
+            orig_generator.flatten(message)
+            orig_buffer.seek(0)
+            while True:
+                line = orig_buffer.readline()
+                self._file.write(line.replace('\n', os.linesep))
+                if line == '\n' or line == '':
+                    break
+            self._file.write('*** EOOH ***' + os.linesep)
+            if isinstance(message, BabylMessage):
+                vis_buffer = StringIO.StringIO()
+                vis_generator = email.generator.Generator(vis_buffer, False, 0)
+                vis_generator.flatten(message.get_visible())
+                while True:
+                    line = vis_buffer.readline()
+                    self._file.write(line.replace('\n', os.linesep))
+                    if line == '\n' or line == '':
+                        break
+            else:
+                orig_buffer.seek(0)
+                while True:
+                    line = orig_buffer.readline()
+                    self._file.write(line.replace('\n', os.linesep))
+                    if line == '\n' or line == '':
+                        break
+            while True:
+                buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
+                if buffer == '':
+                    break
+                self._file.write(buffer.replace('\n', os.linesep))
+        elif isinstance(message, str):
+            body_start = message.find('\n\n') + 2
+            if body_start - 2 != -1:
+                self._file.write(message[:body_start].replace('\n',
+                                                              os.linesep))
+                self._file.write('*** EOOH ***' + os.linesep)
+                self._file.write(message[:body_start].replace('\n',
+                                                              os.linesep))
+                self._file.write(message[body_start:].replace('\n',
+                                                              os.linesep))
+            else:
+                self._file.write('*** EOOH ***' + os.linesep + os.linesep)
+                self._file.write(message.replace('\n', os.linesep))
+        elif hasattr(message, 'readline'):
+            original_pos = message.tell()
+            first_pass = True
+            while True:
+                line = message.readline()
+                self._file.write(line.replace('\n', os.linesep))
+                if line == '\n' or line == '':
+                    self._file.write('*** EOOH ***' + os.linesep)
+                    if first_pass:
+                        first_pass = False
+                        message.seek(original_pos)
+                    else:
+                        break
+            while True:
+                buffer = message.read(4096)     # Buffer size is arbitrary.
+                if buffer == '':
+                    break
+                self._file.write(buffer.replace('\n', os.linesep))
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+        stop = self._file.tell()
+        return (start, stop)
+
+
+class Message(email.message.Message):
+    """Message with mailbox-format-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize a Message instance."""
+        if isinstance(message, email.message.Message):
+            self._become_message(copy.deepcopy(message))
+            if isinstance(message, Message):
+                message._explain_to(self)
+        elif isinstance(message, str):
+            self._become_message(email.message_from_string(message))
+        elif hasattr(message, "read"):
+            self._become_message(email.message_from_file(message))
+        elif message is None:
+            email.message.Message.__init__(self)
+        else:
+            raise TypeError('Invalid message type: %s' % type(message))
+
+    def _become_message(self, message):
+        """Assume the non-format-specific state of message."""
+        for name in ('_headers', '_unixfrom', '_payload', '_charset',
+                     'preamble', 'epilogue', 'defects', '_default_type'):
+            self.__dict__[name] = message.__dict__[name]
+
+    def _explain_to(self, message):
+        """Copy format-specific state to message insofar as possible."""
+        if isinstance(message, Message):
+            return  # There's nothing format-specific to explain.
+        else:
+            raise TypeError('Cannot convert to specified type')
+
+
+class MaildirMessage(Message):
+    """Message with Maildir-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize a MaildirMessage instance."""
+        self._subdir = 'new'
+        self._info = ''
+        self._date = time.time()
+        Message.__init__(self, message)
+
+    def get_subdir(self):
+        """Return 'new' or 'cur'."""
+        return self._subdir
+
+    def set_subdir(self, subdir):
+        """Set subdir to 'new' or 'cur'."""
+        if subdir == 'new' or subdir == 'cur':
+            self._subdir = subdir
+        else:
+            raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
+
+    def get_flags(self):
+        """Return as a string the flags that are set."""
+        if self._info.startswith('2,'):
+            return self._info[2:]
+        else:
+            return ''
+
+    def set_flags(self, flags):
+        """Set the given flags and unset all others."""
+        self._info = '2,' + ''.join(sorted(flags))
+
+    def add_flag(self, flag):
+        """Set the given flag(s) without changing others."""
+        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+    def remove_flag(self, flag):
+        """Unset the given string flag(s) without changing others."""
+        if self.get_flags() != '':
+            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+    def get_date(self):
+        """Return delivery date of message, in seconds since the epoch."""
+        return self._date
+
+    def set_date(self, date):
+        """Set delivery date of message, in seconds since the epoch."""
+        try:
+            self._date = float(date)
+        except ValueError:
+            raise TypeError("can't convert to float: %s" % date)
+
+    def get_info(self):
+        """Get the message's "info" as a string."""
+        return self._info
+
+    def set_info(self, info):
+        """Set the message's "info" string."""
+        if isinstance(info, str):
+            self._info = info
+        else:
+            raise TypeError('info must be a string: %s' % type(info))
+
+    def _explain_to(self, message):
+        """Copy Maildir-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            message.set_flags(self.get_flags())
+            message.set_subdir(self.get_subdir())
+            message.set_date(self.get_date())
+        elif isinstance(message, _mboxMMDFMessage):
+            flags = set(self.get_flags())
+            if 'S' in flags:
+                message.add_flag('R')
+            if self.get_subdir() == 'cur':
+                message.add_flag('O')
+            if 'T' in flags:
+                message.add_flag('D')
+            if 'F' in flags:
+                message.add_flag('F')
+            if 'R' in flags:
+                message.add_flag('A')
+            message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
+        elif isinstance(message, MHMessage):
+            flags = set(self.get_flags())
+            if 'S' not in flags:
+                message.add_sequence('unseen')
+            if 'R' in flags:
+                message.add_sequence('replied')
+            if 'F' in flags:
+                message.add_sequence('flagged')
+        elif isinstance(message, BabylMessage):
+            flags = set(self.get_flags())
+            if 'S' not in flags:
+                message.add_label('unseen')
+            if 'T' in flags:
+                message.add_label('deleted')
+            if 'R' in flags:
+                message.add_label('answered')
+            if 'P' in flags:
+                message.add_label('forwarded')
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class _mboxMMDFMessage(Message):
+    """Message with mbox- or MMDF-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an mboxMMDFMessage instance."""
+        self.set_from('MAILER-DAEMON', True)
+        if isinstance(message, email.message.Message):
+            unixfrom = message.get_unixfrom()
+            if unixfrom is not None and unixfrom.startswith('From '):
+                self.set_from(unixfrom[5:])
+        Message.__init__(self, message)
+
+    def get_from(self):
+        """Return contents of "From " line."""
+        return self._from
+
+    def set_from(self, from_, time_=None):
+        """Set "From " line, formatting and appending time_ if specified."""
+        if time_ is not None:
+            if time_ is True:
+                time_ = time.gmtime()
+            from_ += ' ' + time.asctime(time_)
+        self._from = from_
+
+    def get_flags(self):
+        """Return as a string the flags that are set."""
+        return self.get('Status', '') + self.get('X-Status', '')
+
+    def set_flags(self, flags):
+        """Set the given flags and unset all others."""
+        flags = set(flags)
+        status_flags, xstatus_flags = '', ''
+        for flag in ('R', 'O'):
+            if flag in flags:
+                status_flags += flag
+                flags.remove(flag)
+        for flag in ('D', 'F', 'A'):
+            if flag in flags:
+                xstatus_flags += flag
+                flags.remove(flag)
+        xstatus_flags += ''.join(sorted(flags))
+        try:
+            self.replace_header('Status', status_flags)
+        except KeyError:
+            self.add_header('Status', status_flags)
+        try:
+            self.replace_header('X-Status', xstatus_flags)
+        except KeyError:
+            self.add_header('X-Status', xstatus_flags)
+
+    def add_flag(self, flag):
+        """Set the given flag(s) without changing others."""
+        self.set_flags(''.join(set(self.get_flags()) | set(flag)))
+
+    def remove_flag(self, flag):
+        """Unset the given string flag(s) without changing others."""
+        if 'Status' in self or 'X-Status' in self:
+            self.set_flags(''.join(set(self.get_flags()) - set(flag)))
+
+    def _explain_to(self, message):
+        """Copy mbox- or MMDF-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            flags = set(self.get_flags())
+            if 'O' in flags:
+                message.set_subdir('cur')
+            if 'F' in flags:
+                message.add_flag('F')
+            if 'A' in flags:
+                message.add_flag('R')
+            if 'R' in flags:
+                message.add_flag('S')
+            if 'D' in flags:
+                message.add_flag('T')
+            del message['status']
+            del message['x-status']
+            maybe_date = ' '.join(self.get_from().split()[-5:])
+            try:
+                message.set_date(calendar.timegm(time.strptime(maybe_date,
+                                                      '%a %b %d %H:%M:%S %Y')))
+            except (ValueError, OverflowError):
+                pass
+        elif isinstance(message, _mboxMMDFMessage):
+            message.set_flags(self.get_flags())
+            message.set_from(self.get_from())
+        elif isinstance(message, MHMessage):
+            flags = set(self.get_flags())
+            if 'R' not in flags:
+                message.add_sequence('unseen')
+            if 'A' in flags:
+                message.add_sequence('replied')
+            if 'F' in flags:
+                message.add_sequence('flagged')
+            del message['status']
+            del message['x-status']
+        elif isinstance(message, BabylMessage):
+            flags = set(self.get_flags())
+            if 'R' not in flags:
+                message.add_label('unseen')
+            if 'D' in flags:
+                message.add_label('deleted')
+            if 'A' in flags:
+                message.add_label('answered')
+            del message['status']
+            del message['x-status']
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class mboxMessage(_mboxMMDFMessage):
+    """Message with mbox-specific properties."""
+
+
+class MHMessage(Message):
+    """Message with MH-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an MHMessage instance."""
+        self._sequences = []
+        Message.__init__(self, message)
+
+    def get_sequences(self):
+        """Return a list of sequences that include the message."""
+        return self._sequences[:]
+
+    def set_sequences(self, sequences):
+        """Set the list of sequences that include the message."""
+        self._sequences = list(sequences)
+
+    def add_sequence(self, sequence):
+        """Add sequence to list of sequences including the message."""
+        if isinstance(sequence, str):
+            if not sequence in self._sequences:
+                self._sequences.append(sequence)
+        else:
+            raise TypeError('sequence must be a string: %s' % type(sequence))
+
+    def remove_sequence(self, sequence):
+        """Remove sequence from the list of sequences including the message."""
+        try:
+            self._sequences.remove(sequence)
+        except ValueError:
+            pass
+
+    def _explain_to(self, message):
+        """Copy MH-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' in sequences:
+                message.set_subdir('cur')
+            else:
+                message.set_subdir('cur')
+                message.add_flag('S')
+            if 'flagged' in sequences:
+                message.add_flag('F')
+            if 'replied' in sequences:
+                message.add_flag('R')
+        elif isinstance(message, _mboxMMDFMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' not in sequences:
+                message.add_flag('RO')
+            else:
+                message.add_flag('O')
+            if 'flagged' in sequences:
+                message.add_flag('F')
+            if 'replied' in sequences:
+                message.add_flag('A')
+        elif isinstance(message, MHMessage):
+            for sequence in self.get_sequences():
+                message.add_sequence(sequence)
+        elif isinstance(message, BabylMessage):
+            sequences = set(self.get_sequences())
+            if 'unseen' in sequences:
+                message.add_label('unseen')
+            if 'replied' in sequences:
+                message.add_label('answered')
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class BabylMessage(Message):
+    """Message with Babyl-specific properties."""
+
+    def __init__(self, message=None):
+        """Initialize an BabylMessage instance."""
+        self._labels = []
+        self._visible = Message()
+        Message.__init__(self, message)
+
+    def get_labels(self):
+        """Return a list of labels on the message."""
+        return self._labels[:]
+
+    def set_labels(self, labels):
+        """Set the list of labels on the message."""
+        self._labels = list(labels)
+
+    def add_label(self, label):
+        """Add label to list of labels on the message."""
+        if isinstance(label, str):
+            if label not in self._labels:
+                self._labels.append(label)
+        else:
+            raise TypeError('label must be a string: %s' % type(label))
+
+    def remove_label(self, label):
+        """Remove label from the list of labels on the message."""
+        try:
+            self._labels.remove(label)
+        except ValueError:
+            pass
+
+    def get_visible(self):
+        """Return a Message representation of visible headers."""
+        return Message(self._visible)
+
+    def set_visible(self, visible):
+        """Set the Message representation of visible headers."""
+        self._visible = Message(visible)
+
+    def update_visible(self):
+        """Update and/or sensibly generate a set of visible headers."""
+        for header in self._visible.keys():
+            if header in self:
+                self._visible.replace_header(header, self[header])
+            else:
+                del self._visible[header]
+        for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
+            if header in self and header not in self._visible:
+                self._visible[header] = self[header]
+
+    def _explain_to(self, message):
+        """Copy Babyl-specific state to message insofar as possible."""
+        if isinstance(message, MaildirMessage):
+            labels = set(self.get_labels())
+            if 'unseen' in labels:
+                message.set_subdir('cur')
+            else:
+                message.set_subdir('cur')
+                message.add_flag('S')
+            if 'forwarded' in labels or 'resent' in labels:
+                message.add_flag('P')
+            if 'answered' in labels:
+                message.add_flag('R')
+            if 'deleted' in labels:
+                message.add_flag('T')
+        elif isinstance(message, _mboxMMDFMessage):
+            labels = set(self.get_labels())
+            if 'unseen' not in labels:
+                message.add_flag('RO')
+            else:
+                message.add_flag('O')
+            if 'deleted' in labels:
+                message.add_flag('D')
+            if 'answered' in labels:
+                message.add_flag('A')
+        elif isinstance(message, MHMessage):
+            labels = set(self.get_labels())
+            if 'unseen' in labels:
+                message.add_sequence('unseen')
+            if 'answered' in labels:
+                message.add_sequence('replied')
+        elif isinstance(message, BabylMessage):
+            message.set_visible(self.get_visible())
+            for label in self.get_labels():
+                message.add_label(label)
+        elif isinstance(message, Message):
+            pass
+        else:
+            raise TypeError('Cannot convert to specified type: %s' %
+                            type(message))
+
+
+class MMDFMessage(_mboxMMDFMessage):
+    """Message with MMDF-specific properties."""
+
+
+class _ProxyFile:
+    """A read-only wrapper of a file."""
+
+    def __init__(self, f, pos=None):
+        """Initialize a _ProxyFile."""
+        self._file = f
+        if pos is None:
+            self._pos = f.tell()
+        else:
+            self._pos = pos
+
+    def read(self, size=None):
+        """Read bytes."""
+        return self._read(size, self._file.read)
+
+    def readline(self, size=None):
+        """Read a line."""
+        return self._read(size, self._file.readline)
+
+    def readlines(self, sizehint=None):
+        """Read multiple lines."""
+        result = []
+        for line in self:
+            result.append(line)
+            if sizehint is not None:
+                sizehint -= len(line)
+                if sizehint <= 0:
+                    break
+        return result
+
+    def __iter__(self):
+        """Iterate over lines."""
+        return iter(self.readline, "")
+
+    def tell(self):
+        """Return the position."""
+        return self._pos
+
+    def seek(self, offset, whence=0):
+        """Change position."""
+        if whence == 1:
+            self._file.seek(self._pos)
+        self._file.seek(offset, whence)
+        self._pos = self._file.tell()
+
+    def close(self):
+        """Close the file."""
+        del self._file
+
+    def _read(self, size, read_method):
+        """Read size bytes using read_method."""
+        if size is None:
+            size = -1
+        self._file.seek(self._pos)
+        result = read_method(size)
+        self._pos = self._file.tell()
+        return result
+
+
+class _PartialFile(_ProxyFile):
+    """A read-only wrapper of part of a file."""
+
+    def __init__(self, f, start=None, stop=None):
+        """Initialize a _PartialFile."""
+        _ProxyFile.__init__(self, f, start)
+        self._start = start
+        self._stop = stop
+
+    def tell(self):
+        """Return the position with respect to start."""
+        return _ProxyFile.tell(self) - self._start
+
+    def seek(self, offset, whence=0):
+        """Change position, possibly with respect to start or stop."""
+        if whence == 0:
+            self._pos = self._start
+            whence = 1
+        elif whence == 2:
+            self._pos = self._stop
+            whence = 1
+        _ProxyFile.seek(self, offset, whence)
+
+    def _read(self, size, read_method):
+        """Read size bytes using read_method, honoring start and stop."""
+        remaining = self._stop - self._pos
+        if remaining <= 0:
+            return ''
+        if size is None or size < 0 or size > remaining:
+            size = remaining
+        return _ProxyFile._read(self, size, read_method)
+
+
+def _lock_file(f, dotlock=True):
+    """Lock file f using lockf and dot locking."""
+    dotlock_done = False
+    try:
+        if fcntl:
+            try:
+                fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except IOError, e:
+                if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
+                    raise ExternalClashError('lockf: lock unavailable: %s' %
+                                             f.name)
+                else:
+                    raise
+        if dotlock:
+            try:
+                pre_lock = _create_temporary(f.name + '.lock')
+                pre_lock.close()
+            except IOError, e:
+                if e.errno in (errno.EACCES, errno.EROFS):
+                    return  # Without write access, just skip dotlocking.
+                else:
+                    raise
+            try:
+                if hasattr(os, 'link'):
+                    os.link(pre_lock.name, f.name + '.lock')
+                    dotlock_done = True
+                    os.unlink(pre_lock.name)
+                else:
+                    os.rename(pre_lock.name, f.name + '.lock')
+                    dotlock_done = True
+            except OSError, e:
+                if e.errno == errno.EEXIST or \
+                  (os.name == 'os2' and e.errno == errno.EACCES):
+                    os.remove(pre_lock.name)
+                    raise ExternalClashError('dot lock unavailable: %s' %
+                                             f.name)
+                else:
+                    raise
+    except:
+        if fcntl:
+            fcntl.lockf(f, fcntl.LOCK_UN)
+        if dotlock_done:
+            os.remove(f.name + '.lock')
+        raise
+
+def _unlock_file(f):
+    """Unlock file f using lockf and dot locking."""
+    if fcntl:
+        fcntl.lockf(f, fcntl.LOCK_UN)
+    if os.path.exists(f.name + '.lock'):
+        os.remove(f.name + '.lock')
+
+def _create_carefully(path):
+    """Create a file if it doesn't exist and open for reading and writing."""
+    fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0666)
+    try:
+        return open(path, 'rb+')
+    finally:
+        os.close(fd)
+
+def _create_temporary(path):
+    """Create a temp file based on path and open for reading and writing."""
+    return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
+                                              socket.gethostname(),
+                                              os.getpid()))
+
+def _sync_flush(f):
+    """Ensure changes to file f are physically on disk."""
+    f.flush()
+    if hasattr(os, 'fsync'):
+        os.fsync(f.fileno())
+
+def _sync_close(f):
+    """Close file f, ensuring all changes are physically on disk."""
+    _sync_flush(f)
+    f.close()
+
+## Start: classes from the original module (for backward compatibility).
+
+# Note that the Maildir class, whose name is unchanged, itself offers a next()
+# method for backward compatibility.
+
+class _Mailbox:
+
+    def __init__(self, fp, factory=rfc822.Message):
+        self.fp = fp
+        self.seekp = 0
+        self.factory = factory
+
+    def __iter__(self):
+        return iter(self.next, None)
+
+    def next(self):
+        while 1:
+            self.fp.seek(self.seekp)
+            try:
+                self._search_start()
+            except EOFError:
+                self.seekp = self.fp.tell()
+                return None
+            start = self.fp.tell()
+            self._search_end()
+            self.seekp = stop = self.fp.tell()
+            if start != stop:
+                break
+        return self.factory(_PartialFile(self.fp, start, stop))
+
+# Recommended to use PortableUnixMailbox instead!
+class UnixMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line[:5] == 'From ' and self._isrealfromline(line):
+                self.fp.seek(pos)
+                return
+
+    def _search_end(self):
+        self.fp.readline()      # Throw away header line
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line[:5] == 'From ' and self._isrealfromline(line):
+                self.fp.seek(pos)
+                return
+
+    # An overridable mechanism to test for From-line-ness.  You can either
+    # specify a different regular expression or define a whole new
+    # _isrealfromline() method.  Note that this only gets called for lines
+    # starting with the 5 characters "From ".
+    #
+    # BAW: According to
+    #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
+    # the only portable, reliable way to find message delimiters in a BSD (i.e
+    # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
+    # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
+    # like a good idea, in practice, there are too many variations for more
+    # strict parsing of the line to be completely accurate.
+    #
+    # _strict_isrealfromline() is the old version which tries to do stricter
+    # parsing of the From_ line.  _portable_isrealfromline() simply returns
+    # true, since it's never called if the line doesn't already start with
+    # "From ".
+    #
+    # This algorithm, and the way it interacts with _search_start() and
+    # _search_end() may not be completely correct, because it doesn't check
+    # that the two characters preceding "From " are \n\n or the beginning of
+    # the file.  Fixing this would require a more extensive rewrite than is
+    # necessary.  For convenience, we've added a PortableUnixMailbox class
+    # which does no checking of the format of the 'From' line.
+
+    _fromlinepattern = (r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+"
+                        r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*"
+                        r"[^\s]*\s*"
+                        "$")
+    _regexp = None
+
+    def _strict_isrealfromline(self, line):
+        if not self._regexp:
+            import re
+            self._regexp = re.compile(self._fromlinepattern)
+        return self._regexp.match(line)
+
+    def _portable_isrealfromline(self, line):
+        return True
+
+    _isrealfromline = _strict_isrealfromline
+
+
+class PortableUnixMailbox(UnixMailbox):
+    _isrealfromline = UnixMailbox._portable_isrealfromline
+
+
+class MmdfMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line[:5] == '\001\001\001\001\n':
+                return
+
+    def _search_end(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line == '\001\001\001\001\n':
+                self.fp.seek(pos)
+                return
+
+
+class MHMailbox:
+
+    def __init__(self, dirname, factory=rfc822.Message):
+        import re
+        pat = re.compile('^[1-9][0-9]*$')
+        self.dirname = dirname
+        # the three following lines could be combined into:
+        # list = map(long, filter(pat.match, os.listdir(self.dirname)))
+        list = os.listdir(self.dirname)
+        list = filter(pat.match, list)
+        list = map(long, list)
+        list.sort()
+        # This only works in Python 1.6 or later;
+        # before that str() added 'L':
+        self.boxes = map(str, list)
+        self.boxes.reverse()
+        self.factory = factory
+
+    def __iter__(self):
+        return iter(self.next, None)
+
+    def next(self):
+        if not self.boxes:
+            return None
+        fn = self.boxes.pop()
+        fp = open(os.path.join(self.dirname, fn))
+        msg = self.factory(fp)
+        try:
+            msg._mh_msgno = fn
+        except (AttributeError, TypeError):
+            pass
+        return msg
+
+
+class BabylMailbox(_Mailbox):
+
+    def _search_start(self):
+        while 1:
+            line = self.fp.readline()
+            if not line:
+                raise EOFError
+            if line == '*** EOOH ***\n':
+                return
+
+    def _search_end(self):
+        while 1:
+            pos = self.fp.tell()
+            line = self.fp.readline()
+            if not line:
+                return
+            if line == '\037\014\n' or line == '\037':
+                self.fp.seek(pos)
+                return
+
+## End: classes from the original module (for backward compatibility).
+
+
+class Error(Exception):
+    """Raised for module-specific errors."""
+
+class NoSuchMailboxError(Error):
+    """The specified mailbox does not exist and won't be created."""
+
+class NotEmptyError(Error):
+    """The specified mailbox is not empty and deletion was requested."""
+
+class ExternalClashError(Error):
+    """Another process caused an action to fail."""
+
+class FormatError(Error):
+    """A file appears to have an invalid format."""
diff --git a/lib-python/2.7/test/test_mailbox.py b/lib-python/2.7/test/test_mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_mailbox.py
@@ -0,0 +1,1970 @@
+import os
+import sys
+import time
+import stat
+import socket
+import email
+import email.message
+import re
+import StringIO
+from test import test_support
+import unittest
+import mailbox
+import glob
+try:
+    import fcntl
+except ImportError:
+    pass
+
+# Silence Py3k warning
+rfc822 = test_support.import_module('rfc822', deprecated=True)
+
+class TestBase(unittest.TestCase):
+
+    def _check_sample(self, msg):
+        # Inspect a mailbox.Message representation of the sample message
+        self.assertIsInstance(msg, email.message.Message)
+        self.assertIsInstance(msg, mailbox.Message)
+        for key, value in _sample_headers.iteritems():
+            self.assertIn(value, msg.get_all(key))
+        self.assertTrue(msg.is_multipart())
+        self.assertEqual(len(msg.get_payload()), len(_sample_payloads))
+        for i, payload in enumerate(_sample_payloads):
+            part = msg.get_payload(i)
+            self.assertIsInstance(part, email.message.Message)
+            self.assertNotIsInstance(part, mailbox.Message)
+            self.assertEqual(part.get_payload(), payload)
+
+    def _delete_recursively(self, target):
+        # Delete a file or delete a directory recursively
+        if os.path.isdir(target):
+            for path, dirs, files in os.walk(target, topdown=False):
+                for name in files:
+                    os.remove(os.path.join(path, name))
+                for name in dirs:
+                    os.rmdir(os.path.join(path, name))
+            os.rmdir(target)
+        elif os.path.exists(target):
+            os.remove(target)
+
+
+class TestMailbox(TestBase):
+
+    _factory = None     # Overridden by subclasses to reuse tests
+    _template = 'From: foo\n\n%s'
+
+    def setUp(self):
+        self._path = test_support.TESTFN
+        self._delete_recursively(self._path)
+        self._box = self._factory(self._path)
+
+    def tearDown(self):
+        self._box.close()
+        self._delete_recursively(self._path)
+
+    def test_add(self):
+        # Add copies of a sample message
+        keys = []
+        keys.append(self._box.add(self._template % 0))
+        self.assertEqual(len(self._box), 1)
+        keys.append(self._box.add(mailbox.Message(_sample_message)))
+        self.assertEqual(len(self._box), 2)
+        keys.append(self._box.add(email.message_from_string(_sample_message)))
+        self.assertEqual(len(self._box), 3)
+        keys.append(self._box.add(StringIO.StringIO(_sample_message)))
+        self.assertEqual(len(self._box), 4)
+        keys.append(self._box.add(_sample_message))
+        self.assertEqual(len(self._box), 5)
+        self.assertEqual(self._box.get_string(keys[0]), self._template % 0)
+        for i in (1, 2, 3, 4):
+            self._check_sample(self._box[keys[i]])
+
+    def test_remove(self):
+        # Remove messages using remove()
+        self._test_remove_or_delitem(self._box.remove)
+
+    def test_delitem(self):
+        # Remove messages using __delitem__()
+        self._test_remove_or_delitem(self._box.__delitem__)
+
+    def _test_remove_or_delitem(self, method):
+        # (Used by test_remove() and test_delitem().)
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(self._template % 1)
+        self.assertEqual(len(self._box), 2)
+        method(key0)
+        l = len(self._box)
+        self.assertEqual(l, 1)
+        self.assertRaises(KeyError, lambda: self._box[key0])
+        self.assertRaises(KeyError, lambda: method(key0))
+        self.assertEqual(self._box.get_string(key1), self._template % 1)
+        key2 = self._box.add(self._template % 2)
+        self.assertEqual(len(self._box), 2)
+        method(key2)
+        l = len(self._box)
+        self.assertEqual(l, 1)
+        self.assertRaises(KeyError, lambda: self._box[key2])
+        self.assertRaises(KeyError, lambda: method(key2))
+        self.assertEqual(self._box.get_string(key1), self._template % 1)
+        method(key1)
+        self.assertEqual(len(self._box), 0)
+        self.assertRaises(KeyError, lambda: self._box[key1])
+        self.assertRaises(KeyError, lambda: method(key1))
+
+    def test_discard(self, repetitions=10):
+        # Discard messages
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(self._template % 1)
+        self.assertEqual(len(self._box), 2)
+        self._box.discard(key0)
+        self.assertEqual(len(self._box), 1)
+        self.assertRaises(KeyError, lambda: self._box[key0])
+        self._box.discard(key0)
+        self.assertEqual(len(self._box), 1)
+        self.assertRaises(KeyError, lambda: self._box[key0])
+
+    def test_get(self):
+        # Retrieve messages using get()
+        key0 = self._box.add(self._template % 0)
+        msg = self._box.get(key0)
+        self.assertEqual(msg['from'], 'foo')
+        self.assertEqual(msg.get_payload(), '0')
+        self.assertIs(self._box.get('foo'), None)
+        self.assertFalse(self._box.get('foo', False))
+        self._box.close()
+        self._box = self._factory(self._path, factory=rfc822.Message)
+        key1 = self._box.add(self._template % 1)
+        msg = self._box.get(key1)
+        self.assertEqual(msg['from'], 'foo')
+        self.assertEqual(msg.fp.read(), '1')
+
+    def test_getitem(self):
+        # Retrieve message using __getitem__()
+        key0 = self._box.add(self._template % 0)
+        msg = self._box[key0]
+        self.assertEqual(msg['from'], 'foo')
+        self.assertEqual(msg.get_payload(), '0')
+        self.assertRaises(KeyError, lambda: self._box['foo'])
+        self._box.discard(key0)
+        self.assertRaises(KeyError, lambda: self._box[key0])
+
+    def test_get_message(self):
+        # Get Message representations of messages
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(_sample_message)
+        msg0 = self._box.get_message(key0)
+        self.assertIsInstance(msg0, mailbox.Message)
+        self.assertEqual(msg0['from'], 'foo')
+        self.assertEqual(msg0.get_payload(), '0')
+        self._check_sample(self._box.get_message(key1))
+
+    def test_get_string(self):
+        # Get string representations of messages
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(_sample_message)
+        self.assertEqual(self._box.get_string(key0), self._template % 0)
+        self.assertEqual(self._box.get_string(key1), _sample_message)
+
+    def test_get_file(self):
+        # Get file representations of messages
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(_sample_message)
+        self.assertEqual(self._box.get_file(key0).read().replace(os.linesep, '\n'),
+                         self._template % 0)
+        self.assertEqual(self._box.get_file(key1).read().replace(os.linesep, '\n'),
+                         _sample_message)
+
+    def test_iterkeys(self):
+        # Get keys using iterkeys()
+        self._check_iteration(self._box.iterkeys, do_keys=True, do_values=False)
+
+    def test_keys(self):
+        # Get keys using keys()
+        self._check_iteration(self._box.keys, do_keys=True, do_values=False)
+
+    def test_itervalues(self):
+        # Get values using itervalues()
+        self._check_iteration(self._box.itervalues, do_keys=False,
+                              do_values=True)
+
+    def test_iter(self):
+        # Get values using __iter__()
+        self._check_iteration(self._box.__iter__, do_keys=False,
+                              do_values=True)
+
+    def test_values(self):
+        # Get values using values()
+        self._check_iteration(self._box.values, do_keys=False, do_values=True)
+
+    def test_iteritems(self):
+        # Get keys and values using iteritems()
+        self._check_iteration(self._box.iteritems, do_keys=True,
+                              do_values=True)
+
+    def test_items(self):
+        # Get keys and values using items()
+        self._check_iteration(self._box.items, do_keys=True, do_values=True)
+
+    def _check_iteration(self, method, do_keys, do_values, repetitions=10):
+        for value in method():
+            self.fail("Not empty")
+        keys, values = [], []
+        for i in xrange(repetitions):
+            keys.append(self._box.add(self._template % i))
+            values.append(self._template % i)
+        if do_keys and not do_values:
+            returned_keys = list(method())
+        elif do_values and not do_keys:
+            returned_values = list(method())
+        else:
+            returned_keys, returned_values = [], []
+            for key, value in method():
+                returned_keys.append(key)
+                returned_values.append(value)
+        if do_keys:
+            self.assertEqual(len(keys), len(returned_keys))
+            self.assertEqual(set(keys), set(returned_keys))
+        if do_values:
+            count = 0
+            for value in returned_values:
+                self.assertEqual(value['from'], 'foo')
+                self.assertTrue(int(value.get_payload()) < repetitions,
+                                (value.get_payload(), repetitions))
+                count += 1
+            self.assertEqual(len(values), count)
+
+    def test_has_key(self):
+        # Check existence of keys using has_key()
+        self._test_has_key_or_contains(self._box.has_key)
+
+    def test_contains(self):
+        # Check existence of keys using __contains__()
+        self._test_has_key_or_contains(self._box.__contains__)
+
+    def _test_has_key_or_contains(self, method):
+        # (Used by test_has_key() and test_contains().)
+        self.assertFalse(method('foo'))
+        key0 = self._box.add(self._template % 0)
+        self.assertTrue(method(key0))
+        self.assertFalse(method('foo'))
+        key1 = self._box.add(self._template % 1)
+        self.assertTrue(method(key1))
+        self.assertTrue(method(key0))
+        self.assertFalse(method('foo'))
+        self._box.remove(key0)
+        self.assertFalse(method(key0))
+        self.assertTrue(method(key1))
+        self.assertFalse(method('foo'))
+        self._box.remove(key1)
+        self.assertFalse(method(key1))
+        self.assertFalse(method(key0))
+        self.assertFalse(method('foo'))
+
+    def test_len(self, repetitions=10):
+        # Get message count
+        keys = []
+        for i in xrange(repetitions):
+            self.assertEqual(len(self._box), i)
+            keys.append(self._box.add(self._template % i))
+            self.assertEqual(len(self._box),  i + 1)
+        for i in xrange(repetitions):
+            self.assertEqual(len(self._box), repetitions - i)
+            self._box.remove(keys[i])
+            self.assertEqual(len(self._box), repetitions - i - 1)
+
+    def test_set_item(self):
+        # Modify messages using __setitem__()
+        key0 = self._box.add(self._template % 'original 0')
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'original 0')
+        key1 = self._box.add(self._template % 'original 1')
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % 'original 1')
+        self._box[key0] = self._template % 'changed 0'
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'changed 0')
+        self._box[key1] = self._template % 'changed 1'
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % 'changed 1')
+        self._box[key0] = _sample_message
+        self._check_sample(self._box[key0])
+        self._box[key1] = self._box[key0]
+        self._check_sample(self._box[key1])
+        self._box[key0] = self._template % 'original 0'
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'original 0')
+        self._check_sample(self._box[key1])
+        self.assertRaises(KeyError,
+                          lambda: self._box.__setitem__('foo', 'bar'))
+        self.assertRaises(KeyError, lambda: self._box['foo'])
+        self.assertEqual(len(self._box), 2)
+
+    def test_clear(self, iterations=10):
+        # Remove all messages using clear()
+        keys = []
+        for i in xrange(iterations):
+            self._box.add(self._template % i)
+        for i, key in enumerate(keys):
+            self.assertEqual(self._box.get_string(key), self._template % i)
+        self._box.clear()
+        self.assertEqual(len(self._box), 0)
+        for i, key in enumerate(keys):
+            self.assertRaises(KeyError, lambda: self._box.get_string(key))
+
+    def test_pop(self):
+        # Get and remove a message using pop()
+        key0 = self._box.add(self._template % 0)
+        self.assertIn(key0, self._box)
+        key1 = self._box.add(self._template % 1)
+        self.assertIn(key1, self._box)
+        self.assertEqual(self._box.pop(key0).get_payload(), '0')
+        self.assertNotIn(key0, self._box)
+        self.assertIn(key1, self._box)
+        key2 = self._box.add(self._template % 2)
+        self.assertIn(key2, self._box)
+        self.assertEqual(self._box.pop(key2).get_payload(), '2')
+        self.assertNotIn(key2, self._box)
+        self.assertIn(key1, self._box)
+        self.assertEqual(self._box.pop(key1).get_payload(), '1')
+        self.assertNotIn(key1, self._box)
+        self.assertEqual(len(self._box), 0)
+
+    def test_popitem(self, iterations=10):
+        # Get and remove an arbitrary (key, message) using popitem()
+        keys = []
+        for i in xrange(10):
+            keys.append(self._box.add(self._template % i))
+        seen = []
+        for i in xrange(10):
+            key, msg = self._box.popitem()
+            self.assertIn(key, keys)
+            self.assertNotIn(key, seen)
+            seen.append(key)
+            self.assertEqual(int(msg.get_payload()), keys.index(key))
+        self.assertEqual(len(self._box), 0)
+        for key in keys:
+            self.assertRaises(KeyError, lambda: self._box[key])
+
+    def test_update(self):
+        # Modify multiple messages using update()
+        key0 = self._box.add(self._template % 'original 0')
+        key1 = self._box.add(self._template % 'original 1')
+        key2 = self._box.add(self._template % 'original 2')
+        self._box.update({key0: self._template % 'changed 0',
+                          key2: _sample_message})
+        self.assertEqual(len(self._box), 3)
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'changed 0')
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % 'original 1')
+        self._check_sample(self._box[key2])
+        self._box.update([(key2, self._template % 'changed 2'),
+                    (key1, self._template % 'changed 1'),
+                    (key0, self._template % 'original 0')])
+        self.assertEqual(len(self._box), 3)
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % 'original 0')
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % 'changed 1')
+        self.assertEqual(self._box.get_string(key2),
+                         self._template % 'changed 2')
+        self.assertRaises(KeyError,
+                          lambda: self._box.update({'foo': 'bar',
+                                          key0: self._template % "changed 0"}))
+        self.assertEqual(len(self._box), 3)
+        self.assertEqual(self._box.get_string(key0),
+                         self._template % "changed 0")
+        self.assertEqual(self._box.get_string(key1),
+                         self._template % "changed 1")
+        self.assertEqual(self._box.get_string(key2),
+                         self._template % "changed 2")
+
+    def test_flush(self):
+        # Write changes to disk
+        self._test_flush_or_close(self._box.flush, True)
+
+    def test_lock_unlock(self):
+        # Lock and unlock the mailbox
+        self.assertFalse(os.path.exists(self._get_lock_path()))
+        self._box.lock()
+        self.assertTrue(os.path.exists(self._get_lock_path()))
+        self._box.unlock()
+        self.assertFalse(os.path.exists(self._get_lock_path()))
+
+    def test_close(self):
+        # Close mailbox and flush changes to disk
+        self._test_flush_or_close(self._box.close, False)
+
+    def _test_flush_or_close(self, method, should_call_close):
+        contents = [self._template % i for i in xrange(3)]
+        self._box.add(contents[0])
+        self._box.add(contents[1])
+        self._box.add(contents[2])
+        method()
+        if should_call_close:
+            self._box.close()
+        self._box = self._factory(self._path)
+        keys = self._box.keys()
+        self.assertEqual(len(keys), 3)
+        for key in keys:
+            self.assertIn(self._box.get_string(key), contents)
+
+    def test_dump_message(self):
+        # Write message representations to disk
+        for input in (email.message_from_string(_sample_message),
+                      _sample_message, StringIO.StringIO(_sample_message)):
+            output = StringIO.StringIO()
+            self._box._dump_message(input, output)
+            self.assertEqual(output.getvalue(),
+                             _sample_message.replace('\n', os.linesep))
+        output = StringIO.StringIO()
+        self.assertRaises(TypeError,
+                          lambda: self._box._dump_message(None, output))
+
+    def _get_lock_path(self):
+        # Return the path of the dot lock file. May be overridden.
+        return self._path + '.lock'
+
+
+class TestMailboxSuperclass(TestBase):
+
+    def test_notimplemented(self):
+        # Test that all Mailbox methods raise NotImplementedException.
+        box = mailbox.Mailbox('path')
+        self.assertRaises(NotImplementedError, lambda: box.add(''))
+        self.assertRaises(NotImplementedError, lambda: box.remove(''))
+        self.assertRaises(NotImplementedError, lambda: box.__delitem__(''))
+        self.assertRaises(NotImplementedError, lambda: box.discard(''))
+        self.assertRaises(NotImplementedError, lambda: box.__setitem__('', ''))
+        self.assertRaises(NotImplementedError, lambda: box.iterkeys())
+        self.assertRaises(NotImplementedError, lambda: box.keys())
+        self.assertRaises(NotImplementedError, lambda: box.itervalues().next())
+        self.assertRaises(NotImplementedError, lambda: box.__iter__().next())
+        self.assertRaises(NotImplementedError, lambda: box.values())
+        self.assertRaises(NotImplementedError, lambda: box.iteritems().next())
+        self.assertRaises(NotImplementedError, lambda: box.items())
+        self.assertRaises(NotImplementedError, lambda: box.get(''))
+        self.assertRaises(NotImplementedError, lambda: box.__getitem__(''))
+        self.assertRaises(NotImplementedError, lambda: box.get_message(''))
+        self.assertRaises(NotImplementedError, lambda: box.get_string(''))
+        self.assertRaises(NotImplementedError, lambda: box.get_file(''))
+        self.assertRaises(NotImplementedError, lambda: box.has_key(''))
+        self.assertRaises(NotImplementedError, lambda: box.__contains__(''))
+        self.assertRaises(NotImplementedError, lambda: box.__len__())
+        self.assertRaises(NotImplementedError, lambda: box.clear())
+        self.assertRaises(NotImplementedError, lambda: box.pop(''))
+        self.assertRaises(NotImplementedError, lambda: box.popitem())
+        self.assertRaises(NotImplementedError, lambda: box.update((('', ''),)))
+        self.assertRaises(NotImplementedError, lambda: box.flush())
+        self.assertRaises(NotImplementedError, lambda: box.lock())
+        self.assertRaises(NotImplementedError, lambda: box.unlock())
+        self.assertRaises(NotImplementedError, lambda: box.close())
+
+
+class TestMaildir(TestMailbox):
+
+    _factory = lambda self, path, factory=None: mailbox.Maildir(path, factory)
+
+    def setUp(self):
+        TestMailbox.setUp(self)
+        if os.name in ('nt', 'os2') or sys.platform == 'cygwin':
+            self._box.colon = '!'
+
+    def test_add_MM(self):
+        # Add a MaildirMessage instance
+        msg = mailbox.MaildirMessage(self._template % 0)
+        msg.set_subdir('cur')
+        msg.set_info('foo')
+        key = self._box.add(msg)
+        self.assertTrue(os.path.exists(os.path.join(self._path, 'cur', '%s%sfoo' %
+                                                 (key, self._box.colon))))
+
+    def test_get_MM(self):
+        # Get a MaildirMessage instance
+        msg = mailbox.MaildirMessage(self._template % 0)
+        msg.set_subdir('cur')
+        msg.set_flags('RF')
+        key = self._box.add(msg)
+        msg_returned = self._box.get_message(key)
+        self.assertIsInstance(msg_returned, mailbox.MaildirMessage)
+        self.assertEqual(msg_returned.get_subdir(), 'cur')
+        self.assertEqual(msg_returned.get_flags(), 'FR')
+
+    def test_set_MM(self):
+        # Set with a MaildirMessage instance
+        msg0 = mailbox.MaildirMessage(self._template % 0)
+        msg0.set_flags('TP')
+        key = self._box.add(msg0)
+        msg_returned = self._box.get_message(key)
+        self.assertEqual(msg_returned.get_subdir(), 'new')
+        self.assertEqual(msg_returned.get_flags(), 'PT')
+        msg1 = mailbox.MaildirMessage(self._template % 1)
+        self._box[key] = msg1
+        msg_returned = self._box.get_message(key)
+        self.assertEqual(msg_returned.get_subdir(), 'new')
+        self.assertEqual(msg_returned.get_flags(), '')
+        self.assertEqual(msg_returned.get_payload(), '1')
+        msg2 = mailbox.MaildirMessage(self._template % 2)
+        msg2.set_info('2,S')
+        self._box[key] = msg2
+        self._box[key] = self._template % 3
+        msg_returned = self._box.get_message(key)
+        self.assertEqual(msg_returned.get_subdir(), 'new')
+        self.assertEqual(msg_returned.get_flags(), 'S')
+        self.assertEqual(msg_returned.get_payload(), '3')
+
+    def test_consistent_factory(self):
+        # Add a message.
+        msg = mailbox.MaildirMessage(self._template % 0)
+        msg.set_subdir('cur')
+        msg.set_flags('RF')
+        key = self._box.add(msg)
+
+        # Create new mailbox with
+        class FakeMessage(mailbox.MaildirMessage):
+            pass
+        box = mailbox.Maildir(self._path, factory=FakeMessage)
+        box.colon = self._box.colon
+        msg2 = box.get_message(key)
+        self.assertIsInstance(msg2, FakeMessage)
+
+    def test_initialize_new(self):
+        # Initialize a non-existent mailbox
+        self.tearDown()
+        self._box = mailbox.Maildir(self._path)
+        self._check_basics(factory=rfc822.Message)
+        self._delete_recursively(self._path)
+        self._box = self._factory(self._path, factory=None)
+        self._check_basics()
+
+    def test_initialize_existing(self):
+        # Initialize an existing mailbox
+        self.tearDown()
+        for subdir in '', 'tmp', 'new', 'cur':
+            os.mkdir(os.path.normpath(os.path.join(self._path, subdir)))
+        self._box = mailbox.Maildir(self._path)
+        self._check_basics(factory=rfc822.Message)
+        self._box = mailbox.Maildir(self._path, factory=None)
+        self._check_basics()
+
+    def _check_basics(self, factory=None):
+        # (Used by test_open_new() and test_open_existing().)
+        self.assertEqual(self._box._path, os.path.abspath(self._path))
+        self.assertEqual(self._box._factory, factory)
+        for subdir in '', 'tmp', 'new', 'cur':
+            path = os.path.join(self._path, subdir)
+            mode = os.stat(path)[stat.ST_MODE]
+            self.assertTrue(stat.S_ISDIR(mode), "Not a directory: '%s'" % path)
+
+    def test_list_folders(self):
+        # List folders
+        self._box.add_folder('one')
+        self._box.add_folder('two')
+        self._box.add_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 3)
+        self.assertEqual(set(self._box.list_folders()),
+                         set(('one', 'two', 'three')))
+
+    def test_get_folder(self):
+        # Open folders
+        self._box.add_folder('foo.bar')
+        folder0 = self._box.get_folder('foo.bar')
+        folder0.add(self._template % 'bar')
+        self.assertTrue(os.path.isdir(os.path.join(self._path, '.foo.bar')))
+        folder1 = self._box.get_folder('foo.bar')
+        self.assertEqual(folder1.get_string(folder1.keys()[0]),
+                         self._template % 'bar')
+
+    def test_add_and_remove_folders(self):
+        # Delete folders
+        self._box.add_folder('one')
+        self._box.add_folder('two')
+        self.assertEqual(len(self._box.list_folders()), 2)
+        self.assertEqual(set(self._box.list_folders()), set(('one', 'two')))
+        self._box.remove_folder('one')
+        self.assertEqual(len(self._box.list_folders()), 1)
+        self.assertEqual(set(self._box.list_folders()), set(('two',)))
+        self._box.add_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 2)
+        self.assertEqual(set(self._box.list_folders()), set(('two', 'three')))
+        self._box.remove_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 1)
+        self.assertEqual(set(self._box.list_folders()), set(('two',)))
+        self._box.remove_folder('two')
+        self.assertEqual(len(self._box.list_folders()), 0)
+        self.assertEqual(self._box.list_folders(), [])
+
+    def test_clean(self):
+        # Remove old files from 'tmp'
+        foo_path = os.path.join(self._path, 'tmp', 'foo')
+        bar_path = os.path.join(self._path, 'tmp', 'bar')
+        with open(foo_path, 'w') as f:
+            f.write("@")
+        with open(bar_path, 'w') as f:
+            f.write("@")
+        self._box.clean()
+        self.assertTrue(os.path.exists(foo_path))
+        self.assertTrue(os.path.exists(bar_path))
+        foo_stat = os.stat(foo_path)
+        os.utime(foo_path, (time.time() - 129600 - 2,
+                            foo_stat.st_mtime))
+        self._box.clean()
+        self.assertFalse(os.path.exists(foo_path))
+        self.assertTrue(os.path.exists(bar_path))
+
+    def test_create_tmp(self, repetitions=10):
+        # Create files in tmp directory
+        hostname = socket.gethostname()
+        if '/' in hostname:
+            hostname = hostname.replace('/', r'\057')
+        if ':' in hostname:
+            hostname = hostname.replace(':', r'\072')
+        pid = os.getpid()
+        pattern = re.compile(r"(?P<time>\d+)\.M(?P<M>\d{1,6})P(?P<P>\d+)"
+                             r"Q(?P<Q>\d+)\.(?P<host>[^:/]+)")
+        previous_groups = None
+        for x in xrange(repetitions):
+            tmp_file = self._box._create_tmp()
+            head, tail = os.path.split(tmp_file.name)
+            self.assertEqual(head, os.path.abspath(os.path.join(self._path,
+                                                                "tmp")),
+                             "File in wrong location: '%s'" % head)
+            match = pattern.match(tail)
+            self.assertTrue(match is not None, "Invalid file name: '%s'" % tail)
+            groups = match.groups()
+            if previous_groups is not None:
+                self.assertTrue(int(groups[0] >= previous_groups[0]),
+                             "Non-monotonic seconds: '%s' before '%s'" %
+                             (previous_groups[0], groups[0]))
+                self.assertTrue(int(groups[1] >= previous_groups[1]) or
+                             groups[0] != groups[1],
+                             "Non-monotonic milliseconds: '%s' before '%s'" %
+                             (previous_groups[1], groups[1]))
+                self.assertTrue(int(groups[2]) == pid,
+                             "Process ID mismatch: '%s' should be '%s'" %
+                             (groups[2], pid))
+                self.assertTrue(int(groups[3]) == int(previous_groups[3]) + 1,
+                             "Non-sequential counter: '%s' before '%s'" %
+                             (previous_groups[3], groups[3]))
+                self.assertTrue(groups[4] == hostname,
+                             "Host name mismatch: '%s' should be '%s'" %
+                             (groups[4], hostname))
+            previous_groups = groups
+            tmp_file.write(_sample_message)
+            tmp_file.seek(0)
+            self.assertTrue(tmp_file.read() == _sample_message)
+            tmp_file.close()
+        file_count = len(os.listdir(os.path.join(self._path, "tmp")))
+        self.assertTrue(file_count == repetitions,
+                     "Wrong file count: '%s' should be '%s'" %
+                     (file_count, repetitions))
+
+    def test_refresh(self):
+        # Update the table of contents
+        self.assertEqual(self._box._toc, {})
+        key0 = self._box.add(self._template % 0)
+        key1 = self._box.add(self._template % 1)
+        self.assertEqual(self._box._toc, {})
+        self._box._refresh()
+        self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+                                          key1: os.path.join('new', key1)})
+        key2 = self._box.add(self._template % 2)
+        self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+                                          key1: os.path.join('new', key1)})
+        self._box._refresh()
+        self.assertEqual(self._box._toc, {key0: os.path.join('new', key0),
+                                          key1: os.path.join('new', key1),
+                                          key2: os.path.join('new', key2)})
+
+    def test_lookup(self):
+        # Look up message subpaths in the TOC
+        self.assertRaises(KeyError, lambda: self._box._lookup('foo'))
+        key0 = self._box.add(self._template % 0)
+        self.assertEqual(self._box._lookup(key0), os.path.join('new', key0))
+        os.remove(os.path.join(self._path, 'new', key0))
+        self.assertEqual(self._box._toc, {key0: os.path.join('new', key0)})
+        # Be sure that the TOC is read back from disk (see issue #6896
+        # about bad mtime behaviour on some systems).
+        self._box.flush()
+        self.assertRaises(KeyError, lambda: self._box._lookup(key0))
+        self.assertEqual(self._box._toc, {})
+
+    def test_lock_unlock(self):
+        # Lock and unlock the mailbox. For Maildir, this does nothing.
+        self._box.lock()
+        self._box.unlock()
+
+    def test_folder (self):
+        # Test for bug #1569790: verify that folders returned by .get_folder()
+        # use the same factory function.
+        def dummy_factory (s):
+            return None
+        box = self._factory(self._path, factory=dummy_factory)
+        folder = box.add_folder('folder1')
+        self.assertIs(folder._factory, dummy_factory)
+
+        folder1_alias = box.get_folder('folder1')
+        self.assertIs(folder1_alias._factory, dummy_factory)
+
+    def test_directory_in_folder (self):
+        # Test that mailboxes still work if there's a stray extra directory
+        # in a folder.
+        for i in range(10):
+            self._box.add(mailbox.Message(_sample_message))
+
+        # Create a stray directory
+        os.mkdir(os.path.join(self._path, 'cur', 'stray-dir'))
+
+        # Check that looping still works with the directory present.
+        for msg in self._box:
+            pass
+
+    def test_file_permissions(self):
+        # Verify that message files are created without execute permissions
+        if not hasattr(os, "stat") or not hasattr(os, "umask"):
+            return
+        msg = mailbox.MaildirMessage(self._template % 0)
+        orig_umask = os.umask(0)
+        try:
+            key = self._box.add(msg)
+        finally:
+            os.umask(orig_umask)
+        path = os.path.join(self._path, self._box._lookup(key))
+        mode = os.stat(path).st_mode
+        self.assertEqual(mode & 0111, 0)
+
+    def test_folder_file_perms(self):
+        # From bug #3228, we want to verify that the file created inside a Maildir
+        # subfolder isn't marked as executable.
+        if not hasattr(os, "stat") or not hasattr(os, "umask"):
+            return
+
+        orig_umask = os.umask(0)
+        try:
+            subfolder = self._box.add_folder('subfolder')
+        finally:
+            os.umask(orig_umask)
+
+        path = os.path.join(subfolder._path, 'maildirfolder')
+        st = os.stat(path)
+        perms = st.st_mode
+        self.assertFalse((perms & 0111)) # Execute bits should all be off.
+
+    def test_reread(self):
+
+        # Put the last modified times more than two seconds into the past
+        # (because mtime may have only a two second granularity).
+        for subdir in ('cur', 'new'):
+            os.utime(os.path.join(self._box._path, subdir),
+                     (time.time()-5,)*2)
+
+        # Because mtime has a two second granularity in worst case (FAT), a
+        # refresh is done unconditionally if called for within
+        # two-second-plus-a-bit of the last one, just in case the mbox has
+        # changed; so now we have to wait for that interval to expire.
+        time.sleep(2.01 + self._box._skewfactor)
+
+        # Re-reading causes the ._toc attribute to be assigned a new dictionary
+        # object, so we'll check that the ._toc attribute isn't a different
+        # object.
+        orig_toc = self._box._toc
+        def refreshed():
+            return self._box._toc is not orig_toc
+
+        self._box._refresh()
+        self.assertFalse(refreshed())
+
+        # Now, write something into cur and remove it.  This changes
+        # the mtime and should cause a re-read.
+        filename = os.path.join(self._path, 'cur', 'stray-file')
+        f = open(filename, 'w')
+        f.close()
+        os.unlink(filename)
+        self._box._refresh()
+        self.assertTrue(refreshed())
+
+class _TestMboxMMDF(TestMailbox):
+
+    def tearDown(self):
+        self._box.close()
+        self._delete_recursively(self._path)
+        for lock_remnant in glob.glob(self._path + '.*'):
+            test_support.unlink(lock_remnant)
+
+    def test_add_from_string(self):
+        # Add a string starting with 'From ' to the mailbox
+        key = self._box.add('From foo at bar blah\nFrom: foo\n\n0')
+        self.assertEqual(self._box[key].get_from(), 'foo at bar blah')
+        self.assertEqual(self._box[key].get_payload(), '0')
+
+    def test_add_mbox_or_mmdf_message(self):
+        # Add an mboxMessage or MMDFMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg = class_('From foo at bar blah\nFrom: foo\n\n0')
+            key = self._box.add(msg)
+
+    def test_open_close_open(self):
+        # Open and inspect previously-created mailbox
+        values = [self._template % i for i in xrange(3)]
+        for value in values:
+            self._box.add(value)
+        self._box.close()
+        mtime = os.path.getmtime(self._path)
+        self._box = self._factory(self._path)
+        self.assertEqual(len(self._box), 3)
+        for key in self._box.iterkeys():
+            self.assertIn(self._box.get_string(key), values)
+        self._box.close()
+        self.assertEqual(mtime, os.path.getmtime(self._path))
+
+    def test_add_and_close(self):
+        # Verifying that closing a mailbox doesn't change added items
+        self._box.add(_sample_message)
+        for i in xrange(3):
+            self._box.add(self._template % i)
+        self._box.add(_sample_message)
+        self._box._file.flush()
+        self._box._file.seek(0)
+        contents = self._box._file.read()
+        self._box.close()
+        with open(self._path, 'rb') as f:
+            self.assertEqual(contents, f.read())
+        self._box = self._factory(self._path)
+
+    def test_lock_conflict(self):
+        # Fork off a subprocess that will lock the file for 2 seconds,
+        # unlock it, and then exit.
+        if not hasattr(os, 'fork'):
+            return
+        pid = os.fork()
+        if pid == 0:
+            # In the child, lock the mailbox.
+            self._box.lock()
+            time.sleep(2)
+            self._box.unlock()
+            os._exit(0)
+
+        # In the parent, sleep a bit to give the child time to acquire
+        # the lock.
+        time.sleep(0.5)
+        try:
+            self.assertRaises(mailbox.ExternalClashError,
+                              self._box.lock)
+        finally:
+            # Wait for child to exit.  Locking should now succeed.
+            exited_pid, status = os.waitpid(pid, 0)
+
+        self._box.lock()
+        self._box.unlock()
+
+    def test_relock(self):
+        # Test case for bug #1575506: the mailbox class was locking the
+        # wrong file object in its flush() method.
+        msg = "Subject: sub\n\nbody\n"
+        key1 = self._box.add(msg)
+        self._box.flush()
+        self._box.close()
+
+        self._box = self._factory(self._path)
+        self._box.lock()
+        key2 = self._box.add(msg)
+        self._box.flush()
+        self.assertTrue(self._box._locked)
+        self._box.close()
+
+
+class TestMbox(_TestMboxMMDF):
+
+    _factory = lambda self, path, factory=None: mailbox.mbox(path, factory)
+
+    def test_file_perms(self):
+        # From bug #3228, we want to verify that the mailbox file isn't executable,
+        # even if the umask is set to something that would leave executable bits set.
+        # We only run this test on platforms that support umask.
+        if hasattr(os, 'umask') and hasattr(os, 'stat'):
+            try:
+                old_umask = os.umask(0077)
+                self._box.close()
+                os.unlink(self._path)
+                self._box = mailbox.mbox(self._path, create=True)
+                self._box.add('')
+                self._box.close()
+            finally:
+                os.umask(old_umask)
+
+            st = os.stat(self._path)
+            perms = st.st_mode
+            self.assertFalse((perms & 0111)) # Execute bits should all be off.
+
+class TestMMDF(_TestMboxMMDF):
+
+    _factory = lambda self, path, factory=None: mailbox.MMDF(path, factory)
+
+
+class TestMH(TestMailbox):
+
+    _factory = lambda self, path, factory=None: mailbox.MH(path, factory)
+
+    def test_list_folders(self):
+        # List folders
+        self._box.add_folder('one')
+        self._box.add_folder('two')
+        self._box.add_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 3)
+        self.assertEqual(set(self._box.list_folders()),
+                         set(('one', 'two', 'three')))
+
+    def test_get_folder(self):
+        # Open folders
+        def dummy_factory (s):
+            return None
+        self._box = self._factory(self._path, dummy_factory)
+
+        new_folder = self._box.add_folder('foo.bar')
+        folder0 = self._box.get_folder('foo.bar')
+        folder0.add(self._template % 'bar')
+        self.assertTrue(os.path.isdir(os.path.join(self._path, 'foo.bar')))
+        folder1 = self._box.get_folder('foo.bar')
+        self.assertEqual(folder1.get_string(folder1.keys()[0]),
+                         self._template % 'bar')
+
+        # Test for bug #1569790: verify that folders returned by .get_folder()
+        # use the same factory function.
+        self.assertIs(new_folder._factory, self._box._factory)
+        self.assertIs(folder0._factory, self._box._factory)
+
+    def test_add_and_remove_folders(self):
+        # Delete folders
+        self._box.add_folder('one')
+        self._box.add_folder('two')
+        self.assertEqual(len(self._box.list_folders()), 2)
+        self.assertEqual(set(self._box.list_folders()), set(('one', 'two')))
+        self._box.remove_folder('one')
+        self.assertEqual(len(self._box.list_folders()), 1)
+        self.assertEqual(set(self._box.list_folders()), set(('two', )))
+        self._box.add_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 2)
+        self.assertEqual(set(self._box.list_folders()), set(('two', 'three')))
+        self._box.remove_folder('three')
+        self.assertEqual(len(self._box.list_folders()), 1)
+        self.assertEqual(set(self._box.list_folders()), set(('two', )))
+        self._box.remove_folder('two')
+        self.assertEqual(len(self._box.list_folders()), 0)
+        self.assertEqual(self._box.list_folders(), [])
+
+    def test_sequences(self):
+        # Get and set sequences
+        self.assertEqual(self._box.get_sequences(), {})
+        msg0 = mailbox.MHMessage(self._template % 0)
+        msg0.add_sequence('foo')
+        key0 = self._box.add(msg0)
+        self.assertEqual(self._box.get_sequences(), {'foo':[key0]})
+        msg1 = mailbox.MHMessage(self._template % 1)
+        msg1.set_sequences(['bar', 'replied', 'foo'])
+        key1 = self._box.add(msg1)
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[key0, key1], 'bar':[key1], 'replied':[key1]})
+        msg0.set_sequences(['flagged'])
+        self._box[key0] = msg0
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[key1], 'bar':[key1], 'replied':[key1],
+                          'flagged':[key0]})
+        self._box.remove(key1)
+        self.assertEqual(self._box.get_sequences(), {'flagged':[key0]})
+
+    def test_issue2625(self):
+        msg0 = mailbox.MHMessage(self._template % 0)
+        msg0.add_sequence('foo')
+        key0 = self._box.add(msg0)
+        refmsg0 = self._box.get_message(key0)
+
+    def test_issue7627(self):
+        msg0 = mailbox.MHMessage(self._template % 0)
+        key0 = self._box.add(msg0)
+        self._box.lock()
+        self._box.remove(key0)
+        self._box.unlock()
+
+    def test_pack(self):
+        # Pack the contents of the mailbox
+        msg0 = mailbox.MHMessage(self._template % 0)
+        msg1 = mailbox.MHMessage(self._template % 1)
+        msg2 = mailbox.MHMessage(self._template % 2)
+        msg3 = mailbox.MHMessage(self._template % 3)
+        msg0.set_sequences(['foo', 'unseen'])
+        msg1.set_sequences(['foo'])
+        msg2.set_sequences(['foo', 'flagged'])
+        msg3.set_sequences(['foo', 'bar', 'replied'])
+        key0 = self._box.add(msg0)
+        key1 = self._box.add(msg1)
+        key2 = self._box.add(msg2)
+        key3 = self._box.add(msg3)
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[key0,key1,key2,key3], 'unseen':[key0],
+                          'flagged':[key2], 'bar':[key3], 'replied':[key3]})
+        self._box.remove(key2)
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[key0,key1,key3], 'unseen':[key0], 'bar':[key3],
+                          'replied':[key3]})
+        self._box.pack()
+        self.assertEqual(self._box.keys(), [1, 2, 3])
+        key0 = key0
+        key1 = key0 + 1
+        key2 = key1 + 1
+        self.assertEqual(self._box.get_sequences(),
+                     {'foo':[1, 2, 3], 'unseen':[1], 'bar':[3], 'replied':[3]})
+
+        # Test case for packing while holding the mailbox locked.
+        key0 = self._box.add(msg1)
+        key1 = self._box.add(msg1)
+        key2 = self._box.add(msg1)
+        key3 = self._box.add(msg1)
+
+        self._box.remove(key0)
+        self._box.remove(key2)
+        self._box.lock()
+        self._box.pack()
+        self._box.unlock()
+        self.assertEqual(self._box.get_sequences(),
+                         {'foo':[1, 2, 3, 4, 5],
+                          'unseen':[1], 'bar':[3], 'replied':[3]})
+
+    def _get_lock_path(self):
+        return os.path.join(self._path, '.mh_sequences.lock')
+
+
+class TestBabyl(TestMailbox):
+
+    _factory = lambda self, path, factory=None: mailbox.Babyl(path, factory)
+
+    def tearDown(self):
+        self._box.close()
+        self._delete_recursively(self._path)
+        for lock_remnant in glob.glob(self._path + '.*'):
+            test_support.unlink(lock_remnant)
+
+    def test_labels(self):
+        # Get labels from the mailbox
+        self.assertEqual(self._box.get_labels(), [])
+        msg0 = mailbox.BabylMessage(self._template % 0)
+        msg0.add_label('foo')
+        key0 = self._box.add(msg0)
+        self.assertEqual(self._box.get_labels(), ['foo'])
+        msg1 = mailbox.BabylMessage(self._template % 1)
+        msg1.set_labels(['bar', 'answered', 'foo'])
+        key1 = self._box.add(msg1)
+        self.assertEqual(set(self._box.get_labels()), set(['foo', 'bar']))
+        msg0.set_labels(['blah', 'filed'])
+        self._box[key0] = msg0
+        self.assertEqual(set(self._box.get_labels()),
+                         set(['foo', 'bar', 'blah']))
+        self._box.remove(key1)
+        self.assertEqual(set(self._box.get_labels()), set(['blah']))
+
+
+class TestMessage(TestBase):
+
+    _factory = mailbox.Message      # Overridden by subclasses to reuse tests
+
+    def setUp(self):
+        self._path = test_support.TESTFN
+
+    def tearDown(self):
+        self._delete_recursively(self._path)
+
+    def test_initialize_with_eMM(self):
+        # Initialize based on email.message.Message instance
+        eMM = email.message_from_string(_sample_message)
+        msg = self._factory(eMM)
+        self._post_initialize_hook(msg)
+        self._check_sample(msg)
+
+    def test_initialize_with_string(self):
+        # Initialize based on string
+        msg = self._factory(_sample_message)
+        self._post_initialize_hook(msg)
+        self._check_sample(msg)
+
+    def test_initialize_with_file(self):
+        # Initialize based on contents of file
+        with open(self._path, 'w+') as f:
+            f.write(_sample_message)
+            f.seek(0)
+            msg = self._factory(f)
+            self._post_initialize_hook(msg)
+            self._check_sample(msg)
+
+    def test_initialize_with_nothing(self):
+        # Initialize without arguments
+        msg = self._factory()
+        self._post_initialize_hook(msg)
+        self.assertIsInstance(msg, email.message.Message)
+        self.assertIsInstance(msg, mailbox.Message)
+        self.assertIsInstance(msg, self._factory)
+        self.assertEqual(msg.keys(), [])
+        self.assertFalse(msg.is_multipart())
+        self.assertEqual(msg.get_payload(), None)
+
+    def test_initialize_incorrectly(self):
+        # Initialize with invalid argument
+        self.assertRaises(TypeError, lambda: self._factory(object()))
+
+    def test_become_message(self):
+        # Take on the state of another message
+        eMM = email.message_from_string(_sample_message)
+        msg = self._factory()
+        msg._become_message(eMM)
+        self._check_sample(msg)
+
+    def test_explain_to(self):
+        # Copy self's format-specific data to other message formats.
+        # This test is superficial; better ones are in TestMessageConversion.
+        msg = self._factory()
+        for class_ in (mailbox.Message, mailbox.MaildirMessage,
+                       mailbox.mboxMessage, mailbox.MHMessage,
+                       mailbox.BabylMessage, mailbox.MMDFMessage):
+            other_msg = class_()
+            msg._explain_to(other_msg)
+        other_msg = email.message.Message()
+        self.assertRaises(TypeError, lambda: msg._explain_to(other_msg))
+
+    def _post_initialize_hook(self, msg):
+        # Overridden by subclasses to check extra things after initialization
+        pass
+
+
+class TestMaildirMessage(TestMessage):
+
+    _factory = mailbox.MaildirMessage
+
+    def _post_initialize_hook(self, msg):
+        self.assertEqual(msg._subdir, 'new')
+        self.assertEqual(msg._info,'')
+
+    def test_subdir(self):
+        # Use get_subdir() and set_subdir()
+        msg = mailbox.MaildirMessage(_sample_message)
+        self.assertEqual(msg.get_subdir(), 'new')
+        msg.set_subdir('cur')
+        self.assertEqual(msg.get_subdir(), 'cur')
+        msg.set_subdir('new')
+        self.assertEqual(msg.get_subdir(), 'new')
+        self.assertRaises(ValueError, lambda: msg.set_subdir('tmp'))
+        self.assertEqual(msg.get_subdir(), 'new')
+        msg.set_subdir('new')
+        self.assertEqual(msg.get_subdir(), 'new')
+        self._check_sample(msg)
+
+    def test_flags(self):
+        # Use get_flags(), set_flags(), add_flag(), remove_flag()
+        msg = mailbox.MaildirMessage(_sample_message)
+        self.assertEqual(msg.get_flags(), '')
+        self.assertEqual(msg.get_subdir(), 'new')
+        msg.set_flags('F')
+        self.assertEqual(msg.get_subdir(), 'new')
+        self.assertEqual(msg.get_flags(), 'F')
+        msg.set_flags('SDTP')
+        self.assertEqual(msg.get_flags(), 'DPST')
+        msg.add_flag('FT')
+        self.assertEqual(msg.get_flags(), 'DFPST')
+        msg.remove_flag('TDRP')
+        self.assertEqual(msg.get_flags(), 'FS')
+        self.assertEqual(msg.get_subdir(), 'new')
+        self._check_sample(msg)
+
+    def test_date(self):
+        # Use get_date() and set_date()
+        msg = mailbox.MaildirMessage(_sample_message)
+        diff = msg.get_date() - time.time()
+        self.assertTrue(abs(diff) < 60, diff)
+        msg.set_date(0.0)
+        self.assertEqual(msg.get_date(), 0.0)
+
+    def test_info(self):
+        # Use get_info() and set_info()
+        msg = mailbox.MaildirMessage(_sample_message)
+        self.assertEqual(msg.get_info(), '')
+        msg.set_info('1,foo=bar')
+        self.assertEqual(msg.get_info(), '1,foo=bar')
+        self.assertRaises(TypeError, lambda: msg.set_info(None))
+        self._check_sample(msg)
+
+    def test_info_and_flags(self):
+        # Test interaction of info and flag methods
+        msg = mailbox.MaildirMessage(_sample_message)
+        self.assertEqual(msg.get_info(), '')
+        msg.set_flags('SF')
+        self.assertEqual(msg.get_flags(), 'FS')
+        self.assertEqual(msg.get_info(), '2,FS')
+        msg.set_info('1,')
+        self.assertEqual(msg.get_flags(), '')
+        self.assertEqual(msg.get_info(), '1,')
+        msg.remove_flag('RPT')
+        self.assertEqual(msg.get_flags(), '')
+        self.assertEqual(msg.get_info(), '1,')
+        msg.add_flag('D')
+        self.assertEqual(msg.get_flags(), 'D')
+        self.assertEqual(msg.get_info(), '2,D')
+        self._check_sample(msg)
+
+
+class _TestMboxMMDFMessage(TestMessage):
+
+    _factory = mailbox._mboxMMDFMessage
+
+    def _post_initialize_hook(self, msg):
+        self._check_from(msg)
+
+    def test_initialize_with_unixfrom(self):
+        # Initialize with a message that already has a _unixfrom attribute
+        msg = mailbox.Message(_sample_message)
+        msg.set_unixfrom('From foo at bar blah')
+        msg = mailbox.mboxMessage(msg)
+        self.assertEqual(msg.get_from(), 'foo at bar blah')
+
+    def test_from(self):
+        # Get and set "From " line
+        msg = mailbox.mboxMessage(_sample_message)
+        self._check_from(msg)
+        msg.set_from('foo bar')
+        self.assertEqual(msg.get_from(), 'foo bar')
+        msg.set_from('foo at bar', True)
+        self._check_from(msg, 'foo at bar')
+        msg.set_from('blah at temp', time.localtime())
+        self._check_from(msg, 'blah at temp')
+
+    def test_flags(self):
+        # Use get_flags(), set_flags(), add_flag(), remove_flag()
+        msg = mailbox.mboxMessage(_sample_message)
+        self.assertEqual(msg.get_flags(), '')
+        msg.set_flags('F')
+        self.assertEqual(msg.get_flags(), 'F')
+        msg.set_flags('XODR')
+        self.assertEqual(msg.get_flags(), 'RODX')
+        msg.add_flag('FA')
+        self.assertEqual(msg.get_flags(), 'RODFAX')
+        msg.remove_flag('FDXA')
+        self.assertEqual(msg.get_flags(), 'RO')
+        self._check_sample(msg)
+
+    def _check_from(self, msg, sender=None):
+        # Check contents of "From " line
+        if sender is None:
+            sender = "MAILER-DAEMON"
+        self.assertTrue(re.match(sender + r" \w{3} \w{3} [\d ]\d [\d ]\d:\d{2}:"
+                                 r"\d{2} \d{4}", msg.get_from()))
+
+
+class TestMboxMessage(_TestMboxMMDFMessage):
+
+    _factory = mailbox.mboxMessage
+
+
+class TestMHMessage(TestMessage):
+
+    _factory = mailbox.MHMessage
+
+    def _post_initialize_hook(self, msg):
+        self.assertEqual(msg._sequences, [])
+
+    def test_sequences(self):
+        # Get, set, join, and leave sequences
+        msg = mailbox.MHMessage(_sample_message)
+        self.assertEqual(msg.get_sequences(), [])
+        msg.set_sequences(['foobar'])
+        self.assertEqual(msg.get_sequences(), ['foobar'])
+        msg.set_sequences([])
+        self.assertEqual(msg.get_sequences(), [])
+        msg.add_sequence('unseen')
+        self.assertEqual(msg.get_sequences(), ['unseen'])
+        msg.add_sequence('flagged')
+        self.assertEqual(msg.get_sequences(), ['unseen', 'flagged'])
+        msg.add_sequence('flagged')
+        self.assertEqual(msg.get_sequences(), ['unseen', 'flagged'])
+        msg.remove_sequence('unseen')
+        self.assertEqual(msg.get_sequences(), ['flagged'])
+        msg.add_sequence('foobar')
+        self.assertEqual(msg.get_sequences(), ['flagged', 'foobar'])
+        msg.remove_sequence('replied')
+        self.assertEqual(msg.get_sequences(), ['flagged', 'foobar'])
+        msg.set_sequences(['foobar', 'replied'])
+        self.assertEqual(msg.get_sequences(), ['foobar', 'replied'])
+
+
+class TestBabylMessage(TestMessage):
+
+    _factory = mailbox.BabylMessage
+
+    def _post_initialize_hook(self, msg):
+        self.assertEqual(msg._labels, [])
+
+    def test_labels(self):
+        # Get, set, join, and leave labels
+        msg = mailbox.BabylMessage(_sample_message)
+        self.assertEqual(msg.get_labels(), [])
+        msg.set_labels(['foobar'])
+        self.assertEqual(msg.get_labels(), ['foobar'])
+        msg.set_labels([])
+        self.assertEqual(msg.get_labels(), [])
+        msg.add_label('filed')
+        self.assertEqual(msg.get_labels(), ['filed'])
+        msg.add_label('resent')
+        self.assertEqual(msg.get_labels(), ['filed', 'resent'])
+        msg.add_label('resent')
+        self.assertEqual(msg.get_labels(), ['filed', 'resent'])
+        msg.remove_label('filed')
+        self.assertEqual(msg.get_labels(), ['resent'])
+        msg.add_label('foobar')
+        self.assertEqual(msg.get_labels(), ['resent', 'foobar'])
+        msg.remove_label('unseen')
+        self.assertEqual(msg.get_labels(), ['resent', 'foobar'])
+        msg.set_labels(['foobar', 'answered'])
+        self.assertEqual(msg.get_labels(), ['foobar', 'answered'])
+
+    def test_visible(self):
+        # Get, set, and update visible headers
+        msg = mailbox.BabylMessage(_sample_message)
+        visible = msg.get_visible()
+        self.assertEqual(visible.keys(), [])
+        self.assertIs(visible.get_payload(), None)
+        visible['User-Agent'] = 'FooBar 1.0'
+        visible['X-Whatever'] = 'Blah'
+        self.assertEqual(msg.get_visible().keys(), [])
+        msg.set_visible(visible)
+        visible = msg.get_visible()
+        self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever'])
+        self.assertEqual(visible['User-Agent'], 'FooBar 1.0')
+        self.assertEqual(visible['X-Whatever'], 'Blah')
+        self.assertIs(visible.get_payload(), None)
+        msg.update_visible()
+        self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever'])
+        self.assertIs(visible.get_payload(), None)
+        visible = msg.get_visible()
+        self.assertEqual(visible.keys(), ['User-Agent', 'Date', 'From', 'To',
+                                          'Subject'])
+        for header in ('User-Agent', 'Date', 'From', 'To', 'Subject'):
+            self.assertEqual(visible[header], msg[header])
+
+
+class TestMMDFMessage(_TestMboxMMDFMessage):
+
+    _factory = mailbox.MMDFMessage
+
+
+class TestMessageConversion(TestBase):
+
+    def test_plain_to_x(self):
+        # Convert Message to all formats
+        for class_ in (mailbox.Message, mailbox.MaildirMessage,
+                       mailbox.mboxMessage, mailbox.MHMessage,
+                       mailbox.BabylMessage, mailbox.MMDFMessage):
+            msg_plain = mailbox.Message(_sample_message)
+            msg = class_(msg_plain)
+            self._check_sample(msg)
+
+    def test_x_to_plain(self):
+        # Convert all formats to Message
+        for class_ in (mailbox.Message, mailbox.MaildirMessage,
+                       mailbox.mboxMessage, mailbox.MHMessage,
+                       mailbox.BabylMessage, mailbox.MMDFMessage):
+            msg = class_(_sample_message)
+            msg_plain = mailbox.Message(msg)
+            self._check_sample(msg_plain)
+
+    def test_x_to_invalid(self):
+        # Convert all formats to an invalid format
+        for class_ in (mailbox.Message, mailbox.MaildirMessage,
+                       mailbox.mboxMessage, mailbox.MHMessage,
+                       mailbox.BabylMessage, mailbox.MMDFMessage):
+            self.assertRaises(TypeError, lambda: class_(False))
+
+    def test_maildir_to_maildir(self):
+        # Convert MaildirMessage to MaildirMessage
+        msg_maildir = mailbox.MaildirMessage(_sample_message)
+        msg_maildir.set_flags('DFPRST')
+        msg_maildir.set_subdir('cur')
+        date = msg_maildir.get_date()
+        msg = mailbox.MaildirMessage(msg_maildir)
+        self._check_sample(msg)
+        self.assertEqual(msg.get_flags(), 'DFPRST')
+        self.assertEqual(msg.get_subdir(), 'cur')
+        self.assertEqual(msg.get_date(), date)
+
+    def test_maildir_to_mboxmmdf(self):
+        # Convert MaildirMessage to mboxmessage and MMDFMessage
+        pairs = (('D', ''), ('F', 'F'), ('P', ''), ('R', 'A'), ('S', 'R'),
+                 ('T', 'D'), ('DFPRST', 'RDFA'))
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg_maildir = mailbox.MaildirMessage(_sample_message)
+            msg_maildir.set_date(0.0)
+            for setting, result in pairs:
+                msg_maildir.set_flags(setting)
+                msg = class_(msg_maildir)
+                self.assertEqual(msg.get_flags(), result)
+                self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' %
+                                 time.asctime(time.gmtime(0.0)))
+            msg_maildir.set_subdir('cur')
+            self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA')
+
+    def test_maildir_to_mh(self):
+        # Convert MaildirMessage to MHMessage
+        msg_maildir = mailbox.MaildirMessage(_sample_message)
+        pairs = (('D', ['unseen']), ('F', ['unseen', 'flagged']),
+                 ('P', ['unseen']), ('R', ['unseen', 'replied']), ('S', []),
+                 ('T', ['unseen']), ('DFPRST', ['replied', 'flagged']))
+        for setting, result in pairs:
+            msg_maildir.set_flags(setting)
+            self.assertEqual(mailbox.MHMessage(msg_maildir).get_sequences(),
+                             result)
+
+    def test_maildir_to_babyl(self):
+        # Convert MaildirMessage to Babyl
+        msg_maildir = mailbox.MaildirMessage(_sample_message)
+        pairs = (('D', ['unseen']), ('F', ['unseen']),
+                 ('P', ['unseen', 'forwarded']), ('R', ['unseen', 'answered']),
+                 ('S', []), ('T', ['unseen', 'deleted']),
+                 ('DFPRST', ['deleted', 'answered', 'forwarded']))
+        for setting, result in pairs:
+            msg_maildir.set_flags(setting)
+            self.assertEqual(mailbox.BabylMessage(msg_maildir).get_labels(),
+                             result)
+
+    def test_mboxmmdf_to_maildir(self):
+        # Convert mboxMessage and MMDFMessage to MaildirMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg_mboxMMDF = class_(_sample_message)
+            msg_mboxMMDF.set_from('foo at bar', time.gmtime(0.0))
+            pairs = (('R', 'S'), ('O', ''), ('D', 'T'), ('F', 'F'), ('A', 'R'),
+                     ('RODFA', 'FRST'))
+            for setting, result in pairs:
+                msg_mboxMMDF.set_flags(setting)
+                msg = mailbox.MaildirMessage(msg_mboxMMDF)
+                self.assertEqual(msg.get_flags(), result)
+                self.assertEqual(msg.get_date(), 0.0)
+            msg_mboxMMDF.set_flags('O')
+            self.assertEqual(mailbox.MaildirMessage(msg_mboxMMDF).get_subdir(),
+                             'cur')
+
+    def test_mboxmmdf_to_mboxmmdf(self):
+        # Convert mboxMessage and MMDFMessage to mboxMessage and MMDFMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg_mboxMMDF = class_(_sample_message)
+            msg_mboxMMDF.set_flags('RODFA')
+            msg_mboxMMDF.set_from('foo at bar')
+            for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+                msg2 = class2_(msg_mboxMMDF)
+                self.assertEqual(msg2.get_flags(), 'RODFA')
+                self.assertEqual(msg2.get_from(), 'foo at bar')
+
+    def test_mboxmmdf_to_mh(self):
+        # Convert mboxMessage and MMDFMessage to MHMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg_mboxMMDF = class_(_sample_message)
+            pairs = (('R', []), ('O', ['unseen']), ('D', ['unseen']),
+                     ('F', ['unseen', 'flagged']),
+                     ('A', ['unseen', 'replied']),
+                     ('RODFA', ['replied', 'flagged']))
+            for setting, result in pairs:
+                msg_mboxMMDF.set_flags(setting)
+                self.assertEqual(mailbox.MHMessage(msg_mboxMMDF).get_sequences(),
+                                 result)
+
+    def test_mboxmmdf_to_babyl(self):
+        # Convert mboxMessage and MMDFMessage to BabylMessage
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            msg = class_(_sample_message)
+            pairs = (('R', []), ('O', ['unseen']),
+                     ('D', ['unseen', 'deleted']), ('F', ['unseen']),
+                     ('A', ['unseen', 'answered']),
+                     ('RODFA', ['deleted', 'answered']))
+            for setting, result in pairs:
+                msg.set_flags(setting)
+                self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result)
+
+    def test_mh_to_maildir(self):
+        # Convert MHMessage to MaildirMessage
+        pairs = (('unseen', ''), ('replied', 'RS'), ('flagged', 'FS'))
+        for setting, result in pairs:
+            msg = mailbox.MHMessage(_sample_message)
+            msg.add_sequence(setting)
+            self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result)
+            self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+        msg = mailbox.MHMessage(_sample_message)
+        msg.add_sequence('unseen')
+        msg.add_sequence('replied')
+        msg.add_sequence('flagged')
+        self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'FR')
+        self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+
+    def test_mh_to_mboxmmdf(self):
+        # Convert MHMessage to mboxMessage and MMDFMessage
+        pairs = (('unseen', 'O'), ('replied', 'ROA'), ('flagged', 'ROF'))
+        for setting, result in pairs:
+            msg = mailbox.MHMessage(_sample_message)
+            msg.add_sequence(setting)
+            for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+                self.assertEqual(class_(msg).get_flags(), result)
+        msg = mailbox.MHMessage(_sample_message)
+        msg.add_sequence('unseen')
+        msg.add_sequence('replied')
+        msg.add_sequence('flagged')
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            self.assertEqual(class_(msg).get_flags(), 'OFA')
+
+    def test_mh_to_mh(self):
+        # Convert MHMessage to MHMessage
+        msg = mailbox.MHMessage(_sample_message)
+        msg.add_sequence('unseen')
+        msg.add_sequence('replied')
+        msg.add_sequence('flagged')
+        self.assertEqual(mailbox.MHMessage(msg).get_sequences(),
+                         ['unseen', 'replied', 'flagged'])
+
+    def test_mh_to_babyl(self):
+        # Convert MHMessage to BabylMessage
+        pairs = (('unseen', ['unseen']), ('replied', ['answered']),
+                 ('flagged', []))
+        for setting, result in pairs:
+            msg = mailbox.MHMessage(_sample_message)
+            msg.add_sequence(setting)
+            self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result)
+        msg = mailbox.MHMessage(_sample_message)
+        msg.add_sequence('unseen')
+        msg.add_sequence('replied')
+        msg.add_sequence('flagged')
+        self.assertEqual(mailbox.BabylMessage(msg).get_labels(),
+                         ['unseen', 'answered'])
+
+    def test_babyl_to_maildir(self):
+        # Convert BabylMessage to MaildirMessage
+        pairs = (('unseen', ''), ('deleted', 'ST'), ('filed', 'S'),
+                 ('answered', 'RS'), ('forwarded', 'PS'), ('edited', 'S'),
+                 ('resent', 'PS'))
+        for setting, result in pairs:
+            msg = mailbox.BabylMessage(_sample_message)
+            msg.add_label(setting)
+            self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result)
+            self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+        msg = mailbox.BabylMessage(_sample_message)
+        for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+                      'edited', 'resent'):
+            msg.add_label(label)
+        self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'PRT')
+        self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur')
+
+    def test_babyl_to_mboxmmdf(self):
+        # Convert BabylMessage to mboxMessage and MMDFMessage
+        pairs = (('unseen', 'O'), ('deleted', 'ROD'), ('filed', 'RO'),
+                 ('answered', 'ROA'), ('forwarded', 'RO'), ('edited', 'RO'),
+                 ('resent', 'RO'))
+        for setting, result in pairs:
+            for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+                msg = mailbox.BabylMessage(_sample_message)
+                msg.add_label(setting)
+                self.assertEqual(class_(msg).get_flags(), result)
+        msg = mailbox.BabylMessage(_sample_message)
+        for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+                      'edited', 'resent'):
+            msg.add_label(label)
+        for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage):
+            self.assertEqual(class_(msg).get_flags(), 'ODA')
+
+    def test_babyl_to_mh(self):
+        # Convert BabylMessage to MHMessage
+        pairs = (('unseen', ['unseen']), ('deleted', []), ('filed', []),
+                 ('answered', ['replied']), ('forwarded', []), ('edited', []),
+                 ('resent', []))
+        for setting, result in pairs:
+            msg = mailbox.BabylMessage(_sample_message)
+            msg.add_label(setting)
+            self.assertEqual(mailbox.MHMessage(msg).get_sequences(), result)
+        msg = mailbox.BabylMessage(_sample_message)
+        for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+                      'edited', 'resent'):
+            msg.add_label(label)
+        self.assertEqual(mailbox.MHMessage(msg).get_sequences(),
+                         ['unseen', 'replied'])
+
+    def test_babyl_to_babyl(self):
+        # Convert BabylMessage to BabylMessage
+        msg = mailbox.BabylMessage(_sample_message)
+        msg.update_visible()
+        for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded',
+                      'edited', 'resent'):
+            msg.add_label(label)
+        msg2 = mailbox.BabylMessage(msg)
+        self.assertEqual(msg2.get_labels(), ['unseen', 'deleted', 'filed',
+                                             'answered', 'forwarded', 'edited',
+                                             'resent'])
+        self.assertEqual(msg.get_visible().keys(), msg2.get_visible().keys())
+        for key in msg.get_visible().keys():
+            self.assertEqual(msg.get_visible()[key], msg2.get_visible()[key])
+
+
+class TestProxyFileBase(TestBase):
+
+    def _test_read(self, proxy):
+        # Read by byte
+        proxy.seek(0)
+        self.assertEqual(proxy.read(), 'bar')
+        proxy.seek(1)
+        self.assertEqual(proxy.read(), 'ar')
+        proxy.seek(0)
+        self.assertEqual(proxy.read(2), 'ba')
+        proxy.seek(1)
+        self.assertEqual(proxy.read(-1), 'ar')
+        proxy.seek(2)
+        self.assertEqual(proxy.read(1000), 'r')
+
+    def _test_readline(self, proxy):
+        # Read by line
+        proxy.seek(0)
+        self.assertEqual(proxy.readline(), 'foo' + os.linesep)
+        self.assertEqual(proxy.readline(), 'bar' + os.linesep)
+        self.assertEqual(proxy.readline(), 'fred' + os.linesep)
+        self.assertEqual(proxy.readline(), 'bob')
+        proxy.seek(2)
+        self.assertEqual(proxy.readline(), 'o' + os.linesep)
+        proxy.seek(6 + 2 * len(os.linesep))
+        self.assertEqual(proxy.readline(), 'fred' + os.linesep)
+        proxy.seek(6 + 2 * len(os.linesep))
+        self.assertEqual(proxy.readline(2), 'fr')
+        self.assertEqual(proxy.readline(-10), 'ed' + os.linesep)
+
+    def _test_readlines(self, proxy):
+        # Read multiple lines
+        proxy.seek(0)
+        self.assertEqual(proxy.readlines(), ['foo' + os.linesep,
+                                            'bar' + os.linesep,
+                                            'fred' + os.linesep, 'bob'])
+        proxy.seek(0)
+        self.assertEqual(proxy.readlines(2), ['foo' + os.linesep])
+        proxy.seek(3 + len(os.linesep))
+        self.assertEqual(proxy.readlines(4 + len(os.linesep)),
+                         ['bar' + os.linesep, 'fred' + os.linesep])
+        proxy.seek(3)
+        self.assertEqual(proxy.readlines(1000), [os.linesep, 'bar' + os.linesep,
+                                                 'fred' + os.linesep, 'bob'])
+
+    def _test_iteration(self, proxy):
+        # Iterate by line
+        proxy.seek(0)
+        iterator = iter(proxy)
+        self.assertEqual(list(iterator),
+            ['foo' + os.linesep, 'bar' + os.linesep, 'fred' + os.linesep, 'bob'])
+
+    def _test_seek_and_tell(self, proxy):
+        # Seek and use tell to check position
+        proxy.seek(3)
+        self.assertEqual(proxy.tell(), 3)
+        self.assertEqual(proxy.read(len(os.linesep)), os.linesep)
+        proxy.seek(2, 1)
+        self.assertEqual(proxy.read(1 + len(os.linesep)), 'r' + os.linesep)
+        proxy.seek(-3 - len(os.linesep), 2)
+        self.assertEqual(proxy.read(3), 'bar')
+        proxy.seek(2, 0)
+        self.assertEqual(proxy.read(), 'o' + os.linesep + 'bar' + os.linesep)
+        proxy.seek(100)
+        self.assertEqual(proxy.read(), '')
+
+    def _test_close(self, proxy):
+        # Close a file
+        proxy.close()
+        self.assertRaises(AttributeError, lambda: proxy.close())
+
+
+class TestProxyFile(TestProxyFileBase):
+
+    def setUp(self):
+        self._path = test_support.TESTFN
+        self._file = open(self._path, 'wb+')
+
+    def tearDown(self):
+        self._file.close()
+        self._delete_recursively(self._path)
+
+    def test_initialize(self):
+        # Initialize and check position
+        self._file.write('foo')
+        pos = self._file.tell()
+        proxy0 = mailbox._ProxyFile(self._file)
+        self.assertEqual(proxy0.tell(), pos)
+        self.assertEqual(self._file.tell(), pos)
+        proxy1 = mailbox._ProxyFile(self._file, 0)
+        self.assertEqual(proxy1.tell(), 0)
+        self.assertEqual(self._file.tell(), pos)
+
+    def test_read(self):
+        self._file.write('bar')
+        self._test_read(mailbox._ProxyFile(self._file))
+
+    def test_readline(self):
+        self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+                                                  os.linesep))
+        self._test_readline(mailbox._ProxyFile(self._file))
+
+    def test_readlines(self):
+        self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+                                                  os.linesep))
+        self._test_readlines(mailbox._ProxyFile(self._file))
+
+    def test_iteration(self):
+        self._file.write('foo%sbar%sfred%sbob' % (os.linesep, os.linesep,
+                                                  os.linesep))
+        self._test_iteration(mailbox._ProxyFile(self._file))
+
+    def test_seek_and_tell(self):
+        self._file.write('foo%sbar%s' % (os.linesep, os.linesep))
+        self._test_seek_and_tell(mailbox._ProxyFile(self._file))
+
+    def test_close(self):
+        self._file.write('foo%sbar%s' % (os.linesep, os.linesep))
+        self._test_close(mailbox._ProxyFile(self._file))
+
+
+class TestPartialFile(TestProxyFileBase):
+
+    def setUp(self):
+        self._path = test_support.TESTFN
+        self._file = open(self._path, 'wb+')
+
+    def tearDown(self):
+        self._file.close()
+        self._delete_recursively(self._path)
+
+    def test_initialize(self):
+        # Initialize and check position
+        self._file.write('foo' + os.linesep + 'bar')
+        pos = self._file.tell()
+        proxy = mailbox._PartialFile(self._file, 2, 5)
+        self.assertEqual(proxy.tell(), 0)
+        self.assertEqual(self._file.tell(), pos)
+
+    def test_read(self):
+        self._file.write('***bar***')
+        self._test_read(mailbox._PartialFile(self._file, 3, 6))
+
+    def test_readline(self):
+        self._file.write('!!!!!foo%sbar%sfred%sbob!!!!!' %
+                         (os.linesep, os.linesep, os.linesep))
+        self._test_readline(mailbox._PartialFile(self._file, 5,
+                                                 18 + 3 * len(os.linesep)))
+
+    def test_readlines(self):
+        self._file.write('foo%sbar%sfred%sbob?????' %
+                         (os.linesep, os.linesep, os.linesep))
+        self._test_readlines(mailbox._PartialFile(self._file, 0,
+                                                  13 + 3 * len(os.linesep)))
+
+    def test_iteration(self):
+        self._file.write('____foo%sbar%sfred%sbob####' %
+                         (os.linesep, os.linesep, os.linesep))
+        self._test_iteration(mailbox._PartialFile(self._file, 4,
+                                                  17 + 3 * len(os.linesep)))
+
+    def test_seek_and_tell(self):
+        self._file.write('(((foo%sbar%s$$$' % (os.linesep, os.linesep))
+        self._test_seek_and_tell(mailbox._PartialFile(self._file, 3,
+                                                      9 + 2 * len(os.linesep)))
+
+    def test_close(self):
+        self._file.write('&foo%sbar%s^' % (os.linesep, os.linesep))
+        self._test_close(mailbox._PartialFile(self._file, 1,
+                                              6 + 3 * len(os.linesep)))
+
+
+## Start: tests from the original module (for backward compatibility).
+
+FROM_ = "From some.body at dummy.domain  Sat Jul 24 13:43:35 2004\n"
+DUMMY_MESSAGE = """\
+From: some.body at dummy.domain
+To: me at my.domain
+Subject: Simple Test
+
+This is a dummy message.
+"""
+
+class MaildirTestCase(unittest.TestCase):
+
+    def setUp(self):
+        # create a new maildir mailbox to work with:
+        self._dir = test_support.TESTFN
+        os.mkdir(self._dir)
+        os.mkdir(os.path.join(self._dir, "cur"))
+        os.mkdir(os.path.join(self._dir, "tmp"))
+        os.mkdir(os.path.join(self._dir, "new"))
+        self._counter = 1
+        self._msgfiles = []
+
+    def tearDown(self):
+        map(os.unlink, self._msgfiles)
+        os.rmdir(os.path.join(self._dir, "cur"))
+        os.rmdir(os.path.join(self._dir, "tmp"))
+        os.rmdir(os.path.join(self._dir, "new"))
+        os.rmdir(self._dir)
+
+    def createMessage(self, dir, mbox=False):
+        t = int(time.time() % 1000000)
+        pid = self._counter
+        self._counter += 1
+        filename = os.extsep.join((str(t), str(pid), "myhostname", "mydomain"))
+        tmpname = os.path.join(self._dir, "tmp", filename)
+        newname = os.path.join(self._dir, dir, filename)
+        with open(tmpname, "w") as fp:
+            self._msgfiles.append(tmpname)
+            if mbox:
+                fp.write(FROM_)
+            fp.write(DUMMY_MESSAGE)
+        if hasattr(os, "link"):
+            os.link(tmpname, newname)
+        else:
+            with open(newname, "w") as fp:
+                fp.write(DUMMY_MESSAGE)
+        self._msgfiles.append(newname)
+        return tmpname
+
+    def test_empty_maildir(self):
+        """Test an empty maildir mailbox"""
+        # Test for regression on bug #117490:
+        # Make sure the boxes attribute actually gets set.
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        #self.assertTrue(hasattr(self.mbox, "boxes"))
+        #self.assertTrue(len(self.mbox.boxes) == 0)
+        self.assertIs(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+
+    def test_nonempty_maildir_cur(self):
+        self.createMessage("cur")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        #self.assertTrue(len(self.mbox.boxes) == 1)
+        self.assertIsNot(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+
+    def test_nonempty_maildir_new(self):
+        self.createMessage("new")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        #self.assertTrue(len(self.mbox.boxes) == 1)
+        self.assertIsNot(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+
+    def test_nonempty_maildir_both(self):
+        self.createMessage("cur")
+        self.createMessage("new")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        #self.assertTrue(len(self.mbox.boxes) == 2)
+        self.assertIsNot(self.mbox.next(), None)
+        self.assertIsNot(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+        self.assertIs(self.mbox.next(), None)
+
+    def test_unix_mbox(self):
+        ### should be better!
+        import email.parser
+        fname = self.createMessage("cur", True)
+        n = 0
+        for msg in mailbox.PortableUnixMailbox(open(fname),
+                                               email.parser.Parser().parse):
+            n += 1
+            self.assertEqual(msg["subject"], "Simple Test")
+            self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+        self.assertEqual(n, 1)
+
+## End: classes from the original module (for backward compatibility).
+
+
+_sample_message = """\
+Return-Path: <gkj at gregorykjohnson.com>
+X-Original-To: gkj+person at localhost
+Delivered-To: gkj+person at localhost
+Received: from localhost (localhost [127.0.0.1])
+        by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17
+        for <gkj+person at localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)
+Delivered-To: gkj at sundance.gregorykjohnson.com
+Received: from localhost [127.0.0.1]
+        by localhost with POP3 (fetchmail-6.2.5)
+        for gkj+person at localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)
+Received: from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228])
+        by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746
+        for <gkj at gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)
+Received: by andy.gregorykjohnson.com (Postfix, from userid 1000)
+        id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)
+Date: Wed, 13 Jul 2005 17:23:11 -0400
+From: "Gregory K. Johnson" <gkj at gregorykjohnson.com>
+To: gkj at gregorykjohnson.com
+Subject: Sample message
+Message-ID: <20050713212311.GC4701 at andy.gregorykjohnson.com>
+Mime-Version: 1.0
+Content-Type: multipart/mixed; boundary="NMuMz9nt05w80d4+"
+Content-Disposition: inline
+User-Agent: Mutt/1.5.9i
+
+
+--NMuMz9nt05w80d4+
+Content-Type: text/plain; charset=us-ascii
+Content-Disposition: inline
+
+This is a sample message.
+
+--
+Gregory K. Johnson
+
+--NMuMz9nt05w80d4+
+Content-Type: application/octet-stream
+Content-Disposition: attachment; filename="text.gz"
+Content-Transfer-Encoding: base64
+
+H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs
+3FYlAAAA
+
+--NMuMz9nt05w80d4+--
+"""
+
+_sample_headers = {
+    "Return-Path":"<gkj at gregorykjohnson.com>",
+    "X-Original-To":"gkj+person at localhost",
+    "Delivered-To":"gkj+person at localhost",
+    "Received":"""from localhost (localhost [127.0.0.1])
+        by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17
+        for <gkj+person at localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)""",
+    "Delivered-To":"gkj at sundance.gregorykjohnson.com",
+    "Received":"""from localhost [127.0.0.1]
+        by localhost with POP3 (fetchmail-6.2.5)
+        for gkj+person at localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)""",
+    "Received":"""from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228])
+        by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746
+        for <gkj at gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)""",
+    "Received":"""by andy.gregorykjohnson.com (Postfix, from userid 1000)
+        id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)""",
+    "Date":"Wed, 13 Jul 2005 17:23:11 -0400",
+    "From":""""Gregory K. Johnson" <gkj at gregorykjohnson.com>""",
+    "To":"gkj at gregorykjohnson.com",
+    "Subject":"Sample message",
+    "Mime-Version":"1.0",
+    "Content-Type":"""multipart/mixed; boundary="NMuMz9nt05w80d4+\"""",
+    "Content-Disposition":"inline",
+    "User-Agent": "Mutt/1.5.9i" }
+
+_sample_payloads = ("""This is a sample message.
+
+--
+Gregory K. Johnson
+""",
+"""H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs
+3FYlAAAA
+""")
+
+
+def test_main():
+    tests = (TestMailboxSuperclass, TestMaildir, TestMbox, TestMMDF, TestMH,
+             TestBabyl, TestMessage, TestMaildirMessage, TestMboxMessage,
+             TestMHMessage, TestBabylMessage, TestMMDFMessage,
+             TestMessageConversion, TestProxyFile, TestPartialFile,
+             MaildirTestCase)
+    test_support.run_unittest(*tests)
+    test_support.reap_children()
+
+
+if __name__ == '__main__':
+    test_main()
diff --git a/lib-python/2.7/test/test_old_mailbox.py b/lib-python/2.7/test/test_old_mailbox.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_old_mailbox.py
@@ -0,0 +1,152 @@
+# This set of tests exercises the backward-compatibility class
+# in mailbox.py (the ones without write support).
+
+import mailbox
+import os
+import time
+import unittest
+from test import test_support
+
+# cleanup earlier tests
+try:
+    os.unlink(test_support.TESTFN)
+except os.error:
+    pass
+
+FROM_ = "From some.body at dummy.domain  Sat Jul 24 13:43:35 2004\n"
+DUMMY_MESSAGE = """\
+From: some.body at dummy.domain
+To: me at my.domain
+Subject: Simple Test
+
+This is a dummy message.
+"""
+
+class MaildirTestCase(unittest.TestCase):
+
+    def setUp(self):
+        # create a new maildir mailbox to work with:
+        self._dir = test_support.TESTFN
+        os.mkdir(self._dir)
+        os.mkdir(os.path.join(self._dir, "cur"))
+        os.mkdir(os.path.join(self._dir, "tmp"))
+        os.mkdir(os.path.join(self._dir, "new"))
+        self._counter = 1
+        self._msgfiles = []
+
+    def tearDown(self):
+        map(os.unlink, self._msgfiles)
+        os.rmdir(os.path.join(self._dir, "cur"))
+        os.rmdir(os.path.join(self._dir, "tmp"))
+        os.rmdir(os.path.join(self._dir, "new"))
+        os.rmdir(self._dir)
+
+    def createMessage(self, dir, mbox=False):
+        t = int(time.time() % 1000000)
+        pid = self._counter
+        self._counter += 1
+        filename = os.extsep.join((str(t), str(pid), "myhostname", "mydomain"))
+        tmpname = os.path.join(self._dir, "tmp", filename)
+        newname = os.path.join(self._dir, dir, filename)
+        with open(tmpname, "w") as fp:
+            self._msgfiles.append(tmpname)
+            if mbox:
+                fp.write(FROM_)
+            fp.write(DUMMY_MESSAGE)
+        if hasattr(os, "link"):
+            os.link(tmpname, newname)
+        else:
+            with open(newname, "w") as fp:
+                fp.write(DUMMY_MESSAGE)
+        self._msgfiles.append(newname)
+        return tmpname
+
+    def test_empty_maildir(self):
+        """Test an empty maildir mailbox"""
+        # Test for regression on bug #117490:
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        self.assertTrue(len(self.mbox) == 0)
+        self.assertTrue(self.mbox.next() is None)
+        self.assertTrue(self.mbox.next() is None)
+
+    def test_nonempty_maildir_cur(self):
+        self.createMessage("cur")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        self.assertTrue(len(self.mbox) == 1)
+        self.assertTrue(self.mbox.next() is not None)
+        self.assertTrue(self.mbox.next() is None)
+        self.assertTrue(self.mbox.next() is None)
+
+    def test_nonempty_maildir_new(self):
+        self.createMessage("new")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        self.assertTrue(len(self.mbox) == 1)
+        self.assertTrue(self.mbox.next() is not None)
+        self.assertTrue(self.mbox.next() is None)
+        self.assertTrue(self.mbox.next() is None)
+
+    def test_nonempty_maildir_both(self):
+        self.createMessage("cur")
+        self.createMessage("new")
+        self.mbox = mailbox.Maildir(test_support.TESTFN)
+        self.assertTrue(len(self.mbox) == 2)
+        self.assertTrue(self.mbox.next() is not None)
+        self.assertTrue(self.mbox.next() is not None)
+        self.assertTrue(self.mbox.next() is None)
+        self.assertTrue(self.mbox.next() is None)
+
+    def test_unix_mbox(self):
+        ### should be better!
+        import email.parser
+        fname = self.createMessage("cur", True)
+        n = 0
+        with open(fname) as f:
+            for msg in mailbox.PortableUnixMailbox(f,
+                                               email.parser.Parser().parse):
+                n += 1
+                self.assertEqual(msg["subject"], "Simple Test")
+                self.assertEqual(len(str(msg)), len(FROM_)+len(DUMMY_MESSAGE))
+        self.assertEqual(n, 1)
+
+class MboxTestCase(unittest.TestCase):
+    def setUp(self):
+        # create a new maildir mailbox to work with:
+        self._path = test_support.TESTFN
+
+    def tearDown(self):
+        os.unlink(self._path)
+
+    def test_from_regex (self):
+        # Testing new regex from bug #1633678
+        with open(self._path, 'w') as f:
+            f.write("""From fred at example.com Mon May 31 13:24:50 2004 +0200
+Subject: message 1
+
+body1
+From fred at example.com Mon May 31 13:24:50 2004 -0200
+Subject: message 2
+
+body2
+From fred at example.com Mon May 31 13:24:50 2004
+Subject: message 3
+
+body3
+From fred at example.com Mon May 31 13:24:50 2004
+Subject: message 4
+
+body4
+""")
+        with open(self._path, 'r') as f:
+            box = mailbox.UnixMailbox(f)
+            self.assertTrue(len(list(iter(box))) == 4)
+
+
+    # XXX We still need more tests!
+
+
+def test_main():
+    test_support.run_unittest(MaildirTestCase, MboxTestCase)
+
+
+if __name__ == "__main__":
+    test_main()
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/test/test_os.py
@@ -0,0 +1,824 @@
+# As a test suite for the os module, this is woefully inadequate, but this
+# does add tests for a few functions which have been determined to be more
+# portable than they had been thought to be.
+
+import os
+import errno
+import unittest
+import warnings
+import sys
+import signal
+import subprocess
+import time
+from test import test_support
+import mmap
+import uuid
+
+warnings.filterwarnings("ignore", "tempnam", RuntimeWarning, __name__)
+warnings.filterwarnings("ignore", "tmpnam", RuntimeWarning, __name__)
+
+# Tests creating TESTFN
+class FileTests(unittest.TestCase):
+    def setUp(self):
+        if os.path.exists(test_support.TESTFN):
+            os.unlink(test_support.TESTFN)
+    tearDown = setUp
+
+    def test_access(self):
+        f = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
+        os.close(f)
+        self.assertTrue(os.access(test_support.TESTFN, os.W_OK))
+
+    def test_closerange(self):
+        first = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
+        # We must allocate two consecutive file descriptors, otherwise
+        # it will mess up other file descriptors (perhaps even the three
+        # standard ones).
+        second = os.dup(first)
+        try:
+            retries = 0
+            while second != first + 1:
+                os.close(first)
+                retries += 1
+                if retries > 10:
+                    # XXX test skipped
+                    self.skipTest("couldn't allocate two consecutive fds")
+                first, second = second, os.dup(second)
+        finally:
+            os.close(second)
+        # close a fd that is open, and one that isn't
+        os.closerange(first, first + 2)
+        self.assertRaises(OSError, os.write, first, "a")
+
+    @test_support.cpython_only
+    def test_rename(self):
+        path = unicode(test_support.TESTFN)
+        old = sys.getrefcount(path)
+        self.assertRaises(TypeError, os.rename, path, 0)
+        new = sys.getrefcount(path)
+        self.assertEqual(old, new)
+
+
+class TemporaryFileTests(unittest.TestCase):
+    def setUp(self):
+        self.files = []
+        os.mkdir(test_support.TESTFN)
+
+    def tearDown(self):
+        for name in self.files:
+            os.unlink(name)
+        os.rmdir(test_support.TESTFN)
+
+    def check_tempfile(self, name):
+        # make sure it doesn't already exist:
+        self.assertFalse(os.path.exists(name),
+                    "file already exists for temporary file")
+        # make sure we can create the file
+        open(name, "w")
+        self.files.append(name)
+
+    def test_tempnam(self):
+        if not hasattr(os, "tempnam"):
+            return
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "tempnam", RuntimeWarning,
+                                    r"test_os$")
+            warnings.filterwarnings("ignore", "tempnam", DeprecationWarning)
+            self.check_tempfile(os.tempnam())
+
+            name = os.tempnam(test_support.TESTFN)
+            self.check_tempfile(name)
+
+            name = os.tempnam(test_support.TESTFN, "pfx")
+            self.assertTrue(os.path.basename(name)[:3] == "pfx")
+            self.check_tempfile(name)
+
+    def test_tmpfile(self):
+        if not hasattr(os, "tmpfile"):
+            return
+        # As with test_tmpnam() below, the Windows implementation of tmpfile()
+        # attempts to create a file in the root directory of the current drive.
+        # On Vista and Server 2008, this test will always fail for normal users
+        # as writing to the root directory requires elevated privileges.  With
+        # XP and below, the semantics of tmpfile() are the same, but the user
+        # running the test is more likely to have administrative privileges on
+        # their account already.  If that's the case, then os.tmpfile() should
+        # work.  In order to make this test as useful as possible, rather than
+        # trying to detect Windows versions or whether or not the user has the
+        # right permissions, just try and create a file in the root directory
+        # and see if it raises a 'Permission denied' OSError.  If it does, then
+        # test that a subsequent call to os.tmpfile() raises the same error. If
+        # it doesn't, assume we're on XP or below and the user running the test
+        # has administrative privileges, and proceed with the test as normal.
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "tmpfile", DeprecationWarning)
+
+            if sys.platform == 'win32':
+                name = '\\python_test_os_test_tmpfile.txt'
+                if os.path.exists(name):
+                    os.remove(name)
+                try:
+                    fp = open(name, 'w')
+                except IOError, first:
+                    # open() failed, assert tmpfile() fails in the same way.
+                    # Although open() raises an IOError and os.tmpfile() raises an
+                    # OSError(), 'args' will be (13, 'Permission denied') in both
+                    # cases.
+                    try:
+                        fp = os.tmpfile()
+                    except OSError, second:
+                        self.assertEqual(first.args, second.args)
+                    else:
+                        self.fail("expected os.tmpfile() to raise OSError")
+                    return
+                else:
+                    # open() worked, therefore, tmpfile() should work.  Close our
+                    # dummy file and proceed with the test as normal.
+                    fp.close()
+                    os.remove(name)
+
+            fp = os.tmpfile()
+            fp.write("foobar")
+            fp.seek(0,0)
+            s = fp.read()
+            fp.close()
+            self.assertTrue(s == "foobar")
+
+    def test_tmpnam(self):
+        if not hasattr(os, "tmpnam"):
+            return
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "tmpnam", RuntimeWarning,
+                                    r"test_os$")
+            warnings.filterwarnings("ignore", "tmpnam", DeprecationWarning)
+
+            name = os.tmpnam()
+            if sys.platform in ("win32",):
+                # The Windows tmpnam() seems useless.  From the MS docs:
+                #
+                #     The character string that tmpnam creates consists of
+                #     the path prefix, defined by the entry P_tmpdir in the
+                #     file STDIO.H, followed by a sequence consisting of the
+                #     digit characters '0' through '9'; the numerical value
+                #     of this string is in the range 1 - 65,535.  Changing the
+                #     definitions of L_tmpnam or P_tmpdir in STDIO.H does not
+                #     change the operation of tmpnam.
+                #
+                # The really bizarre part is that, at least under MSVC6,
+                # P_tmpdir is "\\".  That is, the path returned refers to
+                # the root of the current drive.  That's a terrible place to
+                # put temp files, and, depending on privileges, the user
+                # may not even be able to open a file in the root directory.
+                self.assertFalse(os.path.exists(name),
+                            "file already exists for temporary file")
+            else:
+                self.check_tempfile(name)
+
+# Test attributes on return values from os.*stat* family.
+class StatAttributeTests(unittest.TestCase):
+    def setUp(self):
+        os.mkdir(test_support.TESTFN)
+        self.fname = os.path.join(test_support.TESTFN, "f1")
+        f = open(self.fname, 'wb')
+        f.write("ABC")
+        f.close()
+
+    def tearDown(self):
+        os.unlink(self.fname)
+        os.rmdir(test_support.TESTFN)
+
+    def test_stat_attributes(self):
+        if not hasattr(os, "stat"):
+            return
+
+        import stat
+        result = os.stat(self.fname)
+
+        # Make sure direct access works
+        self.assertEqual(result[stat.ST_SIZE], 3)
+        self.assertEqual(result.st_size, 3)
+
+        # Make sure all the attributes are there
+        members = dir(result)
+        for name in dir(stat):
+            if name[:3] == 'ST_':
+                attr = name.lower()
+                if name.endswith("TIME"):
+                    def trunc(x): return int(x)
+                else:
+                    def trunc(x): return x
+                self.assertEqual(trunc(getattr(result, attr)),
+                                 result[getattr(stat, name)])
+                self.assertIn(attr, members)
+
+        try:
+            result[200]
+            self.fail("No exception thrown")
+        except IndexError:
+            pass
+
+        # Make sure that assignment fails
+        try:
+            result.st_mode = 1
+            self.fail("No exception thrown")
+        except (AttributeError, TypeError):
+            pass
+
+        try:
+            result.st_rdev = 1
+            self.fail("No exception thrown")
+        except (AttributeError, TypeError):
+            pass
+
+        try:
+            result.parrot = 1
+            self.fail("No exception thrown")
+        except AttributeError:
+            pass
+
+        # Use the stat_result constructor with a too-short tuple.
+        try:
+            result2 = os.stat_result((10,))
+            self.fail("No exception thrown")
+        except TypeError:
+            pass
+
+        # Use the constructor with a too-long tuple.
+        try:
+            result2 = os.stat_result((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14))
+        except TypeError:
+            pass
+
+
+    def test_statvfs_attributes(self):
+        if not hasattr(os, "statvfs"):
+            return
+
+        try:
+            result = os.statvfs(self.fname)
+        except OSError, e:
+            # On AtheOS, glibc always returns ENOSYS
+            if e.errno == errno.ENOSYS:
+                return
+
+        # Make sure direct access works
+        self.assertEqual(result.f_bfree, result[3])
+
+        # Make sure all the attributes are there.
+        members = ('bsize', 'frsize', 'blocks', 'bfree', 'bavail', 'files',
+                    'ffree', 'favail', 'flag', 'namemax')
+        for value, member in enumerate(members):
+            self.assertEqual(getattr(result, 'f_' + member), result[value])
+
+        # Make sure that assignment really fails
+        try:
+            result.f_bfree = 1
+            self.fail("No exception thrown")
+        except TypeError:
+            pass
+
+        try:
+            result.parrot = 1
+            self.fail("No exception thrown")
+        except AttributeError:
+            pass
+
+        # Use the constructor with a too-short tuple.
+        try:
+            result2 = os.statvfs_result((10,))
+            self.fail("No exception thrown")
+        except TypeError:
+            pass
+
+        # Use the constructor with a too-long tuple.
+        try:
+            result2 = os.statvfs_result((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14))
+        except TypeError:
+            pass
+
+    def test_utime_dir(self):
+        delta = 1000000
+        st = os.stat(test_support.TESTFN)
+        # round to int, because some systems may support sub-second
+        # time stamps in stat, but not in utime.
+        os.utime(test_support.TESTFN, (st.st_atime, int(st.st_mtime-delta)))
+        st2 = os.stat(test_support.TESTFN)
+        self.assertEqual(st2.st_mtime, int(st.st_mtime-delta))
+
+    # Restrict test to Win32, since there is no guarantee other
+    # systems support centiseconds
+    if sys.platform == 'win32':
+        def get_file_system(path):
+            root = os.path.splitdrive(os.path.abspath(path))[0] + '\\'
+            import ctypes
+            kernel32 = ctypes.windll.kernel32
+            buf = ctypes.create_string_buffer("", 100)
+            if kernel32.GetVolumeInformationA(root, None, 0, None, None, None, buf, len(buf)):
+                return buf.value
+
+        if get_file_system(test_support.TESTFN) == "NTFS":
+            def test_1565150(self):
+                t1 = 1159195039.25
+                os.utime(self.fname, (t1, t1))
+                self.assertEqual(os.stat(self.fname).st_mtime, t1)
+
+            def test_large_time(self):
+                t1 = 5000000000 # some day in 2128
+                os.utime(self.fname, (t1, t1))
+                self.assertEqual(os.stat(self.fname).st_mtime, t1)
+
+        def test_1686475(self):
+            # Verify that an open file can be stat'ed
+            try:
+                os.stat(r"c:\pagefile.sys")
+            except WindowsError, e:
+                if e.errno == 2: # file does not exist; cannot run test
+                    return
+                self.fail("Could not stat pagefile.sys")
+
+from test import mapping_tests
+
+class EnvironTests(mapping_tests.BasicTestMappingProtocol):
+    """check that os.environ object conform to mapping protocol"""
+    type2test = None
+    def _reference(self):
+        return {"KEY1":"VALUE1", "KEY2":"VALUE2", "KEY3":"VALUE3"}
+    def _empty_mapping(self):
+        os.environ.clear()
+        return os.environ
+    def setUp(self):
+        self.__save = dict(os.environ)
+        os.environ.clear()
+    def tearDown(self):
+        os.environ.clear()
+        os.environ.update(self.__save)
+
+    # Bug 1110478
+    def test_update2(self):
+        if os.path.exists("/bin/sh"):
+            os.environ.update(HELLO="World")
+            with os.popen("/bin/sh -c 'echo $HELLO'") as popen:
+                value = popen.read().strip()
+                self.assertEqual(value, "World")
+
+class WalkTests(unittest.TestCase):
+    """Tests for os.walk()."""
+
+    def test_traversal(self):
+        import os
+        from os.path import join
+
+        # Build:
+        #     TESTFN/
+        #       TEST1/              a file kid and two directory kids
+        #         tmp1
+        #         SUB1/             a file kid and a directory kid
+        #           tmp2
+        #           SUB11/          no kids
+        #         SUB2/             a file kid and a dirsymlink kid
+        #           tmp3
+        #           link/           a symlink to TESTFN.2
+        #       TEST2/
+        #         tmp4              a lone file
+        walk_path = join(test_support.TESTFN, "TEST1")
+        sub1_path = join(walk_path, "SUB1")
+        sub11_path = join(sub1_path, "SUB11")
+        sub2_path = join(walk_path, "SUB2")
+        tmp1_path = join(walk_path, "tmp1")
+        tmp2_path = join(sub1_path, "tmp2")
+        tmp3_path = join(sub2_path, "tmp3")
+        link_path = join(sub2_path, "link")
+        t2_path = join(test_support.TESTFN, "TEST2")
+        tmp4_path = join(test_support.TESTFN, "TEST2", "tmp4")
+
+        # Create stuff.
+        os.makedirs(sub11_path)
+        os.makedirs(sub2_path)
+        os.makedirs(t2_path)
+        for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path:
+            f = file(path, "w")
+            f.write("I'm " + path + " and proud of it.  Blame test_os.\n")
+            f.close()
+        if hasattr(os, "symlink"):
+            os.symlink(os.path.abspath(t2_path), link_path)
+            sub2_tree = (sub2_path, ["link"], ["tmp3"])
+        else:
+            sub2_tree = (sub2_path, [], ["tmp3"])
+
+        # Walk top-down.
+        all = list(os.walk(walk_path))
+        self.assertEqual(len(all), 4)
+        # We can't know which order SUB1 and SUB2 will appear in.
+        # Not flipped:  TESTFN, SUB1, SUB11, SUB2
+        #     flipped:  TESTFN, SUB2, SUB1, SUB11
+        flipped = all[0][1][0] != "SUB1"
+        all[0][1].sort()
+        self.assertEqual(all[0], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
+        self.assertEqual(all[1 + flipped], (sub1_path, ["SUB11"], ["tmp2"]))
+        self.assertEqual(all[2 + flipped], (sub11_path, [], []))
+        self.assertEqual(all[3 - 2 * flipped], sub2_tree)
+
+        # Prune the search.
+        all = []
+        for root, dirs, files in os.walk(walk_path):
+            all.append((root, dirs, files))
+            # Don't descend into SUB1.
+            if 'SUB1' in dirs:
+                # Note that this also mutates the dirs we appended to all!
+                dirs.remove('SUB1')
+        self.assertEqual(len(all), 2)
+        self.assertEqual(all[0], (walk_path, ["SUB2"], ["tmp1"]))
+        self.assertEqual(all[1], sub2_tree)
+
+        # Walk bottom-up.
+        all = list(os.walk(walk_path, topdown=False))
+        self.assertEqual(len(all), 4)
+        # We can't know which order SUB1 and SUB2 will appear in.
+        # Not flipped:  SUB11, SUB1, SUB2, TESTFN
+        #     flipped:  SUB2, SUB11, SUB1, TESTFN
+        flipped = all[3][1][0] != "SUB1"
+        all[3][1].sort()
+        self.assertEqual(all[3], (walk_path, ["SUB1", "SUB2"], ["tmp1"]))
+        self.assertEqual(all[flipped], (sub11_path, [], []))
+        self.assertEqual(all[flipped + 1], (sub1_path, ["SUB11"], ["tmp2"]))
+        self.assertEqual(all[2 - 2 * flipped], sub2_tree)
+
+        if hasattr(os, "symlink"):
+            # Walk, following symlinks.
+            for root, dirs, files in os.walk(walk_path, followlinks=True):
+                if root == link_path:
+                    self.assertEqual(dirs, [])
+                    self.assertEqual(files, ["tmp4"])
+                    break
+            else:
+                self.fail("Didn't follow symlink with followlinks=True")
+
+    def tearDown(self):
+        # Tear everything down.  This is a decent use for bottom-up on
+        # Windows, which doesn't have a recursive delete command.  The
+        # (not so) subtlety is that rmdir will fail unless the dir's
+        # kids are removed first, so bottom up is essential.
+        for root, dirs, files in os.walk(test_support.TESTFN, topdown=False):
+            for name in files:
+                os.remove(os.path.join(root, name))
+            for name in dirs:
+                dirname = os.path.join(root, name)
+                if not os.path.islink(dirname):
+                    os.rmdir(dirname)
+                else:
+                    os.remove(dirname)
+        os.rmdir(test_support.TESTFN)
+
+class MakedirTests (unittest.TestCase):
+    def setUp(self):
+        os.mkdir(test_support.TESTFN)
+
+    def test_makedir(self):
+        base = test_support.TESTFN
+        path = os.path.join(base, 'dir1', 'dir2', 'dir3')
+        os.makedirs(path)             # Should work
+        path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4')
+        os.makedirs(path)
+
+        # Try paths with a '.' in them
+        self.assertRaises(OSError, os.makedirs, os.curdir)
+        path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', os.curdir)
+        os.makedirs(path)
+        path = os.path.join(base, 'dir1', os.curdir, 'dir2', 'dir3', 'dir4',
+                            'dir5', 'dir6')
+        os.makedirs(path)
+
+
+
+
+    def tearDown(self):
+        path = os.path.join(test_support.TESTFN, 'dir1', 'dir2', 'dir3',
+                            'dir4', 'dir5', 'dir6')
+        # If the tests failed, the bottom-most directory ('../dir6')
+        # may not have been created, so we look for the outermost directory
+        # that exists.
+        while not os.path.exists(path) and path != test_support.TESTFN:
+            path = os.path.dirname(path)
+
+        os.removedirs(path)
+
+class DevNullTests (unittest.TestCase):
+    def test_devnull(self):
+        f = file(os.devnull, 'w')
+        f.write('hello')
+        f.close()
+        f = file(os.devnull, 'r')
+        self.assertEqual(f.read(), '')
+        f.close()
+
+class URandomTests (unittest.TestCase):
+    def test_urandom(self):
+        try:
+            self.assertEqual(len(os.urandom(1)), 1)
+            self.assertEqual(len(os.urandom(10)), 10)
+            self.assertEqual(len(os.urandom(100)), 100)
+            self.assertEqual(len(os.urandom(1000)), 1000)
+            # see http://bugs.python.org/issue3708
+            self.assertRaises(TypeError, os.urandom, 0.9)
+            self.assertRaises(TypeError, os.urandom, 1.1)
+            self.assertRaises(TypeError, os.urandom, 2.0)
+        except NotImplementedError:
+            pass
+
+    def test_execvpe_with_bad_arglist(self):
+        self.assertRaises(ValueError, os.execvpe, 'notepad', [], None)
+
+class Win32ErrorTests(unittest.TestCase):
+    def test_rename(self):
+        self.assertRaises(WindowsError, os.rename, test_support.TESTFN, test_support.TESTFN+".bak")
+
+    def test_remove(self):
+        self.assertRaises(WindowsError, os.remove, test_support.TESTFN)
+
+    def test_chdir(self):
+        self.assertRaises(WindowsError, os.chdir, test_support.TESTFN)
+
+    def test_mkdir(self):
+        f = open(test_support.TESTFN, "w")
+        try:
+            self.assertRaises(WindowsError, os.mkdir, test_support.TESTFN)
+        finally:
+            f.close()
+            os.unlink(test_support.TESTFN)
+
+    def test_utime(self):
+        self.assertRaises(WindowsError, os.utime, test_support.TESTFN, None)
+
+    def test_chmod(self):
+        self.assertRaises(WindowsError, os.chmod, test_support.TESTFN, 0)
+
+class TestInvalidFD(unittest.TestCase):
+    singles = ["fchdir", "fdopen", "dup", "fdatasync", "fstat",
+               "fstatvfs", "fsync", "tcgetpgrp", "ttyname"]
+    #singles.append("close")
+    #We omit close because it doesn'r raise an exception on some platforms
+    def get_single(f):
+        def helper(self):
+            if  hasattr(os, f):
+                self.check(getattr(os, f))
+        return helper
+    for f in singles:
+        locals()["test_"+f] = get_single(f)
+
+    def check(self, f, *args):
+        try:
+            f(test_support.make_bad_fd(), *args)
+        except OSError as e:
+            self.assertEqual(e.errno, errno.EBADF)
+        else:
+            self.fail("%r didn't raise a OSError with a bad file descriptor"
+                      % f)
+
+    def test_isatty(self):
+        if hasattr(os, "isatty"):
+            self.assertEqual(os.isatty(test_support.make_bad_fd()), False)
+
+    def test_closerange(self):
+        if hasattr(os, "closerange"):
+            fd = test_support.make_bad_fd()
+            # Make sure none of the descriptors we are about to close are
+            # currently valid (issue 6542).
+            for i in range(10):
+                try: os.fstat(fd+i)
+                except OSError:
+                    pass
+                else:
+                    break
+            if i < 2:
+                raise unittest.SkipTest(
+                    "Unable to acquire a range of invalid file descriptors")
+            self.assertEqual(os.closerange(fd, fd + i-1), None)
+
+    def test_dup2(self):
+        if hasattr(os, "dup2"):
+            self.check(os.dup2, 20)
+
+    def test_fchmod(self):
+        if hasattr(os, "fchmod"):
+            self.check(os.fchmod, 0)
+
+    def test_fchown(self):
+        if hasattr(os, "fchown"):
+            self.check(os.fchown, -1, -1)
+
+    def test_fpathconf(self):
+        if hasattr(os, "fpathconf"):
+            self.check(os.fpathconf, "PC_NAME_MAX")
+
+    def test_ftruncate(self):
+        if hasattr(os, "ftruncate"):
+            self.check(os.ftruncate, 0)
+
+    def test_lseek(self):
+        if hasattr(os, "lseek"):
+            self.check(os.lseek, 0, 0)
+
+    def test_read(self):
+        if hasattr(os, "read"):
+            self.check(os.read, 1)
+
+    def test_tcsetpgrpt(self):
+        if hasattr(os, "tcsetpgrp"):
+            self.check(os.tcsetpgrp, 0)
+
+    def test_write(self):
+        if hasattr(os, "write"):
+            self.check(os.write, " ")
+
+if sys.platform != 'win32':
+    class Win32ErrorTests(unittest.TestCase):
+        pass
+
+    class PosixUidGidTests(unittest.TestCase):
+        if hasattr(os, 'setuid'):
+            def test_setuid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setuid, 0)
+                self.assertRaises(OverflowError, os.setuid, 1<<32)
+
+        if hasattr(os, 'setgid'):
+            def test_setgid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setgid, 0)
+                self.assertRaises(OverflowError, os.setgid, 1<<32)
+
+        if hasattr(os, 'seteuid'):
+            def test_seteuid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.seteuid, 0)
+                self.assertRaises(OverflowError, os.seteuid, 1<<32)
+
+        if hasattr(os, 'setegid'):
+            def test_setegid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setegid, 0)
+                self.assertRaises(OverflowError, os.setegid, 1<<32)
+
+        if hasattr(os, 'setreuid'):
+            def test_setreuid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setreuid, 0, 0)
+                self.assertRaises(OverflowError, os.setreuid, 1<<32, 0)
+                self.assertRaises(OverflowError, os.setreuid, 0, 1<<32)
+
+            def test_setreuid_neg1(self):
+                # Needs to accept -1.  We run this in a subprocess to avoid
+                # altering the test runner's process state (issue8045).
+                subprocess.check_call([
+                        sys.executable, '-c',
+                        'import os,sys;os.setreuid(-1,-1);sys.exit(0)'])
+
+        if hasattr(os, 'setregid'):
+            def test_setregid(self):
+                if os.getuid() != 0:
+                    self.assertRaises(os.error, os.setregid, 0, 0)
+                self.assertRaises(OverflowError, os.setregid, 1<<32, 0)
+                self.assertRaises(OverflowError, os.setregid, 0, 1<<32)
+
+            def test_setregid_neg1(self):
+                # Needs to accept -1.  We run this in a subprocess to avoid
+                # altering the test runner's process state (issue8045).
+                subprocess.check_call([
+                        sys.executable, '-c',
+                        'import os,sys;os.setregid(-1,-1);sys.exit(0)'])
+else:
+    class PosixUidGidTests(unittest.TestCase):
+        pass
+
+ at unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+class Win32KillTests(unittest.TestCase):
+    def _kill(self, sig):
+        # Start sys.executable as a subprocess and communicate from the
+        # subprocess to the parent that the interpreter is ready. When it
+        # becomes ready, send *sig* via os.kill to the subprocess and check
+        # that the return code is equal to *sig*.
+        import ctypes
+        from ctypes import wintypes
+        import msvcrt
+
+        # Since we can't access the contents of the process' stdout until the
+        # process has exited, use PeekNamedPipe to see what's inside stdout
+        # without waiting. This is done so we can tell that the interpreter
+        # is started and running at a point where it could handle a signal.
+        PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
+        PeekNamedPipe.restype = wintypes.BOOL
+        PeekNamedPipe.argtypes = (wintypes.HANDLE, # Pipe handle
+                                  ctypes.POINTER(ctypes.c_char), # stdout buf
+                                  wintypes.DWORD, # Buffer size
+                                  ctypes.POINTER(wintypes.DWORD), # bytes read
+                                  ctypes.POINTER(wintypes.DWORD), # bytes avail
+                                  ctypes.POINTER(wintypes.DWORD)) # bytes left
+        msg = "running"
+        proc = subprocess.Popen([sys.executable, "-c",
+                                 "import sys;"
+                                 "sys.stdout.write('{}');"
+                                 "sys.stdout.flush();"
+                                 "input()".format(msg)],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                stdin=subprocess.PIPE)
+        self.addCleanup(proc.stdout.close)
+        self.addCleanup(proc.stderr.close)
+        self.addCleanup(proc.stdin.close)
+
+        count, max = 0, 100
+        while count < max and proc.poll() is None:
+            # Create a string buffer to store the result of stdout from the pipe
+            buf = ctypes.create_string_buffer(len(msg))
+            # Obtain the text currently in proc.stdout
+            # Bytes read/avail/left are left as NULL and unused
+            rslt = PeekNamedPipe(msvcrt.get_osfhandle(proc.stdout.fileno()),
+                                 buf, ctypes.sizeof(buf), None, None, None)
+            self.assertNotEqual(rslt, 0, "PeekNamedPipe failed")
+            if buf.value:
+                self.assertEqual(msg, buf.value)
+                break
+            time.sleep(0.1)
+            count += 1
+        else:
+            self.fail("Did not receive communication from the subprocess")
+
+        os.kill(proc.pid, sig)
+        self.assertEqual(proc.wait(), sig)
+
+    def test_kill_sigterm(self):
+        # SIGTERM doesn't mean anything special, but make sure it works
+        self._kill(signal.SIGTERM)
+
+    def test_kill_int(self):
+        # os.kill on Windows can take an int which gets set as the exit code
+        self._kill(100)
+
+    def _kill_with_event(self, event, name):
+        tagname = "test_os_%s" % uuid.uuid1()
+        m = mmap.mmap(-1, 1, tagname)
+        m[0] = '0'
+        # Run a script which has console control handling enabled.
+        proc = subprocess.Popen([sys.executable,
+                   os.path.join(os.path.dirname(__file__),
+                                "win_console_handler.py"), tagname],
+                   creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
+        # Let the interpreter startup before we send signals. See #3137.
+        count, max = 0, 20
+        while count < max and proc.poll() is None:
+            if m[0] == '1':
+                break
+            time.sleep(0.5)
+            count += 1
+        else:
+            self.fail("Subprocess didn't finish initialization")
+        os.kill(proc.pid, event)
+        # proc.send_signal(event) could also be done here.
+        # Allow time for the signal to be passed and the process to exit.
+        time.sleep(0.5)
+        if not proc.poll():
+            # Forcefully kill the process if we weren't able to signal it.
+            os.kill(proc.pid, signal.SIGINT)
+            self.fail("subprocess did not stop on {}".format(name))
+
+    @unittest.skip("subprocesses aren't inheriting CTRL+C property")
+    def test_CTRL_C_EVENT(self):
+        from ctypes import wintypes
+        import ctypes
+
+        # Make a NULL value by creating a pointer with no argument.
+        NULL = ctypes.POINTER(ctypes.c_int)()
+        SetConsoleCtrlHandler = ctypes.windll.kernel32.SetConsoleCtrlHandler
+        SetConsoleCtrlHandler.argtypes = (ctypes.POINTER(ctypes.c_int),
+                                          wintypes.BOOL)
+        SetConsoleCtrlHandler.restype = wintypes.BOOL
+
+        # Calling this with NULL and FALSE causes the calling process to
+        # handle CTRL+C, rather than ignore it. This property is inherited
+        # by subprocesses.
+        SetConsoleCtrlHandler(NULL, 0)
+
+        self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT")
+
+    def test_CTRL_BREAK_EVENT(self):
+        self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT")
+
+
+def test_main():
+    test_support.run_unittest(
+        FileTests,
+        TemporaryFileTests,
+        StatAttributeTests,
+        EnvironTests,
+        WalkTests,
+        MakedirTests,
+        DevNullTests,
+        URandomTests,
+        Win32ErrorTests,
+        TestInvalidFD,
+        PosixUidGidTests,
+        Win32KillTests
+    )
+
+if __name__ == "__main__":
+    test_main()
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -425,10 +425,16 @@
                 raise CompressionError("zlib module is not available")
             self.zlib = zlib
             self.crc = zlib.crc32("") & 0xffffffffL
-            if mode == "r":
-                self._init_read_gz()
-            else:
-                self._init_write_gz()
+            try:
+                if mode == "r":
+                    self._init_read_gz()
+                else:
+                    self._init_write_gz()
+            except:
+                if not self._extfileobj:
+                    fileobj.close()
+                    self.closed = True
+                raise    
 
         if comptype == "bz2":
             try:
@@ -1682,13 +1688,14 @@
 
             if filemode not in "rw":
                 raise ValueError("mode must be 'r' or 'w'")
-
-            t = cls(name, filemode,
-                    _Stream(name, filemode, comptype, fileobj, bufsize),
-                    **kwargs)
-            t._extfileobj = False
-            return t
-
+            fid = _Stream(name, filemode, comptype, fileobj, bufsize)
+            try:
+                t = cls(name, filemode, fid, **kwargs)
+                t._extfileobj = False
+                return t
+            except:
+                fid.close()
+                raise
         elif mode in "aw":
             return cls.taropen(name, mode, fileobj, **kwargs)
 
@@ -1715,13 +1722,18 @@
             gzip.GzipFile
         except (ImportError, AttributeError):
             raise CompressionError("gzip module is not available")
-
+        gz_fid = None
         try:
-            t = cls.taropen(name, mode,
-                gzip.GzipFile(name, mode, compresslevel, fileobj),
-                **kwargs)
+            gz_fid = gzip.GzipFile(name, mode, compresslevel, fileobj)
+            t = cls.taropen(name, mode, gz_fid, **kwargs)
         except IOError:
+            if gz_fid:
+                gz_fid.close()
             raise ReadError("not a gzip file")
+        except:
+            if gz_fid:
+                gz_fid.close()
+            raise 
         t._extfileobj = False
         return t
 
@@ -1738,15 +1750,21 @@
         except ImportError:
             raise CompressionError("bz2 module is not available")
 
-        if fileobj is not None:
-            fileobj = _BZ2Proxy(fileobj, mode)
-        else:
-            fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+        try:
+            if fileobj is not None:
+                bzfileobj = _BZ2Proxy(fileobj, mode)
+            else:
+                bzfileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+            t = cls.taropen(name, mode, bzfileobj, **kwargs)
 
-        try:
-            t = cls.taropen(name, mode, fileobj, **kwargs)
         except (IOError, EOFError):
+            if fileobj is None:
+                bzfileobj.close()
             raise ReadError("not a bzip2 file")
+        except:
+            if fileobj is None:
+                bzfileobj.close()
+            raise 
         t._extfileobj = False
         return t
 
diff --git a/lib-python/modified-2.7/test/test_tarfile.py b/lib-python/modified-2.7/test/test_tarfile.py
--- a/lib-python/modified-2.7/test/test_tarfile.py
+++ b/lib-python/modified-2.7/test/test_tarfile.py
@@ -340,7 +340,8 @@
         # constructor in case of an error. For the test we rely on
         # the fact that opening an empty file raises a ReadError.
         empty = os.path.join(TEMPDIR, "empty")
-        open(empty, "wb").write("")
+        with open(empty, "wb") as fid:
+            fid.write("")
 
         try:
             tar = object.__new__(tarfile.TarFile)
diff --git a/lib-python/2.7/test/test_zipfile.py b/lib-python/modified-2.7/test/test_zipfile.py
copy from lib-python/2.7/test/test_zipfile.py
copy to lib-python/modified-2.7/test/test_zipfile.py
--- a/lib-python/2.7/test/test_zipfile.py
+++ b/lib-python/modified-2.7/test/test_zipfile.py
@@ -234,8 +234,9 @@
 
         # Read the ZIP archive
         with zipfile.ZipFile(f, "r") as zipfp:
-            for line, zipline in zip(self.line_gen, zipfp.open(TESTFN)):
-                self.assertEqual(zipline, line + '\n')
+            with zipfp.open(TESTFN) as f:
+                for line, zipline in zip(self.line_gen, f):
+                    self.assertEqual(zipline, line + '\n')
 
     def test_readline_read_stored(self):
         # Issue #7610: calls to readline() interleaved with calls to read().
@@ -340,7 +341,8 @@
         produces the expected result."""
         with zipfile.ZipFile(TESTFN2, "w") as zipfp:
             zipfp.write(TESTFN)
-            self.assertEqual(zipfp.read(TESTFN), open(TESTFN).read())
+            with open(TESTFN) as f:
+                self.assertEqual(zipfp.read(TESTFN), f.read())
 
     @skipUnless(zlib, "requires zlib")
     def test_per_file_compression(self):
@@ -382,7 +384,8 @@
                 self.assertEqual(writtenfile, correctfile)
 
                 # make sure correct data is in correct file
-                self.assertEqual(fdata, open(writtenfile, "rb").read())
+                with open(writtenfile, "rb") as fid:
+                    self.assertEqual(fdata, fid.read())
                 os.remove(writtenfile)
 
         # remove the test file subdirectories
@@ -401,24 +404,25 @@
                 else:
                     outfile = os.path.join(os.getcwd(), fpath)
 
-                self.assertEqual(fdata, open(outfile, "rb").read())
+                with  open(outfile, "rb") as fid:
+                    self.assertEqual(fdata, fid.read())
                 os.remove(outfile)
 
         # remove the test file subdirectories
         shutil.rmtree(os.path.join(os.getcwd(), 'ziptest2dir'))
 
     def test_writestr_compression(self):
-        zipfp = zipfile.ZipFile(TESTFN2, "w")
-        zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED)
-        if zlib:
-            zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED)
+        with zipfile.ZipFile(TESTFN2, "w") as zipfp:
+            zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED)
+            if zlib:
+                zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED)
 
-        info = zipfp.getinfo('a.txt')
-        self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
+            info = zipfp.getinfo('a.txt')
+            self.assertEqual(info.compress_type, zipfile.ZIP_STORED)
 
-        if zlib:
-            info = zipfp.getinfo('b.txt')
-            self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
+            if zlib:
+                info = zipfp.getinfo('b.txt')
+                self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED)
 
 
     def zip_test_writestr_permissions(self, f, compression):
@@ -646,7 +650,8 @@
 
     def test_write_non_pyfile(self):
         with zipfile.PyZipFile(TemporaryFile(), "w") as zipfp:
-            open(TESTFN, 'w').write('most definitely not a python file')
+            with open(TESTFN, 'w') as f:
+                f.write('most definitely not a python file')
             self.assertRaises(RuntimeError, zipfp.writepy, TESTFN)
             os.remove(TESTFN)
 
@@ -795,7 +800,8 @@
         self.assertRaises(RuntimeError, zipf.open, "foo.txt")
         self.assertRaises(RuntimeError, zipf.testzip)
         self.assertRaises(RuntimeError, zipf.writestr, "bogus.txt", "bogus")
-        open(TESTFN, 'w').write('zipfile test data')
+        with open(TESTFN, 'w') as fp:
+            fp.write('zipfile test data')
         self.assertRaises(RuntimeError, zipf.write, TESTFN)
 
     def test_bad_constructor_mode(self):
@@ -803,7 +809,6 @@
         self.assertRaises(RuntimeError, zipfile.ZipFile, TESTFN, "q")
 
     def test_bad_open_mode(self):
-        """Check that bad modes passed to ZipFile.open are caught."""
         with zipfile.ZipFile(TESTFN, mode="w") as zipf:
             zipf.writestr("foo.txt", "O, for a Muse of Fire!")
 
@@ -851,7 +856,6 @@
 
     def test_comments(self):
         """Check that comments on the archive are handled properly."""
-
         # check default comment is empty
         with zipfile.ZipFile(TESTFN, mode="w") as zipf:
             self.assertEqual(zipf.comment, '')
@@ -953,14 +957,16 @@
         with zipfile.ZipFile(TESTFN, mode="w") as zipf:
             pass
         try:
-            zipf = zipfile.ZipFile(TESTFN, mode="r")
+            with zipfile.ZipFile(TESTFN, mode="r") as zipf:
+                pass
         except zipfile.BadZipfile:
             self.fail("Unable to create empty ZIP file in 'w' mode")
 
         with zipfile.ZipFile(TESTFN, mode="a") as zipf:
             pass
         try:
-            zipf = zipfile.ZipFile(TESTFN, mode="r")
+            with zipfile.ZipFile(TESTFN, mode="r") as zipf:
+                pass
         except:
             self.fail("Unable to create empty ZIP file in 'a' mode")
 
@@ -1160,6 +1166,8 @@
             data1 += zopen1.read(500)
             data2 += zopen2.read(500)
             self.assertEqual(data1, data2)
+            zopen1.close()
+            zopen2.close()
 
     def test_different_file(self):
         # Verify that (when the ZipFile is in control of creating file objects)
@@ -1207,9 +1215,9 @@
 
     def test_store_dir(self):
         os.mkdir(os.path.join(TESTFN2, "x"))
-        zipf = zipfile.ZipFile(TESTFN, "w")
-        zipf.write(os.path.join(TESTFN2, "x"), "x")
-        self.assertTrue(zipf.filelist[0].filename.endswith("x/"))
+        with zipfile.ZipFile(TESTFN, "w") as zipf:
+            zipf.write(os.path.join(TESTFN2, "x"), "x")
+            self.assertTrue(zipf.filelist[0].filename.endswith("x/"))
 
     def tearDown(self):
         shutil.rmtree(TESTFN2)
@@ -1226,7 +1234,8 @@
         for n, s in enumerate(self.seps):
             self.arcdata[s] = s.join(self.line_gen) + s
             self.arcfiles[s] = '%s-%d' % (TESTFN, n)
-            open(self.arcfiles[s], "wb").write(self.arcdata[s])
+            with open(self.arcfiles[s], "wb") as f:
+                f.write(self.arcdata[s])
 
     def make_test_archive(self, f, compression):
         # Create the ZIP archive
@@ -1295,8 +1304,9 @@
         # Read the ZIP archive
         with zipfile.ZipFile(f, "r") as zipfp:
             for sep, fn in self.arcfiles.items():
-                for line, zipline in zip(self.line_gen, zipfp.open(fn, "rU")):
-                    self.assertEqual(zipline, line + '\n')
+                with zipfp.open(fn, "rU") as f:
+                    for line, zipline in zip(self.line_gen, f):
+                        self.assertEqual(zipline, line + '\n')
 
     def test_read_stored(self):
         for f in (TESTFN2, TemporaryFile(), StringIO()):
diff --git a/lib-python/2.7/zipfile.py b/lib-python/modified-2.7/zipfile.py
copy from lib-python/2.7/zipfile.py
copy to lib-python/modified-2.7/zipfile.py
--- a/lib-python/2.7/zipfile.py
+++ b/lib-python/modified-2.7/zipfile.py
@@ -648,6 +648,10 @@
         return data
 
 
+class ZipExtFileWithClose(ZipExtFile):
+    def close(self):
+        self._fileobj.close()
+
 
 class ZipFile:
     """ Class with methods to open, read, write, close, list zip files.
@@ -843,9 +847,9 @@
             try:
                 # Read by chunks, to avoid an OverflowError or a
                 # MemoryError with very large embedded files.
-                f = self.open(zinfo.filename, "r")
-                while f.read(chunk_size):     # Check CRC-32
-                    pass
+                with self.open(zinfo.filename, "r") as f:
+                    while f.read(chunk_size):     # Check CRC-32
+                        pass
             except BadZipfile:
                 return zinfo.filename
 
@@ -864,7 +868,9 @@
 
     def read(self, name, pwd=None):
         """Return file bytes (as a string) for name."""
-        return self.open(name, "r", pwd).read()
+        with self.open(name, "r", pwd) as f:
+            retval = f.read()
+            return retval
 
     def open(self, name, mode="r", pwd=None):
         """Return file-like object for 'name'."""
@@ -881,59 +887,66 @@
         else:
             zef_file = open(self.filename, 'rb')
 
-        # Make sure we have an info object
-        if isinstance(name, ZipInfo):
-            # 'name' is already an info object
-            zinfo = name
+        try:
+            # Make sure we have an info object
+            if isinstance(name, ZipInfo):
+                # 'name' is already an info object
+                zinfo = name
+            else:
+                # Get info object for name
+                zinfo = self.getinfo(name)
+
+            zef_file.seek(zinfo.header_offset, 0)
+
+            # Skip the file header:
+            fheader = zef_file.read(sizeFileHeader)
+            if fheader[0:4] != stringFileHeader:
+                raise BadZipfile, "Bad magic number for file header"
+
+            fheader = struct.unpack(structFileHeader, fheader)
+            fname = zef_file.read(fheader[_FH_FILENAME_LENGTH])
+            if fheader[_FH_EXTRA_FIELD_LENGTH]:
+                zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
+
+            if fname != zinfo.orig_filename:
+                raise BadZipfile, \
+                          'File name in directory "%s" and header "%s" differ.' % (
+                              zinfo.orig_filename, fname)
+
+            # check for encrypted flag & handle password
+            is_encrypted = zinfo.flag_bits & 0x1
+            zd = None
+            if is_encrypted:
+                if not pwd:
+                    pwd = self.pwd
+                if not pwd:
+                    raise RuntimeError, "File %s is encrypted, " \
+                          "password required for extraction" % name
+
+                zd = _ZipDecrypter(pwd)
+                # The first 12 bytes in the cypher stream is an encryption header
+                #  used to strengthen the algorithm. The first 11 bytes are
+                #  completely random, while the 12th contains the MSB of the CRC,
+                #  or the MSB of the file time depending on the header type
+                #  and is used to check the correctness of the password.
+                bytes = zef_file.read(12)
+                h = map(zd, bytes[0:12])
+                if zinfo.flag_bits & 0x8:
+                    # compare against the file type from extended local headers
+                    check_byte = (zinfo._raw_time >> 8) & 0xff
+                else:
+                    # compare against the CRC otherwise
+                    check_byte = (zinfo.CRC >> 24) & 0xff
+                if ord(h[11]) != check_byte:
+                    raise RuntimeError("Bad password for file", name)
+        except:
+            if not self._filePassed:
+                zef_file.close()
+            raise    
+        if self._filePassed:
+            return  ZipExtFile(zef_file, mode, zinfo, zd)
         else:
-            # Get info object for name
-            zinfo = self.getinfo(name)
-
-        zef_file.seek(zinfo.header_offset, 0)
-
-        # Skip the file header:
-        fheader = zef_file.read(sizeFileHeader)
-        if fheader[0:4] != stringFileHeader:
-            raise BadZipfile, "Bad magic number for file header"
-
-        fheader = struct.unpack(structFileHeader, fheader)
-        fname = zef_file.read(fheader[_FH_FILENAME_LENGTH])
-        if fheader[_FH_EXTRA_FIELD_LENGTH]:
-            zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH])
-
-        if fname != zinfo.orig_filename:
-            raise BadZipfile, \
-                      'File name in directory "%s" and header "%s" differ.' % (
-                          zinfo.orig_filename, fname)
-
-        # check for encrypted flag & handle password
-        is_encrypted = zinfo.flag_bits & 0x1
-        zd = None
-        if is_encrypted:
-            if not pwd:
-                pwd = self.pwd
-            if not pwd:
-                raise RuntimeError, "File %s is encrypted, " \
-                      "password required for extraction" % name
-
-            zd = _ZipDecrypter(pwd)
-            # The first 12 bytes in the cypher stream is an encryption header
-            #  used to strengthen the algorithm. The first 11 bytes are
-            #  completely random, while the 12th contains the MSB of the CRC,
-            #  or the MSB of the file time depending on the header type
-            #  and is used to check the correctness of the password.
-            bytes = zef_file.read(12)
-            h = map(zd, bytes[0:12])
-            if zinfo.flag_bits & 0x8:
-                # compare against the file type from extended local headers
-                check_byte = (zinfo._raw_time >> 8) & 0xff
-            else:
-                # compare against the CRC otherwise
-                check_byte = (zinfo.CRC >> 24) & 0xff
-            if ord(h[11]) != check_byte:
-                raise RuntimeError("Bad password for file", name)
-
-        return  ZipExtFile(zef_file, mode, zinfo, zd)
+            return  ZipExtFileWithClose(zef_file, mode, zinfo, zd)
 
     def extract(self, member, path=None, pwd=None):
         """Extract a member from the archive to the current working directory,
@@ -989,7 +1002,6 @@
             if not os.path.isdir(targetpath):
                 os.mkdir(targetpath)
             return targetpath
-
         source = self.open(member, pwd=pwd)
         target = file(targetpath, "wb")
         shutil.copyfileobj(source, target)
diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
 try:
     from thread import _local as local
 except ImportError:
-    local = object    # no threads
+    class local(object):    # no threads
+        pass
 
 class ConvMode:
     encoding = 'ascii'
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/cppyy.rst
@@ -0,0 +1,333 @@
+============================
+cppyy: C++ bindings for PyPy
+============================
+
+The cppyy module provides C++ bindings for PyPy by using the reflection
+information extracted from C++ header files by means of the
+`Reflex package`_.
+For this to work, you have to both install Reflex and build PyPy from the
+reflex-support branch.
+As indicated by this being a branch, support for Reflex is still
+experimental.
+However, it is functional enough to put it in the hands of those who want
+to give it a try.
+In the medium term, cppyy will move away from Reflex and instead use
+`cling`_ as its backend, which is based on `llvm`_.
+Although that will change the logistics on the generation of reflection
+information, it will not change the python-side interface.
+
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`cling`: http://root.cern.ch/drupal/content/cling
+.. _`llvm`: http://llvm.org/
+
+
+Installation
+============
+
+For now, the easiest way of getting the latest version of Reflex, is by
+installing the ROOT package.
+Besides getting the latest version of Reflex, another advantage is that with
+the full ROOT package, you can also use your Reflex-bound code on `CPython`_.
+`Download`_ a binary or install from `source`_.
+Some Linux and Mac systems may have ROOT provided in the list of scientific
+software of their packager.
+A current, standalone version of Reflex should be provided at some point,
+once the dependencies and general packaging have been thought out.
+Also, make sure you have a version of `gccxml`_ installed, which is most
+easily provided by the packager of your system.
+If you read up on gccxml, you'll probably notice that it is no longer being
+developed and hence will not provide C++11 support.
+That's why the medium term plan is to move to `cling`_.
+
+.. _`Download`: http://root.cern.ch/drupal/content/downloading-root
+.. _`source`: http://root.cern.ch/drupal/content/installing-root-source
+.. _`gccxml`: http://www.gccxml.org
+
+Next, get the `PyPy sources`_, select the reflex-support branch, and build
+pypy-c.
+For the build to succeed, the ``$ROOTSYS`` environment variable must point to
+the location of your ROOT installation::
+
+    $ hg clone https://bitbucket.org/pypy/pypy
+    $ cd pypy
+    $ hg up reflex-support
+    $ cd pypy/translator/goal
+    $ python translate.py -O jit --gcrootfinder=shadowstack targetpypystandalone.py --withmod-cppyy
+
+This will build a ``pypy-c`` that includes the cppyy module, and through that,
+Reflex support.
+Of course, if you already have a pre-built version of the ``pypy`` interpreter,
+you can use that for the translation rather than ``python``.
+
+.. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
+
+
+Basic example
+=============
+
+Now test with a trivial example whether all packages are properly installed
+and functional.
+First, create a C++ header file with some class in it (note that all functions
+are made inline for convenience; a real-world example would of course have a
+corresponding source file)::
+
+    $ cat MyClass.h
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_myint(i) {}
+
+        int GetMyInt() { return m_myint; }
+        void SetMyInt(int i) { m_myint = i; }
+
+    public:
+       int m_myint;
+    };
+
+Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
+code::
+
+    $ genreflex MyClass.h
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+
+Now you're ready to use the bindings.
+Since the bindings are designed to look pythonistic, it should be
+straightforward::
+
+    $ pypy-c
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libMyClassDict.so")
+    <CPPLibrary object at 0xb6fd7c4c>
+    >>>> myinst = cppyy.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> myinst.SetMyInt(33)
+    >>>> print myinst.m_myint
+    33
+    >>>> myinst.m_myint = 77
+    >>>> print myinst.GetMyInt()
+    77
+    >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
+
+That's all there is to it!
+
+
+Features
+========
+
+The following is not meant to be an exhaustive list, since cppyy is still
+under active development.
+Furthermore, the intention is that every feature is as natural as possible on
+the python side, so if you find something missing in the list below, simply
+try it out.
+It is not always possible to provide exact mapping between python and C++
+(active memory management is one such case), but by and large, if the use of a
+feature does not strike you as obvious, it is more likely to simply be a bug.
+That is a strong statement to make, but also a worthy goal.
+
+* **abstract classes**: Are represented as python classes, since they are
+  needed to complete the inheritance hierarchies, but will raise an exception
+  if an attempt is made to instantiate from them.
+
+* **arrays**: Supported for builtin data types only, as used from module
+  ``array``.
+  Out-of-bounds checking is limited to those cases where the size is known at
+  compile time (and hence part of the reflection info).
+
+* **builtin data types**: Map onto the expected equivalent python types, with
+  the caveat that there may be size differences, and thus it is possible that
+  exceptions are raised if an overflow is detected.
+
+* **casting**: Is supposed to be unnecessary.
+  Object pointer returns from functions provide the most derived class known
+  in the hierarchy of the object being returned.
+  This is important to preserve object identity as well as to make casting,
+  a pure C++ feature after all, superfluous.
+
+* **classes and structs**: Get mapped onto python classes, where they can be
+  instantiated as expected.
+  If classes are inner classes or live in a namespace, their naming and
+  location will reflect that.
+
+* **data members**: Public data members are represented as python properties
+  and provide read and write access on instances as expected.
+
+* **default arguments**: C++ default arguments work as expected, but python
+  keywords are not supported.
+  It is technically possible to support keywords, but for the C++ interface,
+  the formal argument names have no meaning and are not considered part of the
+  API, hence it is not a good idea to use keywords.
+
+* **doc strings**: The doc string of a method or function contains the C++
+  arguments and return types of all overloads of that name, as applicable.
+
+* **functions**: Work as expected and live in their appropriate namespace
+  (which can be the global one, ``cppyy.gbl``).
+
+* **inheritance**: All combinations of inheritance on the C++ (single,
+  multiple, virtual) are supported in the binding.
+  However, new python classes can only use single inheritance from a bound C++
+  class.
+  Multiple inheritance would introduce two "this" pointers in the binding.
+  This is a current, not a fundamental, limitation.
+  The C++ side will not see any overridden methods on the python side, as
+  cross-inheritance is planned but not yet supported.
+
+* **methods**: Are represented as python methods and work as expected.
+  They are first class objects and can be bound to an instance.
+  Virtual C++ methods work as expected.
+  To select a specific virtual method, do like with normal python classes
+  that override methods: select it from the class that you need, rather than
+  calling the method on the instance.
+
+* **namespaces**: Are represented as python classes.
+  Namespaces are more open-ended than classes, so sometimes initial access may
+  result in updates as data and functions are looked up and constructed
+  lazily.
+  Thus the result of ``dir()`` on a namespace should not be relied upon: it
+  only shows the already accessed members. (TODO: to be fixed by implementing
+  __dir__.)
+  The global namespace is ``cppyy.gbl``.
+
+* **operator conversions**: If defined in the C++ class and a python
+  equivalent exists (i.e. all builtin integer and floating point types, as well
+  as ``bool``), it will map onto that python conversion.
+  Note that ``char*`` is mapped onto ``__str__``.
+
+* **operator overloads**: If defined in the C++ class and if a python
+  equivalent is available (not always the case, think e.g. of ``operator||``),
+  then they work as expected.
+  Special care needs to be taken for global operator overloads in C++: first,
+  make sure that they are actually reflected, especially for the global
+  overloads for ``operator==`` and ``operator!=`` of STL iterators in the case
+  of gcc.
+  Second, make sure that reflection info is loaded in the proper order.
+  I.e. that these global overloads are available before use.
+
+* **pointers**: For builtin data types, see arrays.
+  For objects, a pointer to an object and an object looks the same, unless
+  the pointer is a data member.
+  In that case, assigning to the data member will cause a copy of the pointer
+  and care should be taken about the object's life time.
+  If a pointer is a global variable, the C++ side can replace the underlying
+  object and the python side will immediately reflect that.
+
+* **static data members**: Are represented as python property objects on the
+  class and the meta-class.
+  Both read and write access is as expected.
+
+* **static methods**: Are represented as python's ``staticmethod`` objects
+  and can be called both from the class as well as from instances.
+
+* **strings**: The std::string class is considered a builtin C++ type and
+  mixes quite well with python's str.
+  Python's str can be passed where a ``const char*`` is expected, and an str
+  will be returned if the return type is ``const char*``.
+
+* **templated classes**: Are represented in a meta-class style in python.
+  This looks a little bit confusing, but conceptually is rather natural.
+  For example, given the class ``std::vector<int>``, the meta-class part would
+  be ``std.vector`` in python.
+  Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
+  create an instance of that class, do ``std.vector(int)()``.
+  Note that templates can be build up by handing actual types to the class
+  instantiation (as done in this vector example), or by passing in the list of
+  template arguments as a string.
+  The former is a lot easier to work with if you have template instantiations
+  using classes that themselves are templates (etc.) in the arguments.
+  All classes must already exist in the loaded reflection info.
+
+* **unary operators**: Are supported if a python equivalent exists, and if the
+  operator is defined in the C++ class.
+
+You can always find more detailed examples and see the full of supported
+features by looking at the tests in pypy/module/cppyy/test.
+
+If a feature or reflection info is missing, this is supposed to be handled
+gracefully.
+In fact, there are unit tests explicitly for this purpose (even as their use
+becomes less interesting over time, as the number of missing features
+decreases).
+Only when a missing feature is used, should there be an exception.
+For example, if no reflection info is available for a return type, then a
+class that has a method with that return type can still be used.
+Only that one specific method can not be used.
+
+
+The fast lane
+=============
+
+The following is an experimental feature of cppyy, and that makes it doubly
+experimental, so caveat emptor.
+With a slight modification of Reflex, it can provide function pointers for
+C++ methods, and hence allow PyPy to call those pointers directly, rather than
+calling C++ through a Reflex stub.
+This results in a rather significant speed-up.
+Mind you, the normal stub path is not exactly slow, so for now only use this
+out of curiosity or if you really need it.
+
+To install this patch of Reflex, locate the file genreflex-methptrgetter.patch
+in pypy/module/cppyy and apply it to the genreflex python scripts found in
+``$ROOTSYS/lib``::
+
+    $ cd $ROOTSYS/lib
+    $ patch -p2 < genreflex-methptrgetter.patch
+
+With this patch, ``genreflex`` will have grown the ``--with-methptrgetter``
+option.
+Use this option when running ``genreflex``, and add the
+``-Wno-pmf-conversions`` option to ``g++`` when compiling.
+The rest works the same way: the fast path will be used transparently (which
+also means that you can't actually find out whether it is in use, other than
+by running a micro-benchmark).
+
+
+CPython
+=======
+
+Most of the ideas in cppyy come originally from the `PyROOT`_ project.
+Although PyROOT does not support Reflex directly, it has an alter ego called
+"PyCintex" that, in a somewhat roundabout way, does.
+If you installed ROOT, rather than just Reflex, PyCintex should be available
+immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment
+variable.
+
+.. _`PyROOT`: http://root.cern.ch/drupal/content/pyroot
+
+There are a couple of minor differences between PyCintex and cppyy, most to do
+with naming.
+The one that you will run into directly, is that PyCintex uses a function
+called ``loadDictionary`` rather than ``load_reflection_info``.
+The reason for this is that Reflex calls the shared libraries that contain
+reflection info "dictionaries."
+However, in python, the name `dictionary` already has a well-defined meaning,
+so a more descriptive name was chosen for cppyy.
+In addition, PyCintex requires that the names of shared libraries so loaded
+start with "lib" in their name.
+The basic example above, rewritten for PyCintex thus goes like this::
+
+    $ python
+    >>> import PyCintex
+    >>> PyCintex.loadDictionary("libMyClassDict.so")
+    >>> myinst = PyCintex.gbl.MyClass(42)
+    >>> print myinst.GetMyInt()
+    42
+    >>> myinst.SetMyInt(33)
+    >>> print myinst.m_myint
+    33
+    >>> myinst.m_myint = 77
+    >>> print myinst.GetMyInt()
+    77
+    >>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
+
+Other naming differences are such things as taking an address of an object.
+In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
+made to follow the naming as in ``ctypes`` and hence use ``addressof``
+(PyROOT/PyCintex predate ``ctypes`` by several years, and the ROOT project
+follows camel-case, hence the differences).
+
+Of course, this is python, so if any of the naming is not to your liking, all
+you have to do is provide a wrapper script that you import instead of
+importing the ``cppyy`` or ``PyCintex`` modules directly.
+In that wrapper script you can rename methods exactly the way you need it.
+
+In the cling world, all these differences will be resolved.
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -23,6 +23,8 @@
 
 * Write them in RPython as mixedmodule_, using *rffi* as bindings.
 
+* Write them in C++ and bind them through Reflex_ (EXPERIMENTAL)
+
 .. _ctypes: #CTypes
 .. _\_ffi: #LibFFI
 .. _mixedmodule: #Mixed Modules
@@ -110,3 +112,34 @@
 
 XXX we should provide detailed docs about lltype and rffi, especially if we
     want people to follow that way.
+
+Reflex
+======
+
+This method is only experimental for now, and is being exercised on a branch,
+`reflex-support`_, so you will have to build PyPy yourself.
+The method works by using the `Reflex package`_ to provide reflection
+information of the C++ code, which is then used to automatically generate
+bindings at runtime, which can then be used from python.
+Full details are `available here`_.
+
+.. _`reflex-support`: cppyy.html
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`available here`: cppyy.html
+
+Pros
+----
+
+If it works, it is mostly automatic, and hence easy in use.
+The bindings can make use of direct pointers, in which case the calls are
+very fast.
+
+Cons
+----
+
+C++ is a large language, and these bindings are not yet feature-complete.
+Although missing features should do no harm if you don't use them, if you do
+need a particular feature, it may be necessary to work around it in python
+or with a C++ helper function.
+Although Reflex works on various platforms, the bindings with PyPy have only
+been tested on Linux.
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
     space.call_method(w_list, "reverse")
     return 0
 
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+    """Return a list of the objects in list containing the objects between low
+    and high.  Return NULL and set an exception if unsuccessful.  Analogous
+    to list[low:high].  Negative indices, as when slicing from Python, are not
+    supported."""
+    w_start = space.wrap(low)
+    w_stop = space.wrap(high)
+    return space.getslice(w_list, w_start, w_stop)
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyList_SetSlice(space, w_list, low, high, w_sequence):
     """Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
     This is the equivalent of the Python expression hash(o)."""
     return space.int_w(space.hash(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+    """Set a TypeError indicating that type(o) is not hashable and return -1.
+    This function receives special treatment when stored in a tp_hash slot,
+    allowing a type to explicitly indicate to the interpreter that it is not
+    hashable.
+    """
+    raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Dir(space, w_o):
     """This is equivalent to the Python expression dir(o), returning a (possibly
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
-    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc)
@@ -60,6 +60,15 @@
     args_w = space.fixedview(w_args)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
 
+def wrap_inquirypred(space, w_self, w_args, func):
+    func_inquiry = rffi.cast(inquiry, func)
+    check_num_args(space, w_args, 0)
+    args_w = space.fixedview(w_args)
+    res = generic_cpy_call(space, func_inquiry, w_self)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
+    return space.wrap(bool(res))
+
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
     check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
         w_errors = space.wrap(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+    """Decode a string object by passing it to the codec registered
+    for encoding and return the result as Python object. encoding and
+    errors have the same meaning as the parameters of the same name in
+    the string encode() method.  The codec to be used is looked up
+    using the Python codec registry. Return NULL if an exception was
+    raised by the codec.
+
+    This function is not available in 3.x and does not have a PyBytes alias."""
+    if not PyString_Check(space, w_str):
+        PyErr_BadArgument(space)
+
+    w_encoding = w_errors = space.w_None
+    if encoding:
+        w_encoding = space.wrap(rffi.charp2str(encoding))
+    if errors:
+        w_errors = space.wrap(rffi.charp2str(errors))
+    return space.call_method(w_str, "decode", w_encoding, w_errors)
+
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
-    """Return a list of the objects in list containing the objects between low
-    and high.  Return NULL and set an exception if unsuccessful.  Analogous
-    to list[low:high].  Negative indices, as when slicing from Python, are not
-    supported.
-
-    This function used an int for low and high. This might
-    require changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([Py_ssize_t], PyObject)
 def PyLong_FromSsize_t(space, v):
     """Return a new PyLongObject object from a C Py_ssize_t, or
@@ -1606,15 +1595,6 @@
     for PyObject_Str()."""
     raise NotImplementedError
 
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
-    """Set a TypeError indicating that type(o) is not hashable and return -1.
-    This function receives special treatment when stored in a tp_hash slot,
-    allowing a type to explicitly indicate to the interpreter that it is not
-    hashable.
-    """
-    raise NotImplementedError
-
 @cpython_api([], PyFrameObject)
 def PyEval_GetFrame(space):
     """Return the current thread state's frame, which is NULL if no frame is
@@ -1737,17 +1717,6 @@
     changes in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
-    """Decode a string object by passing it to the codec registered for encoding and
-    return the result as Python object. encoding and errors have the same
-    meaning as the parameters of the same name in the string encode() method.
-    The codec to be used is looked up using the Python codec registry. Return NULL
-    if an exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Encode(space, s, size, encoding, errors):
     """Encode the char buffer of the given size by passing it to the codec
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
         w_t = api.PyList_AsTuple(w_l)
         assert space.unwrap(w_t) == (3, 2, 1)
 
+    def test_list_getslice(self, space, api):
+        w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+        w_s = api.PyList_GetSlice(w_l, 1, 5)
+        assert space.unwrap(w_s) == [2, 1]
+
 class AppTestListObject(AppTestCpythonExtensionBase):
     def test_listobject(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
+    def test_AsDecodedObject(self, space, api):
+        w_str = space.wrap('caf\xe9')
+        encoding = rffi.str2charp("latin-1")
+        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        rffi.free_charp(encoding)
+        assert space.unwrap(w_res) == u"caf\xe9"
+
     def test_eq(self, space, api):
         assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
         assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
+        
+    def test_bool(self):
+        module = self.import_extension('foo', [
+            ("newInt", "METH_VARARGS",
+             """
+                IntLikeObject *intObj;
+                long intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "i", &intval))
+                    return NULL;
+
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+                intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+                if (!intObj) {
+                    return NULL;
+                }
+
+                intObj->value = intval;
+                return (PyObject *)intObj;
+             """)],
+            """
+            typedef struct
+            {
+                PyObject_HEAD
+                int value;
+            } IntLikeObject;
+
+            static int
+            intlike_nb_nonzero(IntLikeObject *v)
+            {
+                if (v->value == -42) {
+                    PyErr_SetNone(PyExc_ValueError);
+                    return -1;
+                }
+                return v->value;
+            }
+
+            PyTypeObject IntLike_Type = {
+                PyObject_HEAD_INIT(0)
+                /*ob_size*/             0,
+                /*tp_name*/             "IntLike",
+                /*tp_basicsize*/        sizeof(IntLikeObject),
+            };
+            static PyNumberMethods intlike_as_number;
+            """)
+        assert not bool(module.newInt(0))
+        assert bool(module.newInt(1))
+        assert bool(module.newInt(-1))
+        raises(ValueError, bool, module.newInt(-42))
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
@@ -30,6 +31,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -10,6 +10,7 @@
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
      scalar_w, W_NDimArray, array)
+from pypy.module.micronumpy.interp_arrayops import where
 from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize, instantiate
 
@@ -35,6 +36,7 @@
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring"]
 TWO_ARG_FUNCTIONS = ["dot", 'take']
+THREE_ARG_FUNCTIONS = ['where']
 
 class FakeSpace(object):
     w_ValueError = None
@@ -445,14 +447,25 @@
             arg = self.args[1].execute(interp)
             if not isinstance(arg, BaseArray):
                 raise ArgumentNotAnArray
-            if not isinstance(arg, BaseArray):
-                raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
             else:
                 assert False # unreachable code
+        elif self.name in THREE_ARG_FUNCTIONS:
+            if len(self.args) != 3:
+                raise ArgumentMismatch
+            arg1 = self.args[1].execute(interp)
+            arg2 = self.args[2].execute(interp)
+            if not isinstance(arg1, BaseArray):
+                raise ArgumentNotAnArray
+            if not isinstance(arg2, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "where":
+                w_res = where(interp.space, arr, arg1, arg2)
+            else:
+                assert False
         else:
             raise WrongFunctionName
         if isinstance(w_res, BaseArray):
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, space, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -1583,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,16 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -270,3 +270,13 @@
         b -> 2
         """)
         assert interp.results[0].value == 3
+
+    def test_where(self):
+        interp = self.run('''
+        a = [1, 0, 3, 0]
+        b = [1, 1, 1, 1]
+        c = [0, 0, 0, 0]
+        d = where(a, b, c)
+        d -> 1
+        ''')
+        assert interp.results[0].value == 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
@@ -1831,6 +1830,12 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
+    def test_array_indexing_bool_setitem_multidim(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,6 @@
 
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_ufunc_instance(self):
         from _numpypy import add, ufunc
@@ -148,12 +147,13 @@
         assert math.isnan(fmax(nan, 0))
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
-        # The numpy docs specify that 
-        # the FIRST NaN should be used if both are NaN
-        # use copysign on both sides to sidestep bug in nan representaion 
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        # Since comparisons with nnan and nan all return false,
+        # use copysign on both sides to sidestep bug in nan representaion
         # on Microsoft win32
         assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
 
+
     def test_fmin(self):
         from _numpypy import fmin
         import math
@@ -166,9 +166,9 @@
         assert (fmin(a, [1]*5) == [ninf, -5, 0, 1, 1]).all()
         assert math.isnan(fmin(nan, 0))
         assert math.isnan(fmin(0, nan))
-        # The numpy docs specify that 
-        # the FIRST NaN should be used if both are NaN
-        # use copysign on both sides to sidestep bug in nan representaion 
+        assert math.isnan(fmin(nan, nan))
+        # The numpy docs specify that the FIRST NaN should be used if both are NaN
+        # use copysign on both sides to sidestep bug in nan representaion
         # on Microsoft win32
         assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -40,7 +40,7 @@
 # In that case, do 5 bits at a time.  The potential drawback is that
 # a table of 2**5 intermediate results is computed.
 
-## FIVEARY_CUTOFF = 8   disabled for now
+FIVEARY_CUTOFF = 8
 
 
 def _mask_digit(x):
@@ -456,7 +456,7 @@
 
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
-        if 1:   ## b.numdigits() <= FIVEARY_CUTOFF:
+        if b.numdigits() <= FIVEARY_CUTOFF:
             # Left-to-right binary exponentiation (HAC Algorithm 14.79)
             # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
             i = b.numdigits() - 1
@@ -469,30 +469,51 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 i -= 1
-##        else:
-##            This code is disabled for now, because it assumes that
-##            SHIFT is a multiple of 5.  It could be fixed but it looks
-##            like it's more troubles than benefits...
-##
-##            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-##            # This is only useful in the case where c != None.
-##            # z still holds 1L
-##            table = [z] * 32
-##            table[0] = z
-##            for i in range(1, 32):
-##                table[i] = _help_mult(table[i-1], a, c)
-##            i = b.numdigits() - 1
-##            while i >= 0:
-##                bi = b.digit(i)
-##                j = SHIFT - 5
-##                while j >= 0:
-##                    index = (bi >> j) & 0x1f
-##                    for k in range(5):
-##                        z = _help_mult(z, z, c)
-##                    if index:
-##                        z = _help_mult(z, table[index], c)
-##                    j -= 5
-##                i -= 1
+        else:
+            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
+            # This is only useful in the case where c != None.
+            # z still holds 1L
+            table = [z] * 32
+            table[0] = z
+            for i in range(1, 32):
+                table[i] = _help_mult(table[i-1], a, c)
+            i = b.numdigits()
+            # Note that here SHIFT is not a multiple of 5.  The difficulty
+            # is to extract 5 bits at a time from 'b', starting from the
+            # most significant digits, so that at the end of the algorithm
+            # it falls exactly to zero.
+            # m  = max number of bits = i * SHIFT
+            # m+ = m rounded up to the next multiple of 5
+            # j  = (m+) % SHIFT = (m+) - (i * SHIFT)
+            # (computed without doing "i * SHIFT", which might overflow)
+            j = i % 5
+            if j != 0:
+                j = 5 - j
+            if not we_are_translated():
+                assert j == (i*SHIFT+4)//5*5 - i*SHIFT
+            #
+            accum = r_uint(0)
+            while True:
+                j -= 5
+                if j >= 0:
+                    index = (accum >> j) & 0x1f
+                else:
+                    # 'accum' does not have enough digit.
+                    # must get the next digit from 'b' in order to complete
+                    i -= 1
+                    if i < 0:
+                        break    # done
+                    bi = b.udigit(i)
+                    index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f
+                    accum = bi
+                    j += SHIFT
+                #
+                for k in range(5):
+                    z = _help_mult(z, z, c)
+                if index:
+                    z = _help_mult(z, table[index], c)
+            #
+            assert j == -5
 
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint.py
@@ -379,6 +379,18 @@
         for n, expected in [(37, 9), (1291, 931), (67889, 39464)]:
             v = two.pow(t, rbigint.fromint(n))
             assert v.toint() == expected
+        #
+        # more tests, comparing against CPython's answer
+        enabled = sample(range(5*32), 10)
+        for i in range(5*32):
+            t = t.mul(two)      # add one random bit
+            if random() >= 0.5:
+                t = t.add(rbigint.fromint(1))
+            if i not in enabled:
+                continue    # don't take forever
+            n = randint(1, sys.maxint)
+            v = two.pow(t, rbigint.fromint(n))
+            assert v.toint() == pow(2, t.tolong(), n)
 
     def test_pow_lln(self):
         x = 10L
diff --git a/pypy/translator/c/src/exception.h b/pypy/translator/c/src/exception.h
--- a/pypy/translator/c/src/exception.h
+++ b/pypy/translator/c/src/exception.h
@@ -43,6 +43,16 @@
           filename, lineno, functionname);
 }
 #endif
+#else   /* !DO_LOG_EXC: define the function anyway, so that we can shut
+           off the prints of a debug_exc by remaking only testing_1.o */
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname);
+#ifndef PYPY_NOT_MAIN_FILE
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname)
+{
+}
+#endif
 #endif  /* DO_LOG_EXC */
 
 /* Hint: functions and macros not defined here, like RPyRaiseException,

From noreply at buildbot.pypy.org  Sun Apr 22 23:00:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 23:00:00 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Ensure NEXT is cleared by C code after
	all.
Message-ID: <20120422210000.6598A8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54629:c343346db868
Date: 2012-04-22 20:37 +0200
http://bitbucket.org/pypy/pypy/changeset/c343346db868/

Log:	Ensure NEXT is cleared by C code after all. Adapt two more tests.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -4,6 +4,7 @@
 from pypy.rpython.annlowlevel import base_ptr_lltype, cast_base_ptr_to_instance
 from pypy.rlib.objectmodel import keepalive_until_here, we_are_translated
 from pypy.rlib.debug import ll_assert
+from pypy.rlib.nonconst import NonConstant
 from pypy.translator.stm.stmgcintf import StmOperations
 
 
@@ -171,7 +172,15 @@
     # This logic is in a sub-function because we want to catch
     # the MemoryErrors that could occur.
     transaction = _cast_voidp_to_transaction(transactionptr)
-    transaction._next_transaction = None
+    #
+    # Sanity-check that C code cleared '_next_transaction' first.
+    # Also needs a bit of nonsensical code to make sure that
+    # '_next_transaction' is always created as a general field.
+    ll_assert(transaction._next_transaction is None,
+              "transaction._next_transaction should be cleared by the C code")
+    if NonConstant(False):
+        transaction._next_transaction = transaction
+    #
     transaction.retry_counter = retry_counter
     new_transactions = transaction.run()
     return _link_new_transactions(new_transactions)
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -21,6 +21,8 @@
         self._add(initial_transaction_ptr)
         while self._pending:
             r, transactionptr = self._pending.popitem()
+            transaction = self.cast_voidp_to_transaction(transactionptr)
+            transaction._next_transaction = None
             nextptr = rstm._stm_run_transaction(transactionptr, 0)
             next = self.cast_voidp_to_transaction(nextptr)
             while next is not None:
diff --git a/pypy/translator/stm/src_stm/fifo.c b/pypy/translator/stm/src_stm/fifo.c
--- a/pypy/translator/stm/src_stm/fifo.c
+++ b/pypy/translator/stm/src_stm/fifo.c
@@ -40,17 +40,14 @@
   return (fifo->first == NULL);
 }
 
-/* static bool_t fifo_is_of_length_1(stm_fifo_t *fifo) */
-/* { */
-/*   return (fifo->first != NULL && fifo->first == fifo->last); */
-/* } */
-
 static void *fifo_popleft(stm_fifo_t *fifo)
 {
   void *item = fifo->first;
   fifo->first = NEXT(item);
   if (fifo->first == NULL)
     fifo->last = NULL;
+  NEXT(item) = NULL;      /* ensure the NEXT is cleared,
+                             to avoid spurious keepalives */
   return item;
 }
 
diff --git a/pypy/translator/stm/src_stm/rpyintf.c b/pypy/translator/stm/src_stm/rpyintf.c
--- a/pypy/translator/stm/src_stm/rpyintf.c
+++ b/pypy/translator/stm/src_stm/rpyintf.c
@@ -93,7 +93,7 @@
           while (1)
             {
               new_transaction_list = perform_transaction(transaction);
-              
+
               /* for now, always break out of this loop,
                  unless 'new_transaction_list' contains precisely one item */
               if (new_transaction_list == NULL)
diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py
--- a/pypy/translator/stm/test/test_ztranslated.py
+++ b/pypy/translator/stm/test/test_ztranslated.py
@@ -1,4 +1,5 @@
 import py
+from pypy.rlib import rstm, rgc
 from pypy.translator.stm.test.support import CompiledSTMTests
 from pypy.translator.stm.test import targetdemo
 
@@ -11,33 +12,19 @@
         assert 'check ok!' in data
 
     def test_bug1(self):
-        from pypy.rlib import rstm, rgc
-        from pypy.module.transaction import threadintf
         #
-        class State:
-            pass
-        state = State()
-        #
-        def _foo(_, retry_counter):
-            rgc.collect(0)
-        def _run_thread():
-            rstm.descriptor_init()
-            rstm.perform_transaction(_foo, X, None)
-            threadintf.release(state.ll_unfinished_lock)
-            rstm.descriptor_done()
+        class InitialTransaction(rstm.Transaction):
+            def run(self):
+                rgc.collect(0)
         #
         class X:
             def __init__(self, count):
                 self.count = count
         def g():
             x = X(1000)
-            rstm.enter_transactional_mode()
-            threadintf.start_new_thread(_run_thread)
-            threadintf.acquire(state.ll_unfinished_lock, True)
-            rstm.leave_transactional_mode()
+            rstm.run_all_transactions(InitialTransaction())
             return x
         def entry_point(argv):
-            state.ll_unfinished_lock = threadintf.allocate_lock()
             x = X(len(argv))
             y = g()
             print '<', x.count, y.count, '>'
@@ -48,7 +35,10 @@
         assert '< 5 1000 >' in data, "got: %r" % (data,)
 
     def test_bug2(self):
-        from pypy.rlib import rstm
+        #
+        class DoNothing(rstm.Transaction):
+            def run(self):
+                pass
         #
         class X2:
             pass
@@ -58,8 +48,8 @@
             x = prebuilt2[count]
             x.foobar = 2                    # 'x' becomes a local
             #
-            rstm.enter_transactional_mode() # 'x' becomes the global again
-            rstm.leave_transactional_mode()
+            rstm.run_all_transactions(DoNothing())
+                                            # 'x' becomes the global again
             #
             y = prebuilt2[count]            # same prebuilt obj
             y.foobar += 10                  # 'y' becomes a local

From noreply at buildbot.pypy.org  Sun Apr 22 23:00:01 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 23:00:01 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix test
Message-ID: <20120422210001.9E6738203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54630:0eb6503b3f3a
Date: 2012-04-22 20:39 +0200
http://bitbucket.org/pypy/pypy/changeset/0eb6503b3f3a/

Log:	Fix test

diff --git a/pypy/translator/stm/test/test_localtracker.py b/pypy/translator/stm/test/test_localtracker.py
--- a/pypy/translator/stm/test/test_localtracker.py
+++ b/pypy/translator/stm/test/test_localtracker.py
@@ -246,4 +246,5 @@
         assert name not in hop.rtyper.annotator.translator._seen_locals, (
             "duplicate name %r" % (name,))
         hop.rtyper.annotator.translator._seen_locals[name] = v
+        hop.exception_cannot_occur()
         return hop.inputconst(lltype.Void, None)

From noreply at buildbot.pypy.org  Sun Apr 22 23:00:02 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 23:00:02 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Start to re-enable test_stmgcintf.
Message-ID: <20120422210002.D81BB8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54631:05577eabf224
Date: 2012-04-22 20:58 +0200
http://bitbucket.org/pypy/pypy/changeset/05577eabf224/

Log:	Start to re-enable test_stmgcintf.

diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -9,7 +9,6 @@
 #define _ET_H
 
 #include <setjmp.h>
-#include "src/commondefs.h"
 
 
 /* see comments in ../stmgcintf.py */
@@ -29,7 +28,7 @@
    and from the GC (see llop.nop(...)) */
 extern void pypy_g__stm_thread_starting(void);
 extern void pypy_g__stm_thread_stopping(void);
-extern void *_stm_run_transaction(void *, long);
+extern void *pypy_g__stm_run_transaction(void *, long);
 extern long pypy_g__stm_getsize(void *);
 extern void pypy_g__stm_enum_callback(void *, void *, void *);
 
diff --git a/pypy/translator/stm/stmgcintf.py b/pypy/translator/stm/stmgcintf.py
--- a/pypy/translator/stm/stmgcintf.py
+++ b/pypy/translator/stm/stmgcintf.py
@@ -13,7 +13,6 @@
     includes = ['src_stm/et.h'],
     pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT,
                         '#define RPY_STM 1'],
-    separate_module_sources = ['\n'],    # hack for test_rffi_stm
 )
 
 def _llexternal(name, args, result, **kwds):
diff --git a/pypy/translator/stm/test/getlib.py b/pypy/translator/stm/test/getlib.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/test/getlib.py
@@ -0,0 +1,51 @@
+from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.translator.stm.stmgcintf import eci
+
+
+src_code = r'''
+
+struct pypy_header0 {
+    long h_tid;
+    void *h_version;
+};
+
+struct pypy_pypy_rlib_rstm_Transaction0 {
+    struct pypy_header0 header;
+    struct pypy_pypy_rlib_rstm_Transaction0 *t_inst__next_transaction;
+};
+
+
+#include "src_stm/et.h"
+#include "src_stm/et.c"
+
+
+void (*stm_thread_starting)(void);
+void (*stm_thread_stopping)(void);
+void *(*stm_run_transaction)(void *, long);
+long (*stm_getsize)(void *);
+void (*stm_enum_callback)(void *, void *, void *);
+
+
+void pypy_g__stm_thread_starting(void) {
+    stm_thread_starting();
+}
+void pypy_g__stm_thread_stopping(void) {
+    stm_thread_stopping();
+}
+void *pypy_g__stm_run_transaction(void *a, long b) {
+    return stm_run_transaction(a, b);
+}
+long pypy_g__stm_getsize(void *a) {
+    return stm_getsize(a);
+}
+void pypy_g__stm_enum_callback(void *a, void *b, void *c) {
+    stm_enum_callback(a, b, c);
+}
+'''
+
+
+assert not hasattr(eci, '_with_ctypes')
+
+eci._with_ctypes = eci.merge(ExternalCompilationInfo(
+    separate_module_sources = [src_code],
+    ))
diff --git a/pypy/translator/stm/test/test_stmgcintf.py b/pypy/translator/stm/test/test_stmgcintf.py
--- a/pypy/translator/stm/test/test_stmgcintf.py
+++ b/pypy/translator/stm/test/test_stmgcintf.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.annlowlevel import llhelper
 from pypy.translator.stm.stmgcintf import StmOperations
+from pypy.translator.stm.test import getlib    # for the side-effects
 from pypy.rpython.memory.gc import stmgc
 
 WORD = stmgc.WORD

From noreply at buildbot.pypy.org  Sun Apr 22 23:00:04 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 23:00:04 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Start porting these tests as C code.
Message-ID: <20120422210004.34BF38203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54632:73a511a0a3df
Date: 2012-04-22 22:00 +0200
http://bitbucket.org/pypy/pypy/changeset/73a511a0a3df/

Log:	Start porting these tests as C code.

diff --git a/pypy/translator/stm/test/getlib.py b/pypy/translator/stm/test/getlib.py
deleted file mode 100644
--- a/pypy/translator/stm/test/getlib.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from pypy.translator.tool.cbuild import ExternalCompilationInfo
-from pypy.translator.stm.stmgcintf import eci
-
-
-src_code = r'''
-
-struct pypy_header0 {
-    long h_tid;
-    void *h_version;
-};
-
-struct pypy_pypy_rlib_rstm_Transaction0 {
-    struct pypy_header0 header;
-    struct pypy_pypy_rlib_rstm_Transaction0 *t_inst__next_transaction;
-};
-
-
-#include "src_stm/et.h"
-#include "src_stm/et.c"
-
-
-void (*stm_thread_starting)(void);
-void (*stm_thread_stopping)(void);
-void *(*stm_run_transaction)(void *, long);
-long (*stm_getsize)(void *);
-void (*stm_enum_callback)(void *, void *, void *);
-
-
-void pypy_g__stm_thread_starting(void) {
-    stm_thread_starting();
-}
-void pypy_g__stm_thread_stopping(void) {
-    stm_thread_stopping();
-}
-void *pypy_g__stm_run_transaction(void *a, long b) {
-    return stm_run_transaction(a, b);
-}
-long pypy_g__stm_getsize(void *a) {
-    return stm_getsize(a);
-}
-void pypy_g__stm_enum_callback(void *a, void *b, void *c) {
-    stm_enum_callback(a, b, c);
-}
-'''
-
-
-assert not hasattr(eci, '_with_ctypes')
-
-eci._with_ctypes = eci.merge(ExternalCompilationInfo(
-    separate_module_sources = [src_code],
-    ))
diff --git a/pypy/translator/stm/test/test_stmgcintf.c b/pypy/translator/stm/test/test_stmgcintf.c
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/test/test_stmgcintf.c
@@ -0,0 +1,130 @@
+
+#define PYPY_LONG_BIT   (sizeof(long) * 8)
+
+struct pypy_header0 {
+    long h_tid;
+    void *h_version;
+};
+
+struct pypy_pypy_rlib_rstm_Transaction0 {
+    struct pypy_header0 header;
+    struct pypy_pypy_rlib_rstm_Transaction0 *t_inst__next_transaction;
+    int foobar;
+    void (*callback)(void);
+};
+
+typedef char bool_t;
+
+
+#include "src_stm/et.h"
+#include "src_stm/et.c"
+
+
+void *(*cb_run_transaction)(void *, long);
+long (*cb_getsize)(void *);
+void (*cb_enum_callback)(void *, void *, void *);
+
+int _thread_started = 0;
+
+
+void pypy_g__stm_thread_starting(void) {
+    assert(_thread_started == 0);
+    _thread_started = 1;
+    stm_set_tls((void *)742, 0);
+}
+void pypy_g__stm_thread_stopping(void) {
+    assert(_thread_started == 1);
+    _thread_started = 0;
+    stm_del_tls();
+}
+void *pypy_g__stm_run_transaction(void *a, long b) {
+    assert(cb_run_transaction != NULL);
+    return cb_run_transaction(a, b);
+}
+long pypy_g__stm_getsize(void *a) {
+    assert(cb_getsize != NULL);
+    return cb_getsize(a);
+}
+void pypy_g__stm_enum_callback(void *a, void *b, void *c) {
+    assert(cb_enum_callback != NULL);
+    cb_enum_callback(a, b, c);
+}
+
+
+void test_set_get_del(void)
+{
+    stm_set_tls((void *)42, 1);
+    assert(stm_get_tls() == (void *)42);
+    stm_del_tls();
+}
+
+void *rt1(void *t1, long retry_counter)
+{
+    struct pypy_pypy_rlib_rstm_Transaction0 *t = t1;
+    assert(retry_counter == 0);
+    assert(t->foobar == 42);
+    t->foobar = 143;
+    return NULL;
+}
+void test_run_all_transactions(void)
+{
+    struct pypy_pypy_rlib_rstm_Transaction0 t;
+    t.foobar = 42;
+    cb_run_transaction = rt1;
+    stm_run_all_transactions(&t, 1);
+    assert(t.foobar == 143);
+}
+
+void *rt2(void *t1, long retry_counter)
+{
+    struct pypy_pypy_rlib_rstm_Transaction0 *t = t1;
+    if (retry_counter > 0) {
+        t->foobar = retry_counter;
+        return NULL;
+    }
+    t->callback();
+    t->foobar = 81;
+    return NULL;
+}
+void run_in_transaction(void(*cb)(void), int expected)
+{
+    struct pypy_pypy_rlib_rstm_Transaction0 t;
+    t.callback = cb;
+    cb_run_transaction = rt2;
+    stm_run_all_transactions(&t, 1);
+    assert(t.foobar == expected);
+}
+
+void tldict(void)
+{
+    void *a1 = (void *)0x4020;
+    void *a2 = (void *)10002;
+    void *a3 = (void *)0x4028;
+    void *a4 = (void *)10004;
+
+    assert(stm_tldict_lookup(a1) == NULL);
+    stm_tldict_add(a1, a2);
+    assert(stm_tldict_lookup(a1) == a2);
+
+    assert (stm_tldict_lookup(a3) == NULL);
+    stm_tldict_add(a3, a4);
+    assert(stm_tldict_lookup(a3) == a4);
+    assert(stm_tldict_lookup(a1) == a2);
+    stm_abort_and_retry();
+}
+void test_tldict(void)
+{
+    run_in_transaction(tldict, 1);
+}
+
+
+#define XTEST(name)  if (!strcmp(argv[1], #name)) { test_##name(); return 0; }
+
+int main(int argc, char **argv)
+{
+    XTEST(set_get_del);
+    XTEST(run_all_transactions);
+    XTEST(tldict);
+    printf("bad test name\n");
+    return 1;
+}
diff --git a/pypy/translator/stm/test/test_stmgcintf.py b/pypy/translator/stm/test/test_stmgcintf.py
--- a/pypy/translator/stm/test/test_stmgcintf.py
+++ b/pypy/translator/stm/test/test_stmgcintf.py
@@ -1,288 +1,20 @@
-import py, random
-from pypy.rpython.lltypesystem import lltype, llmemory, rffi
-from pypy.rpython.annlowlevel import llhelper
-from pypy.translator.stm.stmgcintf import StmOperations
-from pypy.translator.stm.test import getlib    # for the side-effects
-from pypy.rpython.memory.gc import stmgc
+import os
+from pypy.tool.udir import udir
 
-WORD = stmgc.WORD
 
-stm_operations = StmOperations()
+def test_all():
+    executable = str(udir.join('test_stmgcintf'))
+    exitcode = os.system("gcc -g -o '%s' -pthread -I.. test_stmgcintf.c" % (
+        executable,))
+    assert exitcode == 0
+    #
+    for line in open('test_stmgcintf.c'):
+        line = line.strip()
+        if line.startswith('XTEST('):
+            assert line.endswith(');')
+            yield run_one_test, executable, line[6:-2]
 
-DEFAULT_TLS = lltype.Struct('DEFAULT_TLS')
 
-S1 = lltype.Struct('S1', ('hdr', stmgc.StmGC.HDR),
-                         ('x', lltype.Signed),
-                         ('y', lltype.Signed))
-
-# xxx a lot of casts to convince rffi to give us a regular integer :-(
-SIZEOFHDR = rffi.cast(lltype.Signed, rffi.cast(rffi.SHORT,
-                                               rffi.sizeof(S1.hdr)))
-
-
-def test_set_get_del():
-    # assume that they are really thread-local; not checked here
-    s = lltype.malloc(lltype.Struct('S'), flavor='raw')
-    a = llmemory.cast_ptr_to_adr(s)
-    stm_operations.set_tls(a, 1)
-    assert stm_operations.get_tls() == a
-    stm_operations.del_tls()
-    lltype.free(s, flavor='raw')
-
-
-class TestStmGcIntf:
-    _in_transaction = False
-
-    def setup_method(self, meth):
-        TLS = getattr(meth, 'TLS', DEFAULT_TLS)
-        s = lltype.malloc(TLS, flavor='raw', immortal=True)
-        self.tls = s
-        a = llmemory.cast_ptr_to_adr(s)
-        in_transaction = getattr(meth, 'in_transaction', False)
-        in_main_thread = getattr(meth, 'in_main_thread', not in_transaction)
-        stm_operations.set_tls(a, int(in_main_thread))
-        if in_transaction:
-            stm_operations._activate_transaction(1)
-            self._in_transaction = True
-
-    def teardown_method(self, meth):
-        if self._in_transaction:
-            stm_operations._activate_transaction(0)
-        stm_operations.del_tls()
-
-    def test_set_get_del(self):
-        a = llmemory.cast_ptr_to_adr(self.tls)
-        assert stm_operations.get_tls() == a
-
-    def test_tldict(self):
-        a1 = rffi.cast(llmemory.Address, 0x4020)
-        a2 = rffi.cast(llmemory.Address, 10002)
-        a3 = rffi.cast(llmemory.Address, 0x4028)
-        a4 = rffi.cast(llmemory.Address, 10004)
-        #
-        assert stm_operations.tldict_lookup(a1) == llmemory.NULL
-        stm_operations.tldict_add(a1, a2)
-        assert stm_operations.tldict_lookup(a1) == a2
-        #
-        assert stm_operations.tldict_lookup(a3) == llmemory.NULL
-        stm_operations.tldict_add(a3, a4)
-        assert stm_operations.tldict_lookup(a3) == a4
-        assert stm_operations.tldict_lookup(a1) == a2
-    test_tldict.in_transaction = True
-
-    def test_tldict_large(self):
-        content = {}
-        for i in range(12000):
-            key = random.randrange(1000, 2000) * WORD
-            a1 = rffi.cast(llmemory.Address, key)
-            a2 = stm_operations.tldict_lookup(a1)
-            if key in content:
-                assert a2 == content[key]
-            else:
-                assert a2 == llmemory.NULL
-                a2 = rffi.cast(llmemory.Address, random.randrange(2000, 9999))
-                stm_operations.tldict_add(a1, a2)
-                content[key] = a2
-    test_tldict_large.in_transaction = True
-
-    def get_callback_enum(self):
-        def callback(tls, key, value):
-            assert tls == llmemory.cast_ptr_to_adr(self.tls)
-            seen.append((key, value))
-        seen = []
-        p_callback = llhelper(StmOperations.CALLBACK_ENUM, callback)
-        return p_callback, seen
-
-    def test_enum_tldict_empty(self):
-        p_callback, seen = self.get_callback_enum()
-        stm_operations.tldict_enum(p_callback)
-        assert seen == []
-
-    def test_enum_tldict_nonempty(self):
-        a1 = rffi.cast(llmemory.Address, 0x4020)
-        a2 = rffi.cast(llmemory.Address, 10002)
-        a3 = rffi.cast(llmemory.Address, 0x4028)
-        a4 = rffi.cast(llmemory.Address, 10004)
-        #
-        stm_operations.tldict_add(a1, a2)
-        stm_operations.tldict_add(a3, a4)
-        p_callback, seen = self.get_callback_enum()
-        stm_operations.tldict_enum(p_callback)
-        assert (seen == [(a1, a2), (a3, a4)] or
-                seen == [(a3, a4), (a1, a2)])
-    test_enum_tldict_nonempty.in_transaction = True
-
-    def stm_read_case(self, flags, copied=False):
-        # doesn't test STM behavior, but just that it appears to work
-        s1 = lltype.malloc(S1, flavor='raw')
-        s1.hdr.tid = stmgc.GCFLAG_GLOBAL | flags
-        s1.hdr.version = llmemory.NULL
-        s1.x = 42042
-        if copied:
-            s2 = lltype.malloc(S1, flavor='raw')
-            s2.hdr.tid = stmgc.GCFLAG_WAS_COPIED
-            s2.hdr.version = llmemory.NULL
-            s2.x = 84084
-            a1 = llmemory.cast_ptr_to_adr(s1)
-            a2 = llmemory.cast_ptr_to_adr(s2)
-            stm_operations.tldict_add(a1, a2)
-        reader = getattr(stm_operations, 'stm_read_int%d' % WORD)
-        res = reader(llmemory.cast_ptr_to_adr(s1), SIZEOFHDR)   # 'x'
-        lltype.free(s1, flavor='raw')
-        if copied:
-            lltype.free(s2, flavor='raw')
-        return res
-
-    def test_stm_read_word_main_thread(self):
-        res = self.stm_read_case(0)                        # not copied
-        assert res == 42042
-        res = self.stm_read_case(stmgc.GCFLAG_WAS_COPIED)  # but ignored
-        assert res == 42042
-
-    def test_stm_read_word_transactional_thread(self):
-        res = self.stm_read_case(0)                        # not copied
-        assert res == 42042
-        res = self.stm_read_case(stmgc.GCFLAG_WAS_COPIED)  # but ignored
-        assert res == 42042
-        res = self.stm_read_case(stmgc.GCFLAG_WAS_COPIED, copied=True)
-        assert res == 84084
-    test_stm_read_word_transactional_thread.in_transaction = True
-
-    def stm_read_intX(self, TYPE, sizesuffix):
-        print TYPE, sizesuffix
-        S2 = lltype.Struct('S2', ('hdr', stmgc.StmGC.HDR),
-                                 ('c1', TYPE),
-                                 ('c2', TYPE),
-                                 ('c3', TYPE))
-        s2 = lltype.malloc(S2, flavor='raw')
-        s2.hdr.tid = stmgc.GCFLAG_GLOBAL | stmgc.GCFLAG_WAS_COPIED
-        s2.hdr.version = llmemory.NULL
-        s2.c1 = A = rffi.cast(TYPE, -65)
-        s2.c2 = B = rffi.cast(TYPE, -66)
-        s2.c3 = C = rffi.cast(TYPE, -67)
-        size = rffi.sizeof(TYPE)
-        assert sizesuffix in (size, '%df' % size)
-        reader = getattr(stm_operations, 'stm_read_int%s' % sizesuffix)
-        r1 = reader(llmemory.cast_ptr_to_adr(s2), SIZEOFHDR + 0 * size)  # c1
-        r2 = reader(llmemory.cast_ptr_to_adr(s2), SIZEOFHDR + 1 * size)  # c2
-        r3 = reader(llmemory.cast_ptr_to_adr(s2), SIZEOFHDR + 2 * size)  # c3
-        lltype.free(s2, flavor='raw')
-        assert r1 == A and r2 == B and r3 == C
-
-    def test_stm_read_int(self):
-        for size, TYPE in StmOperations.PRIMITIVE_SIZES.items():
-            yield self.stm_read_intX, TYPE, size
-
-    def test_stm_size_getter(self):
-        def getsize(addr):
-            dont_call_me
-        getter = llhelper(StmOperations.GETSIZE, getsize)
-        stm_operations.setup_size_getter(getter)
-        # ^^^ just tests that the function is really defined
-
-    def test_stm_copy_transactional_to_raw(self):
-        # doesn't test STM behavior, but just that it appears to work
-        s1 = lltype.malloc(S1, flavor='raw')
-        s1.hdr.tid = stmgc.GCFLAG_GLOBAL
-        s1.hdr.version = llmemory.NULL
-        s1.x = 909
-        s1.y = 808
-        s2 = lltype.malloc(S1, flavor='raw')
-        s2.hdr.tid = -42    # non-initialized
-        s2.x = -42          # non-initialized
-        s2.y = -42          # non-initialized
-        #
-        s1_adr = llmemory.cast_ptr_to_adr(s1)
-        s2_adr = llmemory.cast_ptr_to_adr(s2)
-        size   = llmemory.sizeof(S1)
-        stm_operations.stm_copy_transactional_to_raw(s1_adr, s2_adr, size)
-        #
-        assert s2.hdr.tid == -42    # not touched
-        assert s2.x == 909
-        assert s2.y == 808
-        #
-        lltype.free(s2, flavor='raw')
-        lltype.free(s1, flavor='raw')
-    test_stm_copy_transactional_to_raw.in_transaction = True
-
-    def test_in_transaction(self):
-        assert stm_operations.in_transaction()
-    test_in_transaction.in_transaction = True
-
-    def test_not_in_transaction(self):
-        assert not stm_operations.in_transaction()
-    test_not_in_transaction.in_main_thread = False
-
-    def test_not_in_transaction_main(self):
-        assert not stm_operations.in_transaction()
-    test_not_in_transaction.in_main_thread = True
-
-    def test_stm_perform_transaction(self):
-        def callback1(x, retry_counter):
-            return rffi.cast(rffi.VOIDP, -123)
-        x = stm_operations.perform_transaction(
-            llhelper(StmOperations.CALLBACK_TX, callback1),
-            lltype.nullptr(rffi.VOIDP.TO))
-        assert x == rffi.cast(rffi.VOIDP, -123)
-
-    def test_stm_try_inevitable(self):
-        # not really testing anything more than the presence of the function
-        stm_operations.try_inevitable()
-    test_stm_try_inevitable.in_transaction = True
-
-    def test_thread_id_main(self):
-        assert stm_operations.thread_id() == 0
-    test_thread_id_main.in_main_thread = True
-
-    def test_thread_id_nonmain(self):
-        assert stm_operations.thread_id() != 0
-    test_thread_id_nonmain.in_main_thread = False
-
-    def test_abort_and_retry(self):
-        def callback1(x, retry_counter):
-            assert 0 <= retry_counter <= 10
-            if retry_counter == 10:
-                return rffi.cast(rffi.VOIDP, -42)
-            stm_operations.abort_and_retry()
-            assert 0   # not reachable
-        x = stm_operations.perform_transaction(
-            llhelper(StmOperations.CALLBACK_TX, callback1),
-            lltype.nullptr(rffi.VOIDP.TO))
-        assert x == rffi.cast(rffi.VOIDP, -42)
-
-    def test_debug_get_state_main_thread(self):
-        st = stm_operations._debug_get_state()
-        assert st == stm_operations.STATE_MAIN_THREAD
-    test_debug_get_state_main_thread.in_main_thread = True
-
-    def test_debug_get_state_inactive(self):
-        st = stm_operations._debug_get_state()
-        assert st == stm_operations.STATE_INACTIVE
-    test_debug_get_state_inactive.in_main_thread = False
-
-    def test_debug_get_state_active(self):
-        def callback1(x, retry_counter):
-            st = stm_operations._debug_get_state()
-            return rffi.cast(rffi.VOIDP, st)
-        x = stm_operations.perform_transaction(
-            llhelper(StmOperations.CALLBACK_TX, callback1),
-            lltype.nullptr(rffi.VOIDP.TO))
-        assert rffi.cast(lltype.Signed, x) == stm_operations.STATE_ACTIVE
-    test_debug_get_state_active.in_main_thread = False
-
-    def test_debug_get_state_active_inevitable(self):
-        def callback1(x, retry_counter):
-            stm_operations.try_inevitable()
-            st = stm_operations._debug_get_state()
-            return rffi.cast(rffi.VOIDP, st)
-        x = stm_operations.perform_transaction(
-            llhelper(StmOperations.CALLBACK_TX, callback1),
-            lltype.nullptr(rffi.VOIDP.TO))
-        assert (rffi.cast(lltype.Signed, x) ==
-                stm_operations.STATE_ACTIVE_INEVITABLE)
-    test_debug_get_state_active_inevitable.in_main_thread = False
-
-
-def test_debug_get_state_not_initialized():
-    st = stm_operations._debug_get_state()
-    assert st == stm_operations.STATE_NOT_INITIALIZED
+def run_one_test(executable, testname):
+    exitcode = os.system("'%s' %s" % (executable, testname))
+    assert exitcode == 0, "exitcode is %r running %r" % (exitcode, testname)

From noreply at buildbot.pypy.org  Sun Apr 22 23:00:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 23:00:05 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Next test
Message-ID: <20120422210005.6DE418203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54633:111c611155d3
Date: 2012-04-22 22:06 +0200
http://bitbucket.org/pypy/pypy/changeset/111c611155d3/

Log:	Next test

diff --git a/pypy/translator/stm/test/test_stmgcintf.c b/pypy/translator/stm/test/test_stmgcintf.c
--- a/pypy/translator/stm/test/test_stmgcintf.c
+++ b/pypy/translator/stm/test/test_stmgcintf.c
@@ -51,30 +51,6 @@
 }
 
 
-void test_set_get_del(void)
-{
-    stm_set_tls((void *)42, 1);
-    assert(stm_get_tls() == (void *)42);
-    stm_del_tls();
-}
-
-void *rt1(void *t1, long retry_counter)
-{
-    struct pypy_pypy_rlib_rstm_Transaction0 *t = t1;
-    assert(retry_counter == 0);
-    assert(t->foobar == 42);
-    t->foobar = 143;
-    return NULL;
-}
-void test_run_all_transactions(void)
-{
-    struct pypy_pypy_rlib_rstm_Transaction0 t;
-    t.foobar = 42;
-    cb_run_transaction = rt1;
-    stm_run_all_transactions(&t, 1);
-    assert(t.foobar == 143);
-}
-
 void *rt2(void *t1, long retry_counter)
 {
     struct pypy_pypy_rlib_rstm_Transaction0 *t = t1;
@@ -95,6 +71,36 @@
     assert(t.foobar == expected);
 }
 
+/************************************************************/
+
+void test_set_get_del(void)
+{
+    stm_set_tls((void *)42, 1);
+    assert(stm_get_tls() == (void *)42);
+    stm_del_tls();
+}
+
+/************************************************************/
+
+void *rt1(void *t1, long retry_counter)
+{
+    struct pypy_pypy_rlib_rstm_Transaction0 *t = t1;
+    assert(retry_counter == 0);
+    assert(t->foobar == 42);
+    t->foobar = 143;
+    return NULL;
+}
+void test_run_all_transactions(void)
+{
+    struct pypy_pypy_rlib_rstm_Transaction0 t;
+    t.foobar = 42;
+    cb_run_transaction = rt1;
+    stm_run_all_transactions(&t, 1);
+    assert(t.foobar == 143);
+}
+
+/************************************************************/
+
 void tldict(void)
 {
     void *a1 = (void *)0x4020;
@@ -112,10 +118,35 @@
     assert(stm_tldict_lookup(a1) == a2);
     stm_abort_and_retry();
 }
-void test_tldict(void)
+void test_tldict(void) { run_in_transaction(tldict, 1); }
+
+/************************************************************/
+
+void tldict_large(void)
 {
-    run_in_transaction(tldict, 1);
+    void *content[1024] = { 0 };
+    int i;
+    for (i=0; i<120000; i++) {
+        long key_index = rand() & 1023;
+        void *a1 = (void *)(10000 + key_index * 8);
+        void *a2 = stm_tldict_lookup(a1);
+
+        if (content[key_index] != NULL) {
+            assert(a2 == content[key_index]);
+        }
+        else {
+            assert(a2 == NULL);
+            while (a2 == NULL)
+                a2 = (void *)rand();
+            stm_tldict_add(a1, a2);
+            content[key_index] = a2;
+        }
+    }
+    stm_abort_and_retry();
 }
+void test_tldict_large(void) { run_in_transaction(tldict_large, 1); }
+
+/************************************************************/
 
 
 #define XTEST(name)  if (!strcmp(argv[1], #name)) { test_##name(); return 0; }
@@ -125,6 +156,7 @@
     XTEST(set_get_del);
     XTEST(run_all_transactions);
     XTEST(tldict);
+    XTEST(tldict_large);
     printf("bad test name\n");
     return 1;
 }

From noreply at buildbot.pypy.org  Sun Apr 22 23:00:06 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 22 Apr 2012 23:00:06 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Tests tests tests... and found one
	missing "return; " :-)
Message-ID: <20120422210006.B536D8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54634:4aa01a6e557f
Date: 2012-04-22 22:51 +0200
http://bitbucket.org/pypy/pypy/changeset/4aa01a6e557f/

Log:	Tests tests tests... and found one missing "return;" :-)

diff --git a/pypy/translator/stm/src_stm/core.c b/pypy/translator/stm/src_stm/core.c
--- a/pypy/translator/stm/src_stm/core.c
+++ b/pypy/translator/stm/src_stm/core.c
@@ -550,6 +550,7 @@
         }
       d->num_commits++;
       common_cleanup(d);
+      return;
     }
 
   // bring that variable over to this CPU core (optimization, maybe)
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -49,14 +49,8 @@
 #endif
 
 
-void* stm_perform_transaction(void*(*)(void*, long), void*, void*);
 void stm_try_inevitable(STM_CCHARP1(why));
 void stm_abort_and_retry(void);
-long stm_debug_get_state(void);  /* -1: descriptor_init() was not called
-                                     0: not in a transaction
-                                     1: in a regular transaction
-                                     2: in an inevitable transaction */
-void _stm_activate_transaction(long);
 
 void stm_copy_transactional_to_raw(void *src, void *dst, long size);
 
diff --git a/pypy/translator/stm/test/test_stmgcintf.c b/pypy/translator/stm/test/test_stmgcintf.c
--- a/pypy/translator/stm/test/test_stmgcintf.c
+++ b/pypy/translator/stm/test/test_stmgcintf.c
@@ -1,3 +1,4 @@
+#include <stddef.h>
 
 #define PYPY_LONG_BIT   (sizeof(long) * 8)
 
@@ -13,6 +14,17 @@
     void (*callback)(void);
 };
 
+typedef struct {
+    struct pypy_header0 header;
+    char value1;
+    short value2;
+    int value4;
+    long long value8;
+    double value8f;
+    float value4f;
+    char last_16_bytes[16];
+} S1;
+
 typedef char bool_t;
 
 
@@ -59,7 +71,7 @@
         return NULL;
     }
     t->callback();
-    t->foobar = 81;
+    t->foobar = '.';
     return NULL;
 }
 void run_in_transaction(void(*cb)(void), int expected)
@@ -69,6 +81,7 @@
     cb_run_transaction = rt2;
     stm_run_all_transactions(&t, 1);
     assert(t.foobar == expected);
+
 }
 
 /************************************************************/
@@ -124,10 +137,10 @@
 
 void tldict_large(void)
 {
-    void *content[1024] = { 0 };
+    void *content[4096] = { 0 };
     int i;
     for (i=0; i<120000; i++) {
-        long key_index = rand() & 1023;
+        long key_index = rand() & 4095;
         void *a1 = (void *)(10000 + key_index * 8);
         void *a2 = stm_tldict_lookup(a1);
 
@@ -148,6 +161,189 @@
 
 /************************************************************/
 
+void enum_tldict_empty(void)
+{
+    stm_tldict_enum();
+}
+void test_enum_tldict_empty(void) {
+    run_in_transaction(enum_tldict_empty, '.'); }
+
+/************************************************************/
+
+int check_enum_1_found;
+void check_enum_1(void *tls, void *a, void *b)
+{
+    int n;
+    assert(tls == (void *)742);
+    if (a == (void *)0x4020 && b == (void *)10002)
+        n = 1;
+    else if (a == (void *)0x4028 && b == (void *)10004)
+        n = 2;
+    else
+        assert(!"unexpected a or b");
+    assert((check_enum_1_found & n) == 0);
+    check_enum_1_found |= n;
+}
+void enum_tldict_nonempty(void)
+{
+    void *a1 = (void *)0x4020;
+    void *a2 = (void *)10002;
+    void *a3 = (void *)0x4028;
+    void *a4 = (void *)10004;
+
+    stm_tldict_add(a1, a2);
+    stm_tldict_add(a3, a4);
+    cb_enum_callback = check_enum_1;
+    check_enum_1_found = 0;
+    stm_tldict_enum();
+    assert(check_enum_1_found == (1|2));
+    stm_abort_and_retry();
+}
+void test_enum_tldict_nonempty(void) {
+    run_in_transaction(enum_tldict_nonempty, 1); }
+
+/************************************************************/
+
+void test_read_main_thread(void)
+{
+    S1 s1;
+    int i;
+    for (i=0; i<2; i++) {
+        s1.header.h_tid = GCFLAG_GLOBAL | (i ? GCFLAG_WAS_COPIED : 0);
+        s1.header.h_version = NULL;
+        s1.value1 = 49;
+        s1.value2 = 3981;
+        s1.value4 = 4204229;
+        s1.value8 = 3419103092099219LL;
+        s1.value8f = 289.25;
+        s1.value4f = -5.5;
+
+        assert(stm_read_int1( &s1, offsetof(S1, value1 )) == 49);
+        assert(stm_read_int2( &s1, offsetof(S1, value2 )) == 3981);
+        assert(stm_read_int4( &s1, offsetof(S1, value4 )) == 4204229);
+        assert(stm_read_int8( &s1, offsetof(S1, value8))== 3419103092099219LL);
+        assert(stm_read_int8f(&s1, offsetof(S1, value8f)) == 289.25);
+        assert(stm_read_int4f(&s1, offsetof(S1, value4f)) == -5.5);
+    }
+}
+
+/************************************************************/
+
+void read_transaction(void)
+{
+    S1 s1, s2;
+    int i;
+    for (i=0; i<2; i++) {
+        s1.header.h_tid = GCFLAG_GLOBAL | (i ? GCFLAG_WAS_COPIED : 0);
+        s1.header.h_version = NULL;
+        s1.value1 = 49;
+        s1.value2 = 3981;
+        s1.value4 = 4204229;
+        s1.value8 = 3419103092099219LL;
+        s1.value8f = 289.25;
+        s1.value4f = -5.5;
+
+        assert(stm_read_int1( &s1, offsetof(S1, value1 )) == 49);
+        assert(stm_read_int2( &s1, offsetof(S1, value2 )) == 3981);
+        assert(stm_read_int4( &s1, offsetof(S1, value4 )) == 4204229);
+        assert(stm_read_int8( &s1, offsetof(S1, value8))== 3419103092099219LL);
+        assert(stm_read_int8f(&s1, offsetof(S1, value8f)) == 289.25);
+        assert(stm_read_int4f(&s1, offsetof(S1, value4f)) == -5.5);
+    }
+
+    s1.header.h_tid = GCFLAG_GLOBAL | GCFLAG_WAS_COPIED;
+    s1.header.h_version = NULL;
+    s2.header.h_tid = GCFLAG_WAS_COPIED;
+    s2.header.h_version = &s1;
+    s2.value1 = -49;
+    s2.value2 = -3981;
+    s2.value4 = -4204229;
+    s2.value8 = -3419103092099219LL;
+    s2.value8f = -289.25;
+    s2.value4f = 5.5;
+    stm_tldict_add(&s1, &s2);
+
+    assert(stm_read_int1( &s1, offsetof(S1, value1 )) == -49);
+    assert(stm_read_int2( &s1, offsetof(S1, value2 )) == -3981);
+    assert(stm_read_int4( &s1, offsetof(S1, value4 )) == -4204229);
+    assert(stm_read_int8( &s1, offsetof(S1, value8))  == -3419103092099219LL);
+    assert(stm_read_int8f(&s1, offsetof(S1, value8f)) == -289.25);
+    assert(stm_read_int4f(&s1, offsetof(S1, value4f)) == 5.5);
+    stm_abort_and_retry();
+}
+void test_read_transaction(void) { run_in_transaction(read_transaction, 1); }
+
+/************************************************************/
+
+int sg_seen = 0;
+S1 sg_global;
+void size_getter(void)
+{
+    S1 *s2 = malloc(sizeof(S1));
+    int i;
+    sg_global.header.h_tid = GCFLAG_GLOBAL | GCFLAG_WAS_COPIED;
+    sg_global.header.h_version = NULL;
+    s2->header.h_tid = GCFLAG_WAS_COPIED;
+    s2->header.h_version = &sg_global;
+    for (i=0; i<16; i++)
+        s2->last_16_bytes[i] = 'A' + i;
+    stm_tldict_add(&sg_global, s2);
+}
+long size_getter_cb(void *x)
+{
+    return offsetof(S1, last_16_bytes) + 15;
+}
+void test_size_getter(void)
+{
+    int i;
+    cb_getsize = size_getter_cb;
+    sg_global.last_16_bytes[15] = '!';
+    run_in_transaction(size_getter, '.');
+    for (i=0; i<15; i++)
+        assert(sg_global.last_16_bytes[i] == 'A' + i);
+    assert(sg_global.last_16_bytes[15] == '!');   /* not overwritten */
+}
+
+/************************************************************/
+
+void copy_transactional_to_raw(void)
+{
+    S1 s1, s2;
+    int i;
+    s1.header.h_tid = GCFLAG_GLOBAL | 99;
+    s1.header.h_version = NULL;
+    for (i=0; i<16; i++)
+        s1.last_16_bytes[i] = 'A' + i;
+    s2.header.h_tid = 101;
+    s2.last_16_bytes[15] = '!';
+    stm_copy_transactional_to_raw(&s1, &s2, offsetof(S1, last_16_bytes) + 15);
+    for (i=0; i<15; i++)
+        assert(s2.last_16_bytes[i] == 'A' + i);
+    assert(s2.last_16_bytes[15] == '!');   /* not overwritten */
+    assert(s2.header.h_tid = 101);         /* not overwritten */
+}
+void test_copy_transactional_to_raw(void) {
+    run_in_transaction(copy_transactional_to_raw, '.');
+}
+
+/************************************************************/
+
+void try_inevitable(void)
+{
+    assert(stm_in_transaction() == 1);
+    assert(stm_thread_id() != 0);
+    /* not really testing anything more than the presence of the function */
+    stm_try_inevitable(STM_EXPLAIN1("some explanation"));
+}
+void test_try_inevitable(void)
+{
+    assert(stm_in_transaction() == 0);
+    assert(stm_thread_id() == 0);
+    run_in_transaction(try_inevitable, '.');
+}
+
+/************************************************************/
+
 
 #define XTEST(name)  if (!strcmp(argv[1], #name)) { test_##name(); return 0; }
 
@@ -157,6 +353,13 @@
     XTEST(run_all_transactions);
     XTEST(tldict);
     XTEST(tldict_large);
+    XTEST(enum_tldict_empty);
+    XTEST(enum_tldict_nonempty);
+    XTEST(read_main_thread);
+    XTEST(read_transaction);
+    XTEST(size_getter);
+    XTEST(copy_transactional_to_raw);
+    XTEST(try_inevitable);
     printf("bad test name\n");
     return 1;
 }

From noreply at buildbot.pypy.org  Mon Apr 23 00:58:23 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 23 Apr 2012 00:58:23 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: dummy op for cpyext/api.py
Message-ID: <20120422225823.4D54E8203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54635:f2e214d6b060
Date: 2012-04-22 23:46 +0300
http://bitbucket.org/pypy/pypy/changeset/f2e214d6b060/

Log:	dummy op for cpyext/api.py

diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -1,5 +1,5 @@
 import os
-from pypy.rpython.lltypesystem.rffi import (CConstant, CExternVariable, 
+from pypy.rpython.lltypesystem.rffi import (CConstant, CExternVariable,
         INT, CCHARPP)
 from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -107,9 +107,12 @@
         if not _validate_fd(fd):
             raise OSError(get_errno(), 'Bad file descriptor')
 else:
+    def _validate_fd(fd):
+        return 1
+
     def validate_fd(fd):
         return 1
-    
+
 def closerange(fd_low, fd_high):
     # this behaves like os.closerange() from Python 2.6.
     for fd in xrange(fd_low, fd_high):

From noreply at buildbot.pypy.org  Mon Apr 23 00:58:24 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 23 Apr 2012 00:58:24 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: be more conservative in failure
	of _validate_fd
Message-ID: <20120422225824.8AA598203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54636:c97ac5d5d8b3
Date: 2012-04-23 01:57 +0300
http://bitbucket.org/pypy/pypy/changeset/c97ac5d5d8b3/

Log:	be more conservative in failure of _validate_fd

diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -5,7 +5,6 @@
 from pypy.rlib import streamio
 from pypy.rlib.rarithmetic import r_longlong
 from pypy.rlib.rstring import StringBuilder
-from pypy.rlib.rposix import validate_fd
 from pypy.module._file.interp_stream import W_AbstractStream, StreamErrors
 from pypy.module.posix.interp_posix import dispatch_filename
 from pypy.interpreter.error import OperationError, operationerrfmt
@@ -272,15 +271,6 @@
 
     def file_fdopen(self, fd, mode="r", buffering=-1):
         try:
-            # Catch invalid fd early to get an OSError rather than an
-            # IOError from streamio
-            validate_fd(fd)
-        except OSError, e:
-            w_error = self.space.call_function(self.space.w_OSError,
-                    self.space.wrap(e.errno), self.space.wrap(e.strerror),
-                    self.space.wrap(e.filename))
-            raise OperationError(self.space.w_OSError, w_error)
-        try:
             self.direct_fdopen(fd, mode, buffering)
         except StreamErrors, e:
             raise wrap_streamerror(self.space, e, self.w_name)
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -265,14 +265,10 @@
 
         if option.runappdirect:
             py.test.skip("works with internals of _file impl on py.py")
-        import platform
-        if platform.system() == 'Windows':
-            # XXX This test crashes until someone implements something like
-            # XXX verify_fd from
-            # XXX http://hg.python.org/cpython/file/80ddbd822227/Modules/posixmodule.c#l434
-            # XXX and adds it to fopen
-            assert False
-
+        if os.name=='nt':
+            mode = 'rb'
+        else:
+            mode = 'r'
         state = [0]
         def read(fd, n=None):
             if fd != 42:
@@ -286,7 +282,7 @@
             return ''
         os.read = read
         stdin = W_File(cls.space)
-        stdin.file_fdopen(42, "r", 1)
+        stdin.file_fdopen(42, mode, 1)
         stdin.name = '<stdin>'
         cls.w_stream = stdin
 
diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -93,7 +93,6 @@
     # XXX XXX XXX you want do check whether the modes are compatible
     # otherwise you get funny results
     os_flags, universal, reading, writing, basemode, binary = decode_mode(mode)
-    rposix.validate_fd(fd)
     _setfd_binary(fd)
     stream = DiskFile(fd)
     return construct_stream_tower(stream, buffering, universal, reading,
@@ -190,7 +189,9 @@
     # When generating on CLI or JVM, these are patched out.
     # See PyPyTarget.target() in targetpypystandalone.py
     def _setfd_binary(fd):
-        _setmode(fd, os.O_BINARY)
+        #Allow this to succeed on invalid fd's
+        if rposix._validate_fd(fd):
+            _setmode(fd, os.O_BINARY)
 
     def ftruncate_win32(fd, size):
         curpos = os.lseek(fd, 0, 1)

From noreply at buildbot.pypy.org  Mon Apr 23 01:16:41 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 23 Apr 2012 01:16:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix translation (@Alex_Gaynor)
Message-ID: <20120422231641.C774E8203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54637:0b78258294b6
Date: 2012-04-23 02:15 +0300
http://bitbucket.org/pypy/pypy/changeset/0b78258294b6/

Log:	fix translation (@Alex_Gaynor)

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -65,7 +65,7 @@
     check_num_args(space, w_args, 0)
     args_w = space.fixedview(w_args)
     res = generic_cpy_call(space, func_inquiry, w_self)
-    if res == -1:
+    if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception()
     return space.wrap(bool(res))
 

From noreply at buildbot.pypy.org  Mon Apr 23 01:16:43 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 23 Apr 2012 01:16:43 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge from default
Message-ID: <20120422231643.0E4988203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54638:b926e74b1d0e
Date: 2012-04-23 02:16 +0300
http://bitbucket.org/pypy/pypy/changeset/b926e74b1d0e/

Log:	merge from default

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -65,7 +65,7 @@
     check_num_args(space, w_args, 0)
     args_w = space.fixedview(w_args)
     res = generic_cpy_call(space, func_inquiry, w_self)
-    if res == -1:
+    if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception()
     return space.wrap(bool(res))
 

From noreply at buildbot.pypy.org  Mon Apr 23 01:37:48 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 23 Apr 2012 01:37:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix translation
Message-ID: <20120422233748.F0F898203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54639:88576b6607fa
Date: 2012-04-23 02:36 +0300
http://bitbucket.org/pypy/pypy/changeset/88576b6607fa/

Log:	fix translation

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -66,7 +66,7 @@
     args_w = space.fixedview(w_args)
     res = generic_cpy_call(space, func_inquiry, w_self)
     if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception()
+        space.fromcache(State).check_and_raise_exception(always=True)
     return space.wrap(bool(res))
 
 def wrap_getattr(space, w_self, w_args, func):

From noreply at buildbot.pypy.org  Mon Apr 23 01:37:50 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 23 Apr 2012 01:37:50 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge default into branch
Message-ID: <20120422233750.3D5D58203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54640:13e2ebeaf8d2
Date: 2012-04-23 02:37 +0300
http://bitbucket.org/pypy/pypy/changeset/13e2ebeaf8d2/

Log:	merge default into branch

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -66,7 +66,7 @@
     args_w = space.fixedview(w_args)
     res = generic_cpy_call(space, func_inquiry, w_self)
     if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception()
+        space.fromcache(State).check_and_raise_exception(always=True)
     return space.wrap(bool(res))
 
 def wrap_getattr(space, w_self, w_args, func):

From noreply at buildbot.pypy.org  Mon Apr 23 08:24:09 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 08:24:09 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix test.
Message-ID: <20120423062409.DE7118203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54641:bbacff2706c3
Date: 2012-04-23 07:46 +0200
http://bitbucket.org/pypy/pypy/changeset/bbacff2706c3/

Log:	Fix test.

diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -33,12 +33,6 @@
 
     threadnum = 0          # 0 = main thread; 1,2,3... = transactional threads
 
-    def setup_size_getter(self, getsize_fn):
-        self._getsize_fn = getsize_fn
-
-    def in_transaction(self):
-        return self.threadnum != 0
-
     def set_tls(self, tls, in_main_thread):
         assert lltype.typeOf(tls) == llmemory.Address
         assert tls
@@ -73,8 +67,9 @@
         assert obj not in tldict
         tldict[obj] = localobj
 
-    def tldict_enum(self, callback):
-        assert lltype.typeOf(callback) == self.CALLBACK_ENUM
+    def tldict_enum(self):
+        from pypy.rpython.memory.gc.stmtls import StmGCTLS
+        callback = StmGCTLS._stm_enum_callback
         tls = self.get_tls()
         for key, value in self._tldicts[self.threadnum].iteritems():
             callback(tls, key, value)
@@ -330,19 +325,24 @@
 
     def test_write_barrier_main_thread(self):
         t, t_adr = self.malloc(S, globl=False)
+        self.checkflags(t_adr, False, False)
         obj = self.gc.stm_writebarrier(t_adr)     # main thread, but not global
         assert obj == t_adr
+        self.checkflags(obj, False, False)
 
     def test_write_barrier_global(self):
+        # check that the main thread never makes a local copy of a global obj
         t, t_adr = self.malloc(S, globl=True)
-        obj = self.gc.stm_writebarrier(t_adr)     # global, even if main thread
-        assert obj != t_adr
+        self.checkflags(t_adr, True, False)
+        obj = self.gc.stm_writebarrier(t_adr)     # main thread, global:
+        assert obj == t_adr                       # doesn't make a copy
+        #self.checkflags(obj, False, False) -- LATER check that it becomes local
 
     def test_commit_transaction_empty(self):
         self.select_thread(1)
         s, s_adr = self.malloc(S)
         t, t_adr = self.malloc(S)
-        self.gc.commit_transaction()    # no roots
+        self.gc.stop_transaction()    # no roots
         main_tls = self.gc.main_thread_tls
         assert main_tls.nursery_free == main_tls.nursery_start   # empty
 
@@ -355,7 +355,7 @@
         assert sr_adr != tr_adr
         s_adr = self.gc.stm_writebarrier(t_adr)
         assert s_adr != t_adr
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
 
     def test_commit_local_obj_with_global_references(self):
         t, t_adr = self.malloc(S)
@@ -368,7 +368,7 @@
         sr2, sr2_adr = self.malloc(SR)
         sr.sr2 = sr2
         sr2.s1 = t
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
 
     def test_commit_with_ref_to_local_copy(self):
         tr, tr_adr = self.malloc(SR)
@@ -377,7 +377,7 @@
         assert sr_adr != tr_adr
         sr = llmemory.cast_adr_to_ptr(sr_adr, lltype.Ptr(SR))
         sr.sr2 = sr
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         assert sr.sr2 == tr
 
     def test_commit_transaction_no_references(self):
@@ -395,7 +395,7 @@
         assert main_tls.nursery_free != main_tls.nursery_start  # contains s
         old_value = main_tls.nursery_free
         #
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         #
         assert main_tls.nursery_free == old_value    # no new object
         assert s.b == 12345     # not updated by the GC code
@@ -451,7 +451,7 @@
         main_tls = self.gc.main_thread_tls
         old_value = main_tls.nursery_free
         #
-        self.gc.collector.commit_transaction()
+        self.gc.collector.stop_transaction()
         #
         assert main_tls.nursery_free - old_value == (
             self.gcsize(SR) + self.gcsize(SR) + self.gcsize(S))
@@ -478,7 +478,7 @@
 
     def test_do_get_size(self):
         s1, s1_adr = self.malloc(S)
-        assert (repr(self.gc.stm_operations._getsize_fn(s1_adr)) ==
+        assert (repr(self.gc._stm_getsize(s1_adr)) ==
                 repr(fake_get_size(s1_adr)))
 
     def test_id_of_global(self):
@@ -495,7 +495,7 @@
         i = self.gc.id(t)
         assert i == llmemory.cast_adr_to_int(s_adr)
         assert i == self.gc.id(s)
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         assert i == self.gc.id(s)
 
     def test_id_of_local_nonsurviving(self):
@@ -504,7 +504,7 @@
         i = self.gc.id(s)
         assert i != llmemory.cast_adr_to_int(s_adr)
         assert i == self.gc.id(s)
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
 
     def test_id_of_local_surviving(self):
         sr1, sr1_adr = self.malloc(SR)
@@ -522,7 +522,7 @@
                          llmemory.cast_adr_to_int(t2_adr),
                          llmemory.cast_adr_to_int(tr1_adr))
         assert i == self.gc.id(t2)
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         s2 = tr1.s1       # tr1 is a root, so not copied yet
         assert s2 and s2.a == 423 and s2._obj0 != t2._obj0
         assert self.gc.id(s2) == i
@@ -540,7 +540,7 @@
         i = self.gc.identityhash(t)
         assert i == mangle_hash(llmemory.cast_adr_to_int(s_adr))
         assert i == self.gc.identityhash(s)
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         assert i == self.gc.identityhash(s)
 
     def test_hash_of_local_nonsurviving(self):
@@ -549,7 +549,7 @@
         i = self.gc.identityhash(s)
         assert i != mangle_hash(llmemory.cast_adr_to_int(s_adr))
         assert i == self.gc.identityhash(s)
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
 
     def test_hash_of_local_surviving(self):
         sr1, sr1_adr = self.malloc(SR)
@@ -566,7 +566,7 @@
                          llmemory.cast_adr_to_int(t2_adr),
                          llmemory.cast_adr_to_int(tr1_adr)))
         assert i == self.gc.identityhash(t2)
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         s2 = tr1.s1       # tr1 is a root, so not copied yet
         assert s2 and s2.a == 424 and s2._obj0 != t2._obj0
         assert self.gc.identityhash(s2) == i
@@ -580,7 +580,7 @@
         twr1_adr = self.gc.stm_writebarrier(swr1_adr)
         twr1 = llmemory.cast_adr_to_ptr(twr1_adr, lltype.Ptr(SWR))
         twr1.wr = wr1
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         wr2 = twr1.wr      # twr1 is a root, so not copied yet
         assert wr2 and wr2._obj0 != wr1._obj0
         assert wr2.wadr == s2_adr   # survives
@@ -594,7 +594,7 @@
         twr1_adr = self.gc.stm_writebarrier(swr1_adr)
         twr1 = llmemory.cast_adr_to_ptr(twr1_adr, lltype.Ptr(SWR))
         twr1.wr = wr1
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         wr2 = twr1.wr      # twr1 is a root, so not copied yet
         assert wr2 and wr2._obj0 != wr1._obj0
         assert wr2.wadr == llmemory.NULL   # dies
@@ -613,7 +613,7 @@
         tr1 = llmemory.cast_adr_to_ptr(tr1_adr, lltype.Ptr(SR))
         tr1.s1 = t2
         t2.a = 4242
-        self.gc.commit_transaction()
+        self.gc.stop_transaction()
         wr2 = twr1.wr      # twr1 is a root, so not copied yet
         assert wr2 and wr2._obj0 != wr1._obj0
         assert wr2.wadr and wr2.wadr.ptr._obj0 != t2_adr.ptr._obj0   # survives

From noreply at buildbot.pypy.org  Mon Apr 23 08:53:53 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 08:53:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fixes:
Message-ID: <20120423065353.C54D18203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54642:73a34fe1ff67
Date: 2012-04-23 08:53 +0200
http://bitbucket.org/pypy/pypy/changeset/73a34fe1ff67/

Log:	Fixes:

	* cannot do bool(res) if res is an INT on 32-bit

	* a return value of -1 doesn't necessarily mean we have an exception

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -65,8 +65,9 @@
     check_num_args(space, w_args, 0)
     args_w = space.fixedview(w_args)
     res = generic_cpy_call(space, func_inquiry, w_self)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
     return space.wrap(bool(res))
 
 def wrap_getattr(space, w_self, w_args, func):

From noreply at buildbot.pypy.org  Mon Apr 23 10:28:01 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 10:28:01 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Re-add rstm.thread_id().
Message-ID: <20120423082801.1A37D8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54643:308b4d036534
Date: 2012-04-23 09:24 +0200
http://bitbucket.org/pypy/pypy/changeset/308b4d036534/

Log:	Re-add rstm.thread_id().

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -11,6 +11,8 @@
 
 NUM_THREADS_DEFAULT = 4     # XXX for now
 
+MAIN_THREAD_ID = 0
+
 
 class TransactionError(Exception):
     pass
@@ -34,6 +36,9 @@
 def in_transaction():
     return bool(stm_operations().in_transaction())
 
+def thread_id():
+    return stm_operations().thread_id()
+
 
 def run_all_transactions(initial_transaction,
                          num_threads = NUM_THREADS_DEFAULT):
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -81,6 +81,7 @@
 class InitialTransaction(rstm.Transaction):
     def run(self):
         ll_assert(self.retry_counter == 0, "no reason to abort-and-retry here")
+        ll_assert(rstm.thread_id() != 0, "thread_id == 0")
         scheduled = []
         for i in range(glob.NUM_THREADS):
             arg = Arg()
@@ -99,6 +100,7 @@
             glob.LENGTH = int(argv[2])
             if len(argv) > 3:
                 glob.USE_MEMORY = bool(int(argv[3]))
+    ll_assert(rstm.thread_id() == 0, "thread_id != 0")
     #
     rstm.run_all_transactions(InitialTransaction(),
                               num_threads=glob.NUM_THREADS)
diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py
--- a/pypy/translator/stm/test/test_ztranslated.py
+++ b/pypy/translator/stm/test/test_ztranslated.py
@@ -6,7 +6,7 @@
 
 class TestSTMTranslated(CompiledSTMTests):
 
-    def test_hello_world(self):
+    def test_targetdemo(self):
         t, cbuilder = self.compile(targetdemo.entry_point)
         data, dataerr = cbuilder.cmdexec('4 5000', err=True)
         assert 'check ok!' in data

From noreply at buildbot.pypy.org  Mon Apr 23 10:28:02 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 10:28:02 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix the FakeStmOperations class.
Message-ID: <20120423082802.5ACEC8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54644:ac2193fe0e1c
Date: 2012-04-23 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/ac2193fe0e1c/

Log:	Fix the FakeStmOperations class.

diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -11,6 +11,12 @@
     def in_transaction(self):
         return self._in_transaction
 
+    def thread_id(self):
+        if self._in_transaction:
+            return -12345    # random number
+        else:
+            return 0
+
     def _add(self, transactionptr):
         r = random.random()
         assert r not in self._pending    # very bad luck if it is
@@ -19,16 +25,20 @@
     def run_all_transactions(self, initial_transaction_ptr, num_threads=4):
         self._pending = {}
         self._add(initial_transaction_ptr)
-        while self._pending:
-            r, transactionptr = self._pending.popitem()
-            transaction = self.cast_voidp_to_transaction(transactionptr)
-            transaction._next_transaction = None
-            nextptr = rstm._stm_run_transaction(transactionptr, 0)
-            next = self.cast_voidp_to_transaction(nextptr)
-            while next is not None:
-                self._add(self.cast_transaction_to_voidp(next))
-                next = next._next_transaction
-        del self._pending
+        self._in_transaction = True
+        try:
+            while self._pending:
+                r, transactionptr = self._pending.popitem()
+                transaction = self.cast_voidp_to_transaction(transactionptr)
+                transaction._next_transaction = None
+                nextptr = rstm._stm_run_transaction(transactionptr, 0)
+                next = self.cast_voidp_to_transaction(nextptr)
+                while next is not None:
+                    self._add(self.cast_transaction_to_voidp(next))
+                    next = next._next_transaction
+        finally:
+            self._in_transaction = False
+            del self._pending
 
     def cast_transaction_to_voidp(self, transaction):
         if transaction is None:
@@ -53,14 +63,19 @@
 def test_in_transaction():
     res = rstm.in_transaction()
     assert res is False
+    res = rstm.thread_id()
+    assert res == 0
 
 def test_run_all_transactions_minimal():
     seen = []
     class Empty(rstm.Transaction):
         def run(self):
-            seen.append(42)
+            res = rstm.in_transaction()
+            seen.append(res is True)
+            res = rstm.thread_id()
+            seen.append(res != 0)
     rstm.run_all_transactions(Empty())
-    assert seen == [42]
+    assert seen == [True, True]
 
 def test_run_all_transactions_recursive():
     seen = []

From noreply at buildbot.pypy.org  Mon Apr 23 10:28:03 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 10:28:03 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Start to fix the 'transaction' module,
 mostly by killing code.
Message-ID: <20120423082803.9FE308203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54645:266863eb0af8
Date: 2012-04-23 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/266863eb0af8/

Log:	Start to fix the 'transaction' module, mostly by killing code.

diff --git a/pypy/module/transaction/__init__.py b/pypy/module/transaction/__init__.py
--- a/pypy/module/transaction/__init__.py
+++ b/pypy/module/transaction/__init__.py
@@ -9,8 +9,8 @@
         'set_num_threads': 'interp_transaction.set_num_threads',
         'add': 'interp_transaction.add',
         'run': 'interp_transaction.run',
-        'add_epoll': 'interp_epoll.add_epoll',        # xxx linux only
-        'remove_epoll': 'interp_epoll.remove_epoll',  # xxx linux only
+        #'add_epoll': 'interp_epoll.add_epoll',        # xxx linux only
+        #'remove_epoll': 'interp_epoll.remove_epoll',  # xxx linux only
         'local': 'interp_local.W_Local',
     }
 
@@ -22,9 +22,9 @@
         "NOT_RPYTHON: patches space.threadlocals to use real threadlocals"
         from pypy.module.transaction import interp_transaction
         MixedModule.__init__(self, space, *args)
-        interp_transaction.state.initialize(space)
-        space.threadlocals = interp_transaction.state
+        space.threadlocals = interp_transaction.getstate(space)
 
     def startup(self, space):
         from pypy.module.transaction import interp_transaction
-        interp_transaction.state.startup(space, space.wrap(self))
+        state = interp_transaction.getstate(space)
+        state.startup(space.wrap(self))
diff --git a/pypy/module/transaction/fifo.py b/pypy/module/transaction/fifo.py
deleted file mode 100644
--- a/pypy/module/transaction/fifo.py
+++ /dev/null
@@ -1,39 +0,0 @@
-
-class Fifo(object):
-    def __init__(self):
-        self.first = None
-        self.last = None
-
-    clear = __init__
-
-    def append(self, newitem):
-        newitem.next = None
-        if self.last is None:
-            self.first = newitem
-        else:
-            self.last.next = newitem
-        self.last = newitem
-
-    def is_empty(self):
-        assert (self.first is None) == (self.last is None)
-        return self.first is None
-
-    def is_of_length_1(self):
-        return self.first is not None and self.first is self.last
-
-    def popleft(self):
-        item = self.first
-        self.first = item.next
-        if self.first is None:
-            self.last = None
-        return item
-
-    def steal(self, otherfifo):
-        if otherfifo.last is not None:
-            if self.last is None:
-                self.first = otherfifo.first
-            else:
-                self.last.next = otherfifo.first
-            self.last = otherfifo.last
-            otherfifo.first = None
-            otherfifo.last = None
diff --git a/pypy/module/transaction/interp_local.py b/pypy/module/transaction/interp_local.py
--- a/pypy/module/transaction/interp_local.py
+++ b/pypy/module/transaction/interp_local.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty,
     descr_get_dict)
-from pypy.module.transaction.interp_transaction import state
+#from pypy.module.transaction.interp_transaction import state
 
 
 class W_Local(Wrappable):
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -1,72 +1,36 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.transaction import threadintf
-from pypy.module.transaction.fifo import Fifo
-from pypy.rlib import rstm, rgc
-from pypy.rlib.debug import ll_assert
-
-
-NUM_THREADS_DEFAULT = 4     # by default
-
-MAIN_THREAD_ID = 0
+from pypy.interpreter.executioncontext import ExecutionContext
+from pypy.rlib import rstm
 
 
 class State(object):
-    # Warning: this is the class of a singleton.  It Must Not Be Used
-    # Inside Any Transaction!!!!  This code is written with the
-    # expectation that it is really a global singleton used for
-    # synchronization between transactions.  So if the STM logic kicks
-    # in and makes local copies of it, we loose.
-    #
-    # Instead, we have a TransactionalState instance that can be
-    # written to by transactions (possibly causing conflicts).
+    """The shared, global state.  Warning, writes to it cause conflicts.
+    XXX fix me to somehow avoid conflicts at the beginning due to setvalue()
+    """
 
-    def initialize(self, space):
+    def __init__(self, space):
         self.space = space
+        self.num_threads = rstm.NUM_THREADS_DEFAULT
         self.running = False
-        self.num_threads = NUM_THREADS_DEFAULT
-        self.transactionalstate = None
-        #
         self.w_error = None
         self.threadobjs = {}      # empty during translation
         self.threadnums = {}      # empty during translation
         self.epolls = {}
-        self.pending = Fifo()
-        self._freeze_()
+        self.pending_before_run = []
 
-    def _freeze_(self):
-        self.ll_lock = threadintf.null_ll_lock
-        self.ll_no_tasks_pending_lock = threadintf.null_ll_lock
-        self.ll_unfinished_lock = threadintf.null_ll_lock
-        self.ll_not_ready_to_start_lock = threadintf.null_ll_lock
-        self.threadobjs.clear()
-        self.threadnums.clear()
-        self.epolls.clear()
-        self.pending.clear()
-        return False
-
-    def startup(self, space, w_module):
-        assert space is self.space
+    def startup(self, w_module):
         if w_module is not None:     # for tests
+            space = self.space
             self.w_error = space.getattr(w_module,
                                          space.wrap('TransactionError'))
-        self.ll_lock = threadintf.allocate_lock()
-        self.ll_no_tasks_pending_lock = threadintf.allocate_lock()
-        self.ll_unfinished_lock = threadintf.allocate_lock()
-        self.lock_unfinished()
-        self.ll_not_ready_to_start_lock = threadintf.allocate_lock()
-        self.startup_run()
-
-    def startup_run(self):
-        # this is called at the start of run() too, in order to make
-        # test_checkmodule happy
         main_ec = self.space.getexecutioncontext()    # create it if needed
-        main_ec._transaction_pending = self.pending
+        main_ec._transaction_pending = self.pending_before_run
 
     def add_thread(self, id, ec):
         # register a new transaction thread
         assert id not in self.threadobjs
-        ec._transaction_pending = Fifo()
+        ec._transaction_pending = []
         self.threadobjs[id] = ec
         self.threadnums[id] = len(self.threadnums)
 
@@ -84,9 +48,13 @@
 
     def setvalue(self, value):
         id = rstm.thread_id()
-        assert id == MAIN_THREAD_ID   # should not be used from a transaction
-        self.threadobjs[id] = value
-        self.threadnums = {id: 0}
+        if id == rstm.MAIN_THREAD_ID:
+            assert len(self.threadobjs) == 0
+            assert len(self.threadnums) == 0
+            self.threadobjs[id] = value
+            self.threadnums[id] = 0
+        else:
+            self.add_thread(id, value)
 
     def getmainthreadvalue(self):
         return self.threadobjs.get(MAIN_THREAD_ID, None)
@@ -98,7 +66,9 @@
         for id in self.threadobjs.keys():
             if id != MAIN_THREAD_ID:
                 del self.threadobjs[id]
-        self.threadnums = {MAIN_THREAD_ID: 0}
+        for id in self.threadnums.keys():
+            if id != MAIN_THREAD_ID:
+                del self.threadnums[id]
         self.epolls.clear()
 
     def get_thread_number(self):
@@ -108,262 +78,89 @@
     def get_total_number_of_threads(self):
         return 1 + self.num_threads
 
-    # ----------
-
     def set_num_threads(self, num):
         if self.running:
             space = self.space
-            raise OperationError(space.w_ValueError,
+            raise OperationError(self.w_error,
                                  space.wrap("cannot change the number of "
                                             "threads when transaction.run() "
                                             "is active"))
         self.num_threads = num
 
-    def lock(self):
-        """Acquire the main lock.  This plays a role similar to the GIL
-        in that it must be acquired in order to have the _run_thread()
-        code execute; but it is released around every execution of a
-        transaction."""
-        threadintf.acquire(self.ll_lock, True)
 
-    def unlock(self):
-        """Release the main lock."""
-        threadintf.release(self.ll_lock)
-
-    def lock_no_tasks_pending(self):
-        """This lock is acquired when state.pending.is_empty()."""
-        threadintf.acquire(self.ll_no_tasks_pending_lock, True)
-
-    def unlock_no_tasks_pending(self):
-        """Release the ll_no_tasks_pending_lock."""
-        threadintf.release(self.ll_no_tasks_pending_lock)
-
-    def is_locked_no_tasks_pending(self):
-        """Test ll_no_tasks_pending_lock for debugging."""
-        just_locked = threadintf.acquire(self.ll_no_tasks_pending_lock, False)
-        if just_locked:
-            threadintf.release(self.ll_no_tasks_pending_lock)
-        return not just_locked
-
-    def lock_unfinished(self):
-        """This lock is normally acquired.  It is released when all threads
-        are done."""
-        threadintf.acquire(self.ll_unfinished_lock, True)
-
-    def unlock_unfinished(self):
-        """Release ll_unfinished_lock."""
-        threadintf.release(self.ll_unfinished_lock)
-
-    def lock_not_ready_to_start(self):
-        """This lock is acquired when the threads are still starting.
-        It is released when all threads are past their initialization."""
-        threadintf.acquire(self.ll_not_ready_to_start_lock, True)
-
-    def unlock_not_ready_to_start(self):
-        """Release ll_not_ready_to_start_lock."""
-        threadintf.release(self.ll_not_ready_to_start_lock)
-
-
-class TransactionalState(object):
-
-    def __init__(self):
-        self._reraise_exception = None
-
-    def has_exception(self):
-        return self._reraise_exception is not None
-
-    def must_reraise_exception(self, reraise_callback):
-        self._reraise_exception = reraise_callback
-
-    def close_exceptions(self):
-        if self._reraise_exception is not None:
-            self._reraise_exception(self)
-
-
-state = State()
+def getstate(space):
+    return space.fromcache(State)
 
 
 @unwrap_spec(num=int)
 def set_num_threads(space, num):
     if num < 1:
         num = 1
-    state.set_num_threads(num)
+    getstate(space).set_num_threads(num)
 
 
-class AbstractPending(object):
+class SpaceTransaction(rstm.Transaction):
+
+    def __init__(self, space, w_callback, args):
+        self.space = space
+        self.state = getstate(space)
+        self.w_callback = w_callback
+        self.args = args
 
     def register(self):
-        """Register this AbstractPending instance in the pending list
+        """Register this SpaceTransaction instance in the pending list
         belonging to the current thread.  If called from the main
         thread, it is the global list.  If called from a transaction,
         it is a thread-local list that will be merged with the global
-        list when the transaction is done."""
-        ec = state.getvalue()
+        list when the transaction is done.
+        NOTE: never register() the same instance multiple times.
+        """
+        ec = self.state.getvalue()
         ec._transaction_pending.append(self)
 
     def run(self):
-        # may also be overridden
-        rstm.perform_transaction(AbstractPending._run_in_transaction,
-                                 AbstractPending, self)
+        if self.retry_counter > 0:
+            self.register() # retrying: will be done later, try others first
+            return
+        #
+        ec = self.space.getexecutioncontext()    # create it if needed
+        assert len(ec._transaction_pending) == 0
+        #
+        self.space.call_args(self.w_callback, self.args)
+        #
+        result = ec._transaction_pending
+        ec._transaction_pending = []
+        return result
 
-    @staticmethod
-    def _run_in_transaction(pending, retry_counter):
-        if retry_counter > 0:
-            pending.register() # retrying: will be done later, try others first
-            return
-        ts = state.transactionalstate
-        if ts.has_exception():
-            return   # return early if there is already an exception to reraise
-        try:
-            pending.run_in_transaction(state.space)
-        except Exception, e:
-            ts = state.transactionalstate
-            ts.got_exception_applevel = e
-            ts.must_reraise_exception(_reraise_from_applevel)
 
+class InitialTransaction(rstm.Transaction):
 
-class Pending(AbstractPending):
-    def __init__(self, w_callback, args):
-        self.w_callback = w_callback
-        self.args = args
+    def __init__(self, state):
+        self.state = state
 
-    def run_in_transaction(self, space):
-        space.call_args(self.w_callback, self.args)
-
-
-def _reraise_from_applevel(transactionalstate):
-    e = transactionalstate.got_exception_applevel
-    transactionalstate.got_exception_applevel = None
-    raise e
+    def run(self):
+        # initially: return the list of already-added transactions as
+        # the list of transactions to run next, and clear it
+        result = self.state.pending_before_run[:]
+        del self.state.pending_before_run[:]
+        return result
 
 
 def add(space, w_callback, __args__):
-    Pending(w_callback, __args__).register()
-
-
-def _add_list(new_pending_list):
-    if new_pending_list.is_empty():
-        return
-    was_empty = state.pending.is_empty()
-    state.pending.steal(new_pending_list)
-    if was_empty:
-        state.unlock_no_tasks_pending()
-
-
-def _setup_thread(_, retry_counter):
-    """Setup a thread.  Run as a transaction because it allocates."""
-    assert state.running
-    my_thread_id = rstm.thread_id()
-    my_ec = state.space.createexecutioncontext()
-    state.add_thread(my_thread_id, my_ec)
-
-
-def _run_thread():
-    """The main function running one of the threads."""
-    # Note that we cannot allocate any object here outside a transaction,
-    # so we need to be very careful.
-    rstm.descriptor_init()
-    state.lock()
-    rstm.perform_transaction(_setup_thread, AbstractPending, None)
-    # wait until all threads reach this point to continue
-    if state.num_waiting_threads + 1 < state.num_threads:
-        state.num_waiting_threads += 1
-        state.unlock()
-        state.lock_not_ready_to_start()
-        state.lock()
-    else:
-        state.num_waiting_threads = 0
-    state.unlock_not_ready_to_start()
-    #
-    my_transactions_pending = state.getvalue()._transaction_pending
-    #
-    while True:
-        if state.pending.is_empty():
-            ll_assert(state.is_locked_no_tasks_pending(),
-                      "inconsistently unlocked no_tasks_pending")
-            state.num_waiting_threads += 1
-            if state.num_waiting_threads == state.num_threads:
-                state.finished = True
-                state.unlock_no_tasks_pending()
-            state.unlock()
-            #
-            state.lock_no_tasks_pending()
-            state.unlock_no_tasks_pending()
-            #
-            state.lock()
-            state.num_waiting_threads -= 1
-            if state.finished:
-                break
-        else:
-            pending = state.pending.popleft()
-            if state.pending.is_empty():
-                state.lock_no_tasks_pending()
-            state.unlock()
-            #
-            while True:
-                pending.run()
-                # for now, always break out of this loop, unless
-                # 'my_transactions_pending' contains precisely one item
-                if not my_transactions_pending.is_of_length_1():
-                    break
-                pending = my_transactions_pending.popleft()
-            #
-            state.lock()
-            _add_list(my_transactions_pending)
-    #
-    if state.num_waiting_threads == 0:    # only the last thread to leave
-        state.unlock_unfinished()
-    state.unlock()
-    rstm.descriptor_done()
-
-
- at rgc.no_collect
-def _run():
-    # --- start the threads --- don't use the GC here any more! ---
-    #
-    for i in range(state.num_threads):
-        threadintf.start_new_thread(_run_thread)
-    #
-    state.lock_unfinished()  # wait for all threads to finish
-    #
-    # --- done, we can use the GC again ---
+    transaction = SpaceTransaction(space, w_callback, __args__)
+    transaction.register()
 
 
 def run(space):
+    state = getstate(space)
     if state.running:
         raise OperationError(
             state.w_error,
             space.wrap("recursive invocation of transaction.run()"))
-    state.startup_run()
-    assert rstm.thread_id() == MAIN_THREAD_ID
-    assert not state.is_locked_no_tasks_pending()
-    if state.pending.is_empty():
-        return
-    #
-    # 'num_waiting_threads' is the number of threads that are currently
-    # waiting for more work to do.  When it becomes equal to
-    # 'num_threads' then we are done: we set 'finished' to True and this
-    # causes all threads to leave.  Only accessed during a
-    # 'state.lock'-protected region.
-    state.num_waiting_threads = 0
-    state.finished = False
-    #
-    state.transactionalstate = TransactionalState()
-    state.lock_not_ready_to_start()
     state.running = True
-    #
-    # start the threads and wait for all of them to finish
-    rstm.enter_transactional_mode()
-    _run()
-    rstm.leave_transactional_mode()
-    #
-    assert state.num_waiting_threads == 0
-    assert state.pending.is_empty()
-    assert not state.is_locked_no_tasks_pending()
-    state.clear_all_values_apart_from_main()
-    state.running = False
-    ts = state.transactionalstate
-    state.transactionalstate = None
-    #
-    # now re-raise the exception that we got in a transaction
-    ts.close_exceptions()
+    try:
+        rstm.run_all_transactions(InitialTransaction(state),
+                                  num_threads = state.num_threads)
+    finally:
+        state.running = False
+        assert len(state.pending_before_run) == 0
diff --git a/pypy/module/transaction/threadintf.py b/pypy/module/transaction/threadintf.py
deleted file mode 100644
--- a/pypy/module/transaction/threadintf.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import thread
-from pypy.module.thread import ll_thread
-from pypy.rlib.objectmodel import we_are_translated
-from pypy.rpython.annlowlevel import llhelper
-from pypy.rlib.debug import fatalerror
-
-
-null_ll_lock = ll_thread.null_ll_lock
-
-def allocate_lock():
-    if we_are_translated():
-        return ll_thread.allocate_ll_lock()
-    else:
-        return thread.allocate_lock()
-
-def acquire(lock, wait):
-    if we_are_translated():
-        return ll_thread.acquire_NOAUTO(lock, wait)
-    else:
-        return lock.acquire(wait)
-
-def release(lock):
-    if we_are_translated():
-        ll_thread.release_NOAUTO(lock)
-    else:
-        lock.release()
-
-def start_new_thread(callback):
-    if we_are_translated():
-        llcallback = llhelper(ll_thread.CALLBACK, callback)
-        ident = ll_thread.c_thread_start_NOGIL(llcallback)
-        if ident == -1:
-            fatalerror("cannot start thread")
-    else:
-        thread.start_new_thread(callback, ())

From noreply at buildbot.pypy.org  Mon Apr 23 10:28:04 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 10:28:04 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix test
Message-ID: <20120423082804.DC7158203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54646:741a4cb8b6c6
Date: 2012-04-23 10:18 +0200
http://bitbucket.org/pypy/pypy/changeset/741a4cb8b6c6/

Log:	Fix test

diff --git a/pypy/module/transaction/test/test_interp_transaction.py b/pypy/module/transaction/test/test_interp_transaction.py
--- a/pypy/module/transaction/test/test_interp_transaction.py
+++ b/pypy/module/transaction/test/test_interp_transaction.py
@@ -3,32 +3,28 @@
 
 
 class FakeSpace:
-    def getbuiltinmodule(self, name):
-        assert name == 'transaction'
-        return 'transaction module'
-    def getattr(self, w_obj, w_name):
-        assert w_obj == 'transaction module'
-        return 'some stuff from the transaction module'
-    def wrap(self, x):
-        return 'wrapped stuff'
+    def __init__(self):
+        self._spacecache = {}
     def getexecutioncontext(self):
-        ec = interp_transaction.state.getvalue()
+        state = interp_transaction.getstate(self)
+        ec = state.getvalue()
         if ec is None:
-            ec = self.createexecutioncontext()
-            interp_transaction.state.setvalue(ec)
+            ec = FakeEC()
+            state.setvalue(ec)
         return ec
-    def createexecutioncontext(self):
-        return FakeEC()
     def call_args(self, w_callback, args):
         w_callback(*args)
+    def fromcache(self, Cls):
+        if Cls not in self._spacecache:
+            self._spacecache[Cls] = Cls(self)
+        return self._spacecache[Cls]
 
 class FakeEC:
     pass
 
 def make_fake_space():
     space = FakeSpace()
-    interp_transaction.state.initialize(space)
-    interp_transaction.state.startup(space, None)
+    interp_transaction.getstate(space).startup(None)
     return space
 
 

From noreply at buildbot.pypy.org  Mon Apr 23 10:28:06 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 10:28:06 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Kill test
Message-ID: <20120423082806.268388203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54647:328a42a15741
Date: 2012-04-23 10:18 +0200
http://bitbucket.org/pypy/pypy/changeset/328a42a15741/

Log:	Kill test

diff --git a/pypy/module/transaction/test/test_fifo.py b/pypy/module/transaction/test/test_fifo.py
deleted file mode 100644
--- a/pypy/module/transaction/test/test_fifo.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from pypy.module.transaction.fifo import Fifo
-
-class Item:
-    def __init__(self, value):
-        self.value = value
-
-def test_one_item():
-    f = Fifo()
-    assert f.is_empty()
-    f.append(Item(123))
-    assert not f.is_empty()
-    item = f.popleft()
-    assert f.is_empty()
-    assert item.value == 123
-
-def test_three_items():
-    f = Fifo()
-    for i in [10, 20, 30]:
-        f.append(Item(i))
-        assert not f.is_empty()
-    for i in [10, 20, 30]:
-        assert not f.is_empty()
-        item = f.popleft()
-        assert item.value == i
-    assert f.is_empty()
-
-def test_steal():
-    for n1 in range(3):
-        for n2 in range(3):
-            f1 = Fifo()
-            f2 = Fifo()
-            for i in range(n1): f1.append(Item(10 + i))
-            for i in range(n2): f2.append(Item(20 + i))
-            f1.steal(f2)
-            assert f2.is_empty()
-            for x in range(10, 10+n1) + range(20, 20+n2):
-                assert not f1.is_empty()
-                item = f1.popleft()
-                assert item.value == x
-            assert f1.is_empty()

From noreply at buildbot.pypy.org  Mon Apr 23 10:28:07 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 10:28:07 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix local.
Message-ID: <20120423082807.6950B8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54648:c76e8c2a1758
Date: 2012-04-23 10:23 +0200
http://bitbucket.org/pypy/pypy/changeset/c76e8c2a1758/

Log:	Fix local.

diff --git a/pypy/module/transaction/interp_local.py b/pypy/module/transaction/interp_local.py
--- a/pypy/module/transaction/interp_local.py
+++ b/pypy/module/transaction/interp_local.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.typedef import (TypeDef, interp2app, GetSetProperty,
     descr_get_dict)
-#from pypy.module.transaction.interp_transaction import state
+from pypy.module.transaction.interp_transaction import getstate
 
 
 class W_Local(Wrappable):
@@ -18,27 +18,29 @@
     """
 
     def __init__(self, space):
+        self.state = getstate(space)
         self.dicts = []
-        self._update_dicts(space)
+        self._update_dicts()
         # unless we call transaction.set_num_threads() afterwards, this
         # 'local' object is now initialized with the correct number of
         # dictionaries, to avoid conflicts later if _update_dicts() is
         # called in a transaction.
 
-    def _update_dicts(self, space):
+    def _update_dicts(self):
+        state = self.state
         new = state.get_total_number_of_threads() - len(self.dicts)
         if new <= 0:
             return
         # update the list without appending to it (to keep it non-resizable)
-        self.dicts = self.dicts + [space.newdict(instance=True)
+        self.dicts = self.dicts + [state.space.newdict(instance=True)
                                    for i in range(new)]
 
     def getdict(self, space):
-        n = state.get_thread_number()
+        n = self.state.get_thread_number()
         try:
             return self.dicts[n]
         except IndexError:
-            self._update_dicts(space)
+            self._update_dicts()
             assert n < len(self.dicts)
             return self.dicts[n]
 
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -6,16 +6,14 @@
 
 class FakeStmOperations:
     _in_transaction = 0
+    _thread_id = 0
     _mapping = {}
 
     def in_transaction(self):
         return self._in_transaction
 
     def thread_id(self):
-        if self._in_transaction:
-            return -12345    # random number
-        else:
-            return 0
+        return self._thread_id
 
     def _add(self, transactionptr):
         r = random.random()
@@ -25,9 +23,12 @@
     def run_all_transactions(self, initial_transaction_ptr, num_threads=4):
         self._pending = {}
         self._add(initial_transaction_ptr)
+        thread_ids = [-10000 - 123 * i for i in range(num_threads)]  # random
         self._in_transaction = True
         try:
             while self._pending:
+                self._thread_id = thread_ids.pop(0)
+                thread_ids.append(self._thread_id)
                 r, transactionptr = self._pending.popitem()
                 transaction = self.cast_voidp_to_transaction(transactionptr)
                 transaction._next_transaction = None
@@ -38,6 +39,7 @@
                     next = next._next_transaction
         finally:
             self._in_transaction = False
+            self._thread_id = 0
             del self._pending
 
     def cast_transaction_to_voidp(self, transaction):

From noreply at buildbot.pypy.org  Mon Apr 23 10:28:38 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 10:28:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: yet another passing test
Message-ID: <20120423082838.6103A8203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54649:e8a35d94d312
Date: 2012-04-23 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e8a35d94d312/

Log:	yet another passing test

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1836,6 +1836,13 @@
         a[a & 1 == 0] = 15
         assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
 
+    def test_array_indexing_bool_setitem_2(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a = a[::2]
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 5], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])

From noreply at buildbot.pypy.org  Mon Apr 23 10:28:39 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 10:28:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120423082839.C25C28203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54650:92a93f454539
Date: 2012-04-23 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/92a93f454539/

Log:	merge

diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
     space.call_method(w_list, "reverse")
     return 0
 
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+    """Return a list of the objects in list containing the objects between low
+    and high.  Return NULL and set an exception if unsuccessful.  Analogous
+    to list[low:high].  Negative indices, as when slicing from Python, are not
+    supported."""
+    w_start = space.wrap(low)
+    w_stop = space.wrap(high)
+    return space.getslice(w_list, w_start, w_stop)
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyList_SetSlice(space, w_list, low, high, w_sequence):
     """Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
     This is the equivalent of the Python expression hash(o)."""
     return space.int_w(space.hash(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+    """Set a TypeError indicating that type(o) is not hashable and return -1.
+    This function receives special treatment when stored in a tp_hash slot,
+    allowing a type to explicitly indicate to the interpreter that it is not
+    hashable.
+    """
+    raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Dir(space, w_o):
     """This is equivalent to the Python expression dir(o), returning a (possibly
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
-    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc)
@@ -60,6 +60,16 @@
     args_w = space.fixedview(w_args)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
 
+def wrap_inquirypred(space, w_self, w_args, func):
+    func_inquiry = rffi.cast(inquiry, func)
+    check_num_args(space, w_args, 0)
+    args_w = space.fixedview(w_args)
+    res = generic_cpy_call(space, func_inquiry, w_self)
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
+    return space.wrap(bool(res))
+
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
     check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
         w_errors = space.wrap(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+    """Decode a string object by passing it to the codec registered
+    for encoding and return the result as Python object. encoding and
+    errors have the same meaning as the parameters of the same name in
+    the string encode() method.  The codec to be used is looked up
+    using the Python codec registry. Return NULL if an exception was
+    raised by the codec.
+
+    This function is not available in 3.x and does not have a PyBytes alias."""
+    if not PyString_Check(space, w_str):
+        PyErr_BadArgument(space)
+
+    w_encoding = w_errors = space.w_None
+    if encoding:
+        w_encoding = space.wrap(rffi.charp2str(encoding))
+    if errors:
+        w_errors = space.wrap(rffi.charp2str(errors))
+    return space.call_method(w_str, "decode", w_encoding, w_errors)
+
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
-    """Return a list of the objects in list containing the objects between low
-    and high.  Return NULL and set an exception if unsuccessful.  Analogous
-    to list[low:high].  Negative indices, as when slicing from Python, are not
-    supported.
-
-    This function used an int for low and high. This might
-    require changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([Py_ssize_t], PyObject)
 def PyLong_FromSsize_t(space, v):
     """Return a new PyLongObject object from a C Py_ssize_t, or
@@ -1606,15 +1595,6 @@
     for PyObject_Str()."""
     raise NotImplementedError
 
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
-    """Set a TypeError indicating that type(o) is not hashable and return -1.
-    This function receives special treatment when stored in a tp_hash slot,
-    allowing a type to explicitly indicate to the interpreter that it is not
-    hashable.
-    """
-    raise NotImplementedError
-
 @cpython_api([], PyFrameObject)
 def PyEval_GetFrame(space):
     """Return the current thread state's frame, which is NULL if no frame is
@@ -1737,17 +1717,6 @@
     changes in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
-    """Decode a string object by passing it to the codec registered for encoding and
-    return the result as Python object. encoding and errors have the same
-    meaning as the parameters of the same name in the string encode() method.
-    The codec to be used is looked up using the Python codec registry. Return NULL
-    if an exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Encode(space, s, size, encoding, errors):
     """Encode the char buffer of the given size by passing it to the codec
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
         w_t = api.PyList_AsTuple(w_l)
         assert space.unwrap(w_t) == (3, 2, 1)
 
+    def test_list_getslice(self, space, api):
+        w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+        w_s = api.PyList_GetSlice(w_l, 1, 5)
+        assert space.unwrap(w_s) == [2, 1]
+
 class AppTestListObject(AppTestCpythonExtensionBase):
     def test_listobject(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
+    def test_AsDecodedObject(self, space, api):
+        w_str = space.wrap('caf\xe9')
+        encoding = rffi.str2charp("latin-1")
+        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        rffi.free_charp(encoding)
+        assert space.unwrap(w_res) == u"caf\xe9"
+
     def test_eq(self, space, api):
         assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
         assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
+        
+    def test_bool(self):
+        module = self.import_extension('foo', [
+            ("newInt", "METH_VARARGS",
+             """
+                IntLikeObject *intObj;
+                long intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "i", &intval))
+                    return NULL;
+
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+                intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+                if (!intObj) {
+                    return NULL;
+                }
+
+                intObj->value = intval;
+                return (PyObject *)intObj;
+             """)],
+            """
+            typedef struct
+            {
+                PyObject_HEAD
+                int value;
+            } IntLikeObject;
+
+            static int
+            intlike_nb_nonzero(IntLikeObject *v)
+            {
+                if (v->value == -42) {
+                    PyErr_SetNone(PyExc_ValueError);
+                    return -1;
+                }
+                return v->value;
+            }
+
+            PyTypeObject IntLike_Type = {
+                PyObject_HEAD_INIT(0)
+                /*ob_size*/             0,
+                /*tp_name*/             "IntLike",
+                /*tp_basicsize*/        sizeof(IntLikeObject),
+            };
+            static PyNumberMethods intlike_as_number;
+            """)
+        assert not bool(module.newInt(0))
+        assert bool(module.newInt(1))
+        assert bool(module.newInt(-1))
+        raises(ValueError, bool, module.newInt(-42))

From noreply at buildbot.pypy.org  Mon Apr 23 10:31:13 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon, 23 Apr 2012 10:31:13 +0200 (CEST)
Subject: [pypy-commit] pypy pytest: merge from default
Message-ID: <20120423083113.7E6AF8203C@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: pytest
Changeset: r54651:f61b3f64c7ab
Date: 2012-04-23 10:14 +0200
http://bitbucket.org/pypy/pypy/changeset/f61b3f64c7ab/

Log:	merge from default

diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/py/bin/_findpy.py b/py/bin/_findpy.py
new file mode 100644
--- /dev/null
+++ b/py/bin/_findpy.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+#
+# find and import a version of 'py'
+#
+import sys
+import os
+from os.path import dirname as opd, exists, join, basename, abspath
+
+def searchpy(current):
+    while 1:
+        last = current
+        initpy = join(current, '__init__.py')
+        if not exists(initpy):
+            pydir = join(current, 'py')
+            # recognize py-package and ensure it is importable
+            if exists(pydir) and exists(join(pydir, '__init__.py')):
+                #for p in sys.path:
+                #    if p == current:
+                #        return True
+                if current != sys.path[0]:  # if we are already first, then ok
+                    sys.stderr.write("inserting into sys.path: %s\n" % current)
+                    sys.path.insert(0, current)
+                return True
+        current = opd(current)
+        if last == current:
+            return False
+
+if not searchpy(abspath(os.curdir)):
+    if not searchpy(opd(abspath(sys.argv[0]))):
+        if not searchpy(opd(__file__)):
+            pass # let's hope it is just on sys.path
+
+import py
+import pytest
+
+if __name__ == '__main__':
+    print ("py lib is at %s" % py.__file__)
diff --git a/py/bin/py.test b/py/bin/py.test
new file mode 100755
--- /dev/null
+++ b/py/bin/py.test
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from _findpy import pytest
+raise SystemExit(pytest.main())
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -483,7 +483,7 @@
                 return a_str.strip(' ')
             elif n == 1:
                 return a_str.rstrip(' ')
-            else: 
+            else:
                 return a_str.lstrip(' ')
         s = a.build_types(f, [int, annmodel.SomeString(no_nul=True)])
         assert s.no_nul
@@ -3737,6 +3737,25 @@
         s = a.build_types(f, [int])
         assert s.listdef.listitem.range_step == 0
 
+    def test_specialize_arg_memo(self):
+        @objectmodel.specialize.memo()
+        def g(n):
+            return n
+        @objectmodel.specialize.arg(0)
+        def f(i):
+            return g(i)
+        def main(i):
+            if i == 2:
+                return f(2)
+            elif i == 3:
+                return f(3)
+            else:
+                raise NotImplementedError
+
+        a = self.RPythonAnnotator()
+        s = a.build_types(main, [int])
+        assert isinstance(s, annmodel.SomeInteger)
+
 
 def g(n):
     return [0,1,2,n]
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -158,6 +158,12 @@
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-1.html
 .. __: http://morepypy.blogspot.com/2008/02/python-finalizers-semantics-part-2.html
 
+Note that this difference might show up indirectly in some cases.  For
+example, a generator left pending in the middle is --- again ---
+garbage-collected later in PyPy than in CPython.  You can see the
+difference if the ``yield`` keyword it is suspended at is itself
+enclosed in a ``try:`` or a ``with:`` block.
+
 Using the default GC called ``minimark``, the built-in function ``id()``
 works like it does in CPython.  With other GCs it returns numbers that
 are not real addresses (because an object can move around several times)
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -169,9 +173,17 @@
     def _combine_starstarargs_wrapped(self, w_starstararg):
         # unpack the ** arguments
         space = self.space
+        keywords, values_w = space.view_as_kwargs(w_starstararg)
+        if keywords is not None: # this path also taken for empty dicts
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
+            return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
-            if not space.is_true(w_starstararg):
-                return False # don't call unpackiterable - it's jit-opaque
             keys_w = space.unpackiterable(w_starstararg)
         else:
             try:
@@ -186,11 +198,8 @@
                                    "a mapping, not %s" % (typename,)))
                 raise
             keys_w = space.unpackiterable(w_keys)
-        if keys_w:
-            self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
-            return True
-        else:
-            return False    # empty dict; don't disable the JIT
+        self._do_combine_starstarargs_wrapped(keys_w, w_starstararg)
+        return True
 
     def _do_combine_starstarargs_wrapped(self, keys_w, w_starstararg):
         space = self.space
@@ -227,6 +236,20 @@
             self.keywords_w = self.keywords_w + keywords_w
         self.keyword_names_w = keys_w
 
+    @jit.look_inside_iff(lambda self, keywords, keywords_w:
+            jit.isconstant(len(keywords) and
+            jit.isconstant(self.keywords)))
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
+
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
         or raise a real ValueError if the length is wrong."""
@@ -385,7 +408,7 @@
 
         # collect extra keyword arguments into the **kwarg
         if has_kwarg:
-            w_kwds = self.space.newdict()
+            w_kwds = self.space.newdict(kwargs=True)
             if num_remainingkwds:
                 #
                 limit = len(keywords)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -914,6 +914,12 @@
         """
         return None
 
+    def view_as_kwargs(self, w_dict):
+        """ if w_dict is a kwargs-dict, return two lists, one of unwrapped
+        strings and one of wrapped values. otherwise return (None, None)
+        """
+        return (None, None)
+
     def newlist_str(self, list_s):
         return self.newlist([self.wrap(s) for s in list_s])
 
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -75,7 +75,10 @@
     def unpackiterable(self, it):
         return list(it)
 
-    def newdict(self):
+    def view_as_kwargs(self, x):
+        return None, None
+
+    def newdict(self, kwargs=False):
         return {}
 
     def newlist(self, l=[]):
@@ -488,6 +491,57 @@
         assert len(l) == 1
         assert l[0] == space.wrap(5)
 
+    def test_starstarargs_special(self):
+        class kwargs(object):
+            def __init__(self, k, v):
+                self.k = k
+                self.v = v
+        class MyDummySpace(DummySpace):
+            def view_as_kwargs(self, kw):
+                if isinstance(kw, kwargs):
+                    return kw.k, kw.v
+                return None, None
+        space = MyDummySpace()
+        for i in range(3):
+            kwds = [("c", 3)]
+            kwds_w = dict(kwds[:i])
+            keywords = kwds_w.keys()
+            keywords_w = kwds_w.values()
+            rest = dict(kwds[i:])
+            w_kwds = kwargs(rest.keys(), rest.values())
+            if i == 2:
+                w_kwds = None
+            assert len(keywords) == len(keywords_w)
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c"]), defaults_w=[4])
+            assert l == [1, 2, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "b1", "c"]), defaults_w=[4, 5])
+            assert l == [1, 2, 4, 3]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            args._match_signature(None, l, Signature(["a", "b", "c", "d"]), defaults_w=[4, 5])
+            assert l == [1, 2, 3, 5]
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["c", "b", "a", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None, None]
+            py.test.raises(ArgErr, args._match_signature, None, l,
+                           Signature(["a", "b", "c1", "d"]), defaults_w=[4, 5])
+            args = Arguments(space, [1, 2], keywords[:], keywords_w[:], w_starstararg=w_kwds)
+            l = [None, None, None]
+            args._match_signature(None, l, Signature(["a", "b"], None, "**"))
+            assert l == [1, 2, {'c': 3}]
+        excinfo = py.test.raises(OperationError, Arguments, space, [], ["a"],
+                                 [1], w_starstararg=kwargs(["a"], [2]))
+        assert excinfo.value.w_type is TypeError
+
+
+
 class TestErrorHandling(object):
     def test_missing_args(self):
         # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg,
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -2253,24 +2253,6 @@
     """Concat two strings giving a new Unicode string."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
-def PyUnicode_Split(space, s, sep, maxsplit):
-    """Split a string giving a list of Unicode strings.  If sep is NULL, splitting
-    will be done at all whitespace substrings.  Otherwise, splits occur at the given
-    separator.  At most maxsplit splits will be done.  If negative, no limit is
-    set.  Separators are not included in the resulting list.
-
-    This function used an int type for maxsplit. This might require
-    changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, rffi.INT_real], PyObject)
-def PyUnicode_Splitlines(space, s, keepend):
-    """Split a Unicode string at line breaks, returning a list of Unicode strings.
-    CRLF is considered to be one line break.  If keepend is 0, the Line break
-    characters are not included in the resulting strings."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.CCHARP], PyObject)
 def PyUnicode_Translate(space, str, table, errors):
     """Translate a string by applying a character mapping table to it and return the
@@ -2287,29 +2269,6 @@
     use the default error handling."""
     raise NotImplementedError
 
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real], Py_ssize_t, error=-2)
-def PyUnicode_Find(space, str, substr, start, end, direction):
-    """Return the first position of substr in str*[*start:end] using the given
-    direction (direction == 1 means to do a forward search, direction == -1 a
-    backward search).  The return value is the index of the first match; a value of
-    -1 indicates that no match was found, and -2 indicates that an error
-    occurred and an exception has been set.
-
-    This function used an int type for start and end. This
-    might require changes in your code for properly supporting 64-bit
-    systems."""
-    raise NotImplementedError
-
- at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
-def PyUnicode_Count(space, str, substr, start, end):
-    """Return the number of non-overlapping occurrences of substr in
-    str[start:end].  Return -1 if an error occurred.
-
-    This function returned an int type and used an int
-    type for start and end. This might require changes in your code for
-    properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([PyObject, PyObject, rffi.INT_real], PyObject)
 def PyUnicode_RichCompare(space, left, right, op):
     """Rich compare two unicode strings and return one of the following:
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,7 +453,35 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, 1) == 1
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+
+    def test_count(self, space, api):
+        w_str = space.wrap(u"abcabdab")
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, -1) == 2
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), 0, 2) == 1
+        assert api.PyUnicode_Count(w_str, space.wrap(u"ab"), -5, 30) == 2
+
+    def test_find(self, space, api):
+        w_str = space.wrap(u"abcabcd")
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, 1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, 1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 3, 7, -1) == 5
+        assert api.PyUnicode_Find(w_str, space.wrap(u"c"), 0, 4, -1) == 2
+        assert api.PyUnicode_Find(w_str, space.wrap(u"z"), 0, 4, -1) == -1
+
+    def test_split(self, space, api):
+        w_str = space.wrap(u"a\nb\nc\nd")
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), -1)))
+        assert r"[u'a', u'b', u'c\nd']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(w_str, space.wrap('\n'), 2)))
+        assert r"[u'a', u'b', u'c d']" == space.unwrap(space.repr(
+                api.PyUnicode_Split(space.wrap(u'a\nb  c d'), None, 2)))
+        assert "[u'a', u'b', u'c', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 0)))
+        assert r"[u'a\n', u'b\n', u'c\n', u'd']" == space.unwrap(space.repr(
+                api.PyUnicode_Splitlines(w_str, 1)))
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,8 +593,51 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
 
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t], Py_ssize_t, error=-1)
+def PyUnicode_Count(space, w_str, w_substr, start, end):
+    """Return the number of non-overlapping occurrences of substr in
+    str[start:end].  Return -1 if an error occurred."""
+    w_count = space.call_method(w_str, "count", w_substr,
+                                space.wrap(start), space.wrap(end))
+    return space.int_w(w_count)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t, Py_ssize_t, rffi.INT_real],
+             Py_ssize_t, error=-2)
+def PyUnicode_Find(space, w_str, w_substr, start, end, direction):
+    """Return the first position of substr in str*[*start:end] using
+    the given direction (direction == 1 means to do a forward search,
+    direction == -1 a backward search).  The return value is the index
+    of the first match; a value of -1 indicates that no match was
+    found, and -2 indicates that an error occurred and an exception
+    has been set."""
+    if rffi.cast(lltype.Signed, direction) > 0:
+        w_pos = space.call_method(w_str, "find", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    else:
+        w_pos = space.call_method(w_str, "rfind", w_substr,
+                                  space.wrap(start), space.wrap(end))
+    return space.int_w(w_pos)
+
+ at cpython_api([PyObject, PyObject, Py_ssize_t], PyObject)
+def PyUnicode_Split(space, w_str, w_sep, maxsplit):
+    """Split a string giving a list of Unicode strings.  If sep is
+    NULL, splitting will be done at all whitespace substrings.
+    Otherwise, splits occur at the given separator.  At most maxsplit
+    splits will be done.  If negative, no limit is set.  Separators
+    are not included in the resulting list."""
+    if w_sep is None:
+        w_sep = space.w_None
+    return space.call_method(w_str, "split", w_sep, space.wrap(maxsplit))
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyUnicode_Splitlines(space, w_str, keepend):
+    """Split a Unicode string at line breaks, returning a list of
+    Unicode strings.  CRLF is considered to be one line break.  If
+    keepend is 0, the Line break characters are not included in the
+    resulting strings."""
+    return space.call_method(w_str, "splitlines", space.wrap(keepend))
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -513,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1435,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1410,6 +1410,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -762,6 +762,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -772,7 +774,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -785,6 +788,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -796,7 +801,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -244,6 +244,7 @@
         print guards
         assert len(guards) <= 20
 
+
     def test_stararg_virtual(self):
         def main(x):
             def g(*args):
@@ -486,3 +487,38 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_kwargs_virtual2(self):
+        log = self.run("""
+        def f(*args, **kwargs):
+            kwargs['a'] = kwargs['z'] * 0
+            return g(1, *args, **kwargs)
+
+        def g(x, y, z=2, a=1):
+            return x - y + z + a
+
+        def main(stop):
+            res = 0
+            i = 0
+            while i < stop:
+                res = f(res, z=i) # ID: call
+                i += 1
+            return res""", [1000])
+        assert log.result == 500
+        loop, = log.loops_by_id('call')
+        print loop.ops_by_id('call')
+        assert loop.match("""
+            i65 = int_lt(i58, i29)
+            guard_true(i65, descr=...)
+            guard_not_invalidated(..., descr=...)
+            i66 = force_token()
+            i67 = force_token()
+            i69 = int_sub_ovf(1, i56)
+            guard_no_overflow(..., descr=...)
+            i70 = int_add_ovf(i69, i58)
+            guard_no_overflow(..., descr=...)
+            i71 = int_add(i58, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
+
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -110,7 +110,7 @@
         "NOT_RPYTHON"
         raise NotImplementedError
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return w_some_obj()
 
diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -434,6 +434,13 @@
         self.lastblock = block
         self.pushvalue(w_result)
 
+    def BUILD_LIST_FROM_ARG(self, _, next_instr):
+        # This opcode was added with pypy-1.8.  Here is a simpler
+        # version, enough for annotation.
+        last_val = self.popvalue()
+        self.pushvalue(self.space.newlist([]))
+        self.pushvalue(last_val)
+
     # XXX Unimplemented 2.7 opcodes ----------------
 
     # Set literals, set comprehensions
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -33,7 +33,7 @@
 
     @staticmethod
     def allocate_and_init_instance(space, w_type=None, module=False,
-                                   instance=False, strdict=False):
+                                   instance=False, strdict=False, kwargs=False):
 
         if space.config.objspace.std.withcelldict and module:
             from pypy.objspace.std.celldict import ModuleDictStrategy
@@ -46,11 +46,15 @@
             assert w_type is None
             strategy = space.fromcache(StringDictStrategy)
 
+        elif kwargs:
+            assert w_type is None
+            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
+            strategy = space.fromcache(KwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
-
         if w_type is None:
             w_type = space.w_dict
+
         storage = strategy.get_empty_storage()
         w_self = space.allocate_instance(W_DictMultiObject, w_type)
         W_DictMultiObject.__init__(w_self, space, strategy, storage)
@@ -91,7 +95,8 @@
                     getitem_str delitem length \
                     clear w_keys values \
                     items iter setdefault \
-                    popitem listview_str listview_int".split()
+                    popitem listview_str listview_int \
+                    view_as_kwargs".split()
 
     def make_method(method):
         def f(self, *args):
@@ -165,6 +170,9 @@
     def listview_int(self, w_dict):
         return None
 
+    def view_as_kwargs(self, w_dict):
+        return (None, None)
+
 class EmptyDictStrategy(DictStrategy):
 
     erase, unerase = rerased.new_erasing_pair("empty")
@@ -254,6 +262,9 @@
     def popitem(self, w_dict):
         raise KeyError
 
+    def view_as_kwargs(self, w_dict):
+        return ([], [])
+
 registerimplementation(W_DictMultiObject)
 
 # DictImplementation lattice
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/kwargsdict.py
@@ -0,0 +1,165 @@
+## ----------------------------------------------------------------------------
+## dict strategy (see dictmultiobject.py)
+
+from pypy.rlib import rerased, jit
+from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               IteratorImplementation,
+                                               ObjectDictStrategy,
+                                               StringDictStrategy)
+
+
+class KwargsDictStrategy(DictStrategy):
+    erase, unerase = rerased.new_erasing_pair("kwargsdict")
+    erase = staticmethod(erase)
+    unerase = staticmethod(unerase)
+
+    def wrap(self, key):
+        return self.space.wrap(key)
+
+    def unwrap(self, wrapped):
+        return self.space.str_w(wrapped)
+
+    def get_empty_storage(self):
+        d = ([], [])
+        return self.erase(d)
+
+    def is_correct_type(self, w_obj):
+        space = self.space
+        return space.is_w(space.type(w_obj), space.w_str)
+
+    def _never_equal_to(self, w_lookup_type):
+        return False
+
+    def iter(self, w_dict):
+        return KwargsDictIterator(self.space, self, w_dict)
+
+    def w_keys(self, w_dict):
+        return self.space.newlist([self.space.wrap(key) for key in self.unerase(w_dict.dstorage)[0]])
+
+    def setitem(self, w_dict, w_key, w_value):
+        space = self.space
+        if self.is_correct_type(w_key):
+            self.setitem_str(w_dict, self.unwrap(w_key), w_value)
+            return
+        else:
+            self.switch_to_object_strategy(w_dict)
+            w_dict.setitem(w_key, w_value)
+
+    def setitem_str(self, w_dict, key, w_value):
+        self._setitem_str_indirection(w_dict, key, w_value)
+
+    @jit.look_inside_iff(lambda self, w_dict, key, w_value:
+            jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _setitem_str_indirection(self, w_dict, key, w_value):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                values_w[i] = w_value
+                break
+        else:
+            # limit the size so that the linear searches don't become too long
+            if len(keys) >= 16:
+                self.switch_to_string_strategy(w_dict)
+                w_dict.setitem_str(key, w_value)
+            else:
+                keys.append(key)
+                values_w.append(w_value)
+
+    def setdefault(self, w_dict, w_key, w_default):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.setdefault(w_key, w_default)
+
+    def delitem(self, w_dict, w_key):
+        # XXX could do better, but is it worth it?
+        self.switch_to_object_strategy(w_dict)
+        return w_dict.delitem(w_key)
+
+    def length(self, w_dict):
+        return len(self.unerase(w_dict.dstorage)[0])
+
+    def getitem_str(self, w_dict, key):
+        return self._getitem_str_indirection(w_dict, key)
+
+    @jit.look_inside_iff(lambda self, w_dict, key: jit.isconstant(self.length(w_dict)) and jit.isconstant(key))
+    def _getitem_str_indirection(self, w_dict, key):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            if keys[i] == key:
+                return values_w[i]
+        return None
+
+    def getitem(self, w_dict, w_key):
+        space = self.space
+        if self.is_correct_type(w_key):
+            return self.getitem_str(w_dict, self.unwrap(w_key))
+        elif self._never_equal_to(space.type(w_key)):
+            return None
+        else:
+            self.switch_to_object_strategy(w_dict)
+            return w_dict.getitem(w_key)
+
+    def w_keys(self, w_dict):
+        l = self.unerase(w_dict.dstorage)[0]
+        return self.space.newlist_str(l[:])
+
+    def values(self, w_dict):
+        return self.unerase(w_dict.dstorage)[1][:] # to make non-resizable
+
+    def items(self, w_dict):
+        space = self.space
+        keys, values_w = self.unerase(w_dict.dstorage)
+        result = []
+        for i in range(len(keys)):
+            result.append(space.newtuple([self.wrap(keys[i]), values_w[i]]))
+        return result
+
+    def popitem(self, w_dict):
+        keys, values_w = self.unerase(w_dict.dstorage)
+        key = keys.pop()
+        w_value = values_w.pop()
+        return (self.wrap(key), w_value)
+
+    def clear(self, w_dict):
+        w_dict.dstorage = self.get_empty_storage()
+
+    def switch_to_object_strategy(self, w_dict):
+        strategy = self.space.fromcache(ObjectDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        d_new = strategy.unerase(strategy.get_empty_storage())
+        for i in range(len(keys)):
+            d_new[self.wrap(keys[i])] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = strategy.erase(d_new)
+
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(StringDictStrategy)
+        keys, values_w = self.unerase(w_dict.dstorage)
+        storage = strategy.get_empty_storage()
+        d_new = strategy.unerase(storage)
+        for i in range(len(keys)):
+            d_new[keys[i]] = values_w[i]
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+    def view_as_kwargs(self, w_dict):
+        return self.unerase(w_dict.dstorage)
+
+
+class KwargsDictIterator(IteratorImplementation):
+    def __init__(self, space, strategy, dictimplementation):
+        IteratorImplementation.__init__(self, space, strategy, dictimplementation)
+        keys, values_w = strategy.unerase(self.dictimplementation.dstorage)
+        self.iterator = iter(range(len(keys)))
+        # XXX this potentially leaks
+        self.keys = keys
+        self.values_w = values_w
+
+    def next_entry(self):
+        # note that this 'for' loop only runs once, at most
+        for i in self.iterator:
+            return self.space.wrap(self.keys[i]), self.values_w[i]
+        else:
+            return None, None
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -313,11 +313,11 @@
     def newlist_str(self, list_s):
         return W_ListObject.newlist_str(self, list_s)
 
-    def newdict(self, module=False, instance=False,
+    def newdict(self, module=False, instance=False, kwargs=False,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
                 self, module=module, instance=instance,
-                strdict=strdict)
+                strdict=strdict, kwargs=kwargs)
 
     def newset(self):
         from pypy.objspace.std.setobject import newset
@@ -472,6 +472,11 @@
             return w_obj.getitems_int()
         return None
 
+    def view_as_kwargs(self, w_dict):
+        if type(w_dict) is W_DictMultiObject:
+            return w_dict.view_as_kwargs()
+        return (None, None)
+
     def _uses_list_iter(self, w_obj):
         from pypy.objspace.descroperation import list_iter
         return self.lookup(w_obj, '__iter__') is list_iter(self)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -0,0 +1,120 @@
+import py
+from pypy.conftest import gettestobjspace, option
+from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject
+from pypy.objspace.std.kwargsdict import *
+
+space = FakeSpace()
+strategy = KwargsDictStrategy(space)
+
+def test_create():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem(space.wrap("a")) == 1
+    assert d.getitem(space.wrap("b")) == 2
+    assert d.getitem(space.wrap("c")) == 3
+    assert d.w_keys() == keys
+    assert d.values() == values
+
+def test_set_existing():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("a", 4) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("b", 5) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 3
+    assert d.setitem_str("c", 6) is None
+    assert d.getitem_str("a") == 4
+    assert d.getitem_str("b") == 5
+    assert d.getitem_str("c") == 6
+    assert d.getitem(space.wrap("a")) == 4
+    assert d.getitem(space.wrap("b")) == 5
+    assert d.getitem(space.wrap("c")) == 6
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c"]
+    assert values == [4, 5, 6]
+
+
+def test_set_new():
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") is None
+    assert d.setitem_str("d", 4) is None
+    assert d.getitem_str("a") == 1
+    assert d.getitem_str("b") == 2
+    assert d.getitem_str("c") == 3
+    assert d.getitem_str("d") == 4
+    assert d.w_keys() == keys
+    assert d.values() == values
+    assert keys == ["a", "b", "c", "d"]
+    assert values == [1, 2, 3, 4]
+
+def test_limit_size():
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    for i in range(100):
+        assert d.setitem_str("d%s" % i, 4) is None
+    assert d.strategy is not strategy
+    assert "StringDictStrategy" == d.strategy.__class__.__name__
+
+def test_keys_doesnt_wrap():
+    space = FakeSpace()
+    space.newlist = None
+    strategy = KwargsDictStrategy(space)
+    keys = ["a", "b", "c"]
+    values = [1, 2, 3]
+    storage = strategy.erase((keys, values))
+    d = W_DictMultiObject(space, strategy, storage)
+    w_l = d.w_keys() # does not crash
+
+
+from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
+def get_impl(self):
+    storage = strategy.erase(([], []))
+    return W_DictMultiObject(space, strategy, storage)
+class TestKwargsDictImplementation(BaseTestRDictImplementation):
+    StrategyClass = KwargsDictStrategy
+    get_impl = get_impl
+    def test_delitem(self):
+        pass # delitem devolves for now
+
+class TestDevolvedKwargsDictImplementation(BaseTestDevolvedDictImplementation):
+    get_impl = get_impl
+    StrategyClass = KwargsDictStrategy
+
+
+class AppTestKwargsDictStrategy(object):
+    def setup_class(cls):
+        if option.runappdirect:
+            py.test.skip("__repr__ doesn't work on appdirect")
+
+    def w_get_strategy(self, obj):
+        import __pypy__
+        r = __pypy__.internal_repr(obj)
+        return r[r.find("(") + 1: r.find(")")]
+
+    def test_create(self):
+        def f(**args):
+            return args
+        d = f(a=1)
+        assert "KwargsDictStrategy" in self.get_strategy(d)
+
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,31 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        res = 'bar' in x
+        assert res is True
+        #
+        class MyError(Exception):
+            pass
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise MyError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(MyError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -204,6 +204,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.call_location()
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,9 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +18,17 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +40,7 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:

From noreply at buildbot.pypy.org  Mon Apr 23 10:31:14 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon, 23 Apr 2012 10:31:14 +0200 (CEST)
Subject: [pypy-commit] pypy pytest: add a interp level reinterpetation to
	the failure demo
Message-ID: <20120423083114.B7D0F8203C@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: pytest
Changeset: r54652:ca3203a42a65
Date: 2012-04-23 10:25 +0200
http://bitbucket.org/pypy/pypy/changeset/ca3203a42a65/

Log:	add a interp level reinterpetation to the failure demo

diff --git a/pypy/tool/pytest/pypy_test_failure_demo.py b/pypy/tool/pytest/pypy_test_failure_demo.py
--- a/pypy/tool/pytest/pypy_test_failure_demo.py
+++ b/pypy/tool/pytest/pypy_test_failure_demo.py
@@ -8,6 +8,10 @@
 def test_interp_func(space): 
     assert space.is_true(space.w_None) 
 
+def test_interp_reinterpret(space):
+    a = 1
+    assert a == 2
+
 class TestInterpTest: 
     def test_interp_method(self): 
         assert self.space.is_true(self.space.w_False) 

From noreply at buildbot.pypy.org  Mon Apr 23 10:31:15 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon, 23 Apr 2012 10:31:15 +0200 (CEST)
Subject: [pypy-commit] pypy pytest: use the reinterp assert mode,
 so we get inperp level reinterpretation back
Message-ID: <20120423083115.EF6208203C@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: pytest
Changeset: r54653:6ec7a3f49f4d
Date: 2012-04-23 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/6ec7a3f49f4d/

Log:	use the reinterp assert mode, so we get inperp level
	reinterpretation back

diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assert=plain -rf
+addopts = --assert=reinterp -rf

From noreply at buildbot.pypy.org  Mon Apr 23 10:31:17 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon, 23 Apr 2012 10:31:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge from pytest
Message-ID: <20120423083117.399538203C@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54654:5ce4a6d6c2ec
Date: 2012-04-23 10:30 +0200
http://bitbucket.org/pypy/pypy/changeset/5ce4a6d6c2ec/

Log:	merge from pytest

diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assert=plain -rf
+addopts = --assert=reinterp -rf
diff --git a/pypy/tool/pytest/pypy_test_failure_demo.py b/pypy/tool/pytest/pypy_test_failure_demo.py
--- a/pypy/tool/pytest/pypy_test_failure_demo.py
+++ b/pypy/tool/pytest/pypy_test_failure_demo.py
@@ -8,6 +8,10 @@
 def test_interp_func(space): 
     assert space.is_true(space.w_None) 
 
+def test_interp_reinterpret(space):
+    a = 1
+    assert a == 2
+
 class TestInterpTest: 
     def test_interp_method(self): 
         assert self.space.is_true(self.space.w_False) 

From noreply at buildbot.pypy.org  Mon Apr 23 11:00:30 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Mon, 23 Apr 2012 11:00:30 +0200 (CEST)
Subject: [pypy-commit] pypy pytest: kill tool/pytest/modcheck for
	pytest.importorskip
Message-ID: <20120423090030.D25E68203C@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: pytest
Changeset: r54655:3bcc57452f5c
Date: 2012-04-23 10:59 +0200
http://bitbucket.org/pypy/pypy/changeset/3bcc57452f5c/

Log:	kill tool/pytest/modcheck for pytest.importorskip

diff --git a/pypy/rpython/module/test/test_ll_os_path.py b/pypy/rpython/module/test/test_ll_os_path.py
--- a/pypy/rpython/module/test/test_ll_os_path.py
+++ b/pypy/rpython/module/test/test_ll_os_path.py
@@ -2,14 +2,15 @@
 
 import sys, os
 
-from pypy.tool.pytest.modcheck import skipimporterror
-skipimporterror("ctypes")
-
 from pypy.rpython.lltypesystem.module.ll_os_path import Implementation as impl
 from pypy.rpython.module.support import ll_strcpy
 from pypy.rpython.test.test_llinterp import interpret
 from pypy.tool.udir import udir
 
+
+def setup_module(mod):
+    py.test.importorskip('ctypes')
+
 def test_exists():
     filename = impl.to_rstr(str(py.path.local(__file__)))
     assert impl.ll_os_path_exists(filename) == True
diff --git a/pypy/tool/pytest/modcheck.py b/pypy/tool/pytest/modcheck.py
deleted file mode 100644
--- a/pypy/tool/pytest/modcheck.py
+++ /dev/null
@@ -1,14 +0,0 @@
-import py
-
-def skipimporterror(name):
-    if not hasimport(name):
-        __tracebackhide__ = True
-        py.test.skip("cannot import %r module" % (name,))
-
-def hasimport(name):
-    try:
-        __import__(name)
-    except ImportError:
-        return False
-    else:
-        return True

From noreply at buildbot.pypy.org  Mon Apr 23 11:54:28 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Apr 2012 11:54:28 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: replace the used_keywords
 list of bools with a much shorter list
Message-ID: <20120423095428.2CC8C8203C@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54656:075a2619161a
Date: 2012-04-22 11:37 +0200
http://bitbucket.org/pypy/pypy/changeset/075a2619161a/

Log:	replace the used_keywords list of bools with a much shorter list
	that is as long as the number of potentially missing arguments the
	content of the list is an index into keywords_w

	the idea is that scope_w should not be passed into a residual call.
	that would make it escape, which is bad, scope_w is part of an
	(ideally virtual) frame later

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -296,35 +296,45 @@
 
         # handle keyword arguments
         num_remainingkwds = 0
-        used_keywords = None
         keywords_w = self.keywords_w
+        kwds_mapping = None
         if num_kwds:
-            used_keywords = [False] * num_kwds
+            # kwds_mapping maps target indexes in the scope (minus input_argcount)
+            # to positions in the keywords_w list
+            kwds_mapping = [-1] * (co_argcount - input_argcount)
             # match the keywords given at the call site to the argument names
             # the called function takes
+            # this function must not take a scope_w, to make the scope not
+            # escape
             num_remainingkwds = _match_keywords(
                     signature, blindargs, input_argcount, keywords,
-                    keywords_w, scope_w, used_keywords,
-                    self._jit_few_keywords)
+                    kwds_mapping, self._jit_few_keywords)
             if num_remainingkwds:
                 if w_kwds is not None:
                     # collect extra keyword arguments into the **kwarg
                     _collect_keyword_args(
                             self.space, keywords, keywords_w, w_kwds,
-                            used_keywords, self.keyword_names_w, self._jit_few_keywords)
+                            kwds_mapping, self.keyword_names_w, self._jit_few_keywords)
                 else:
                     if co_argcount == 0:
                         raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0)
                     raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords,
-                                            used_keywords, self.keyword_names_w)
+                                            kwds_mapping, self.keyword_names_w)
 
-        # check for missing arguments and fill them with defaults, if available
+        # check for missing arguments and fill them from the kwds,
+        # or with defaults, if available
         missing = 0
         if input_argcount < co_argcount:
             def_first = co_argcount - (0 if defaults_w is None else len(defaults_w))
+            j = 0
+            kwds_index = -1
             for i in range(input_argcount, co_argcount):
-                if scope_w[i] is not None:
-                    continue
+                if kwds_mapping is not None:
+                    kwds_index = kwds_mapping[j]
+                    j += 1
+                    if kwds_index != -1:
+                        scope_w[i] = keywords_w[kwds_index]
+                        continue
                 defnum = i - def_first
                 if defnum >= 0:
                     scope_w[i] = defaults_w[defnum]
@@ -441,10 +451,10 @@
         i += 1
 
 @jit.look_inside_iff(
-    lambda signature, blindargs, input_argcount, keywords, keywords_w,
-           scope_w, used_keywords, jiton: jiton)
+    lambda signature, blindargs, input_argcount, keywords,
+           scope_w, kwds_mapping, jiton: jiton)
 def _match_keywords(signature, blindargs, input_argcount,
-                    keywords, keywords_w, scope_w, used_keywords, _):
+                    keywords, kwds_mapping, _):
     # letting JIT unroll the loop is *only* safe if the callsite didn't
     # use **args because num_kwds can be arbitrarily large otherwise.
     num_kwds = num_remainingkwds = len(keywords)
@@ -466,22 +476,25 @@
             if blindargs <= j:
                 raise ArgErrMultipleValues(name)
         else:
-            assert scope_w[j] is None
-            scope_w[j] = keywords_w[i]
-            used_keywords[i] = True # mark as used
+            kwds_mapping[j - input_argcount] = i # map to the right index
             num_remainingkwds -= 1
     return num_remainingkwds
 
 @jit.look_inside_iff(
-    lambda space, keywords, keywords_w, w_kwds, used_keywords,
+    lambda space, keywords, keywords_w, w_kwds, kwds_mapping,
         keyword_names_w, jiton: jiton)
-def _collect_keyword_args(space, keywords, keywords_w, w_kwds, used_keywords,
+def _collect_keyword_args(space, keywords, keywords_w, w_kwds, kwds_mapping,
                           keyword_names_w, _):
     limit = len(keywords)
     if keyword_names_w is not None:
         limit -= len(keyword_names_w)
     for i in range(len(keywords)):
-        if not used_keywords[i]:
+        # again a dangerous-looking loop that either the JIT unrolls
+        # or that is not too bad, because len(kwds_mapping) is small
+        for j in kwds_mapping:
+            if i == j:
+                break
+        else:
             if i < limit:
                 w_key = space.wrap(keywords[i])
             else:
@@ -701,13 +714,13 @@
 
 class ArgErrUnknownKwds(ArgErr):
 
-    def __init__(self, space, num_remainingkwds, keywords, used_keywords,
+    def __init__(self, space, num_remainingkwds, keywords, kwds_mapping,
                  keyword_names_w):
         name = ''
         self.num_kwds = num_remainingkwds
         if num_remainingkwds == 1:
             for i in range(len(keywords)):
-                if not used_keywords[i]:
+                if i not in kwds_mapping:
                     name = keywords[i]
                     if name is None:
                         # We'll assume it's unicode. Encode it.
diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py
--- a/pypy/interpreter/test/test_argument.py
+++ b/pypy/interpreter/test/test_argument.py
@@ -632,11 +632,14 @@
 
     def test_unknown_keywords(self):
         space = DummySpace()
-        err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [True, False], None)
+        err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [0], None)
         s = err.getmsg()
         assert s == "got an unexpected keyword argument 'b'"
+        err = ArgErrUnknownKwds(space, 1, ['a', 'b'], [1], None)
+        s = err.getmsg()
+        assert s == "got an unexpected keyword argument 'a'"
         err = ArgErrUnknownKwds(space, 2, ['a', 'b', 'c'],
-                                [True, False, False], None)
+                                [0], None)
         s = err.getmsg()
         assert s == "got 2 unexpected keyword arguments"
 
@@ -646,7 +649,7 @@
                 defaultencoding = 'utf-8'
         space = DummySpaceUnicode()
         err = ArgErrUnknownKwds(space, 1, ['a', None, 'b', 'c'],
-                                [True, False, True, True],
+                                [0, 3, 2],
                                 [unichr(0x1234), u'b', u'c'])
         s = err.getmsg()
         assert s == "got an unexpected keyword argument '\xe1\x88\xb4'"

From noreply at buildbot.pypy.org  Mon Apr 23 11:54:29 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Apr 2012 11:54:29 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: give an error when the
 predicate of look_inside_iff has a different number of
Message-ID: <20120423095429.731BE8203C@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54657:2ebd2356d035
Date: 2012-04-22 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/2ebd2356d035/

Log:	give an error when the predicate of look_inside_iff has a different
	number of arguments than the decorated function.

diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -148,6 +148,8 @@
             thing._annspecialcase_ = "specialize:call_location"
 
         args = _get_args(func)
+        predicateargs = _get_args(predicate)
+        assert len(args) == len(predicateargs), "%s and predicate %s need the same numbers of arguments" % (func, predicate)
         d = {
             "dont_look_inside": dont_look_inside,
             "predicate": predicate,

From noreply at buildbot.pypy.org  Mon Apr 23 11:54:30 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Apr 2012 11:54:30 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: fix
Message-ID: <20120423095430.B6CBB8203C@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54658:c095078e5892
Date: 2012-04-22 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/c095078e5892/

Log:	fix

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -451,8 +451,8 @@
         i += 1
 
 @jit.look_inside_iff(
-    lambda signature, blindargs, input_argcount, keywords,
-           scope_w, kwds_mapping, jiton: jiton)
+    lambda signature, blindargs, input_argcount,
+           keywords, kwds_mapping, jiton: jiton)
 def _match_keywords(signature, blindargs, input_argcount,
                     keywords, kwds_mapping, _):
     # letting JIT unroll the loop is *only* safe if the callsite didn't

From noreply at buildbot.pypy.org  Mon Apr 23 11:54:31 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Apr 2012 11:54:31 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: JIT improvements
Message-ID: <20120423095431.F0F168203C@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54659:63bcd5a3d3af
Date: 2012-04-22 12:55 +0200
http://bitbucket.org/pypy/pypy/changeset/63bcd5a3d3af/

Log:	JIT improvements

diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -301,7 +301,10 @@
         if num_kwds:
             # kwds_mapping maps target indexes in the scope (minus input_argcount)
             # to positions in the keywords_w list
-            kwds_mapping = [-1] * (co_argcount - input_argcount)
+            kwds_mapping = [0] * (co_argcount - input_argcount)
+            # initialize manually, for the JIT :-(
+            for i in range(len(kwds_mapping)):
+                kwds_mapping[i] = -1
             # match the keywords given at the call site to the argument names
             # the called function takes
             # this function must not take a scope_w, to make the scope not
@@ -332,7 +335,7 @@
                 if kwds_mapping is not None:
                     kwds_index = kwds_mapping[j]
                     j += 1
-                    if kwds_index != -1:
+                    if kwds_index >= 0:
                         scope_w[i] = keywords_w[kwds_index]
                         continue
                 defnum = i - def_first

From noreply at buildbot.pypy.org  Mon Apr 23 11:54:33 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Apr 2012 11:54:33 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: merge default
Message-ID: <20120423095433.AADAE8203C@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54660:906ef20427d6
Date: 2012-04-22 12:55 +0200
http://bitbucket.org/pypy/pypy/changeset/906ef20427d6/

Log:	merge default

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3746,9 +3746,9 @@
             return g(i)
         def main(i):
             if i == 2:
-                return f(i)
+                return f(2)
             elif i == 3:
-                return f(i)
+                return f(3)
             else:
                 raise NotImplementedError
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -513,7 +513,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1435,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1410,6 +1410,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -762,6 +762,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -772,7 +774,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -785,6 +788,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -796,7 +801,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,31 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        res = 'bar' in x
+        assert res is True
+        #
+        class MyError(Exception):
+            pass
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise MyError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(MyError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -206,6 +206,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.call_location()
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,9 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +18,17 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +40,7 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:

From noreply at buildbot.pypy.org  Mon Apr 23 11:54:35 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Apr 2012 11:54:35 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-arguments: a special strategy for the
 especially common case of an empty kwargs dict
Message-ID: <20120423095435.131358203C@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: virtual-arguments
Changeset: r54661:57a4fb6a5b6a
Date: 2012-04-22 20:09 +0200
http://bitbucket.org/pypy/pypy/changeset/57a4fb6a5b6a/

Log:	a special strategy for the especially common case of an empty kwargs
	dict

diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -48,8 +48,8 @@
 
         elif kwargs:
             assert w_type is None
-            from pypy.objspace.std.kwargsdict import KwargsDictStrategy
-            strategy = space.fromcache(KwargsDictStrategy)
+            from pypy.objspace.std.kwargsdict import EmptyKwargsDictStrategy
+            strategy = space.fromcache(EmptyKwargsDictStrategy)
         else:
             strategy = space.fromcache(EmptyDictStrategy)
         if w_type is None:
diff --git a/pypy/objspace/std/kwargsdict.py b/pypy/objspace/std/kwargsdict.py
--- a/pypy/objspace/std/kwargsdict.py
+++ b/pypy/objspace/std/kwargsdict.py
@@ -3,11 +3,20 @@
 
 from pypy.rlib import rerased, jit
 from pypy.objspace.std.dictmultiobject import (DictStrategy,
+                                               EmptyDictStrategy,
                                                IteratorImplementation,
                                                ObjectDictStrategy,
                                                StringDictStrategy)
 
 
+class EmptyKwargsDictStrategy(EmptyDictStrategy):
+    def switch_to_string_strategy(self, w_dict):
+        strategy = self.space.fromcache(KwargsDictStrategy)
+        storage = strategy.get_empty_storage()
+        w_dict.strategy = strategy
+        w_dict.dstorage = storage
+
+
 class KwargsDictStrategy(DictStrategy):
     erase, unerase = rerased.new_erasing_pair("kwargsdict")
     erase = staticmethod(erase)
diff --git a/pypy/objspace/std/test/test_kwargsdict.py b/pypy/objspace/std/test/test_kwargsdict.py
--- a/pypy/objspace/std/test/test_kwargsdict.py
+++ b/pypy/objspace/std/test/test_kwargsdict.py
@@ -100,6 +100,14 @@
     d = W_DictMultiObject(space, strategy, storage)
     assert (space.view_as_kwargs(d) == [], [])
 
+def test_from_empty_to_kwargs():
+    strategy = EmptyKwargsDictStrategy(space)
+    storage = strategy.get_empty_storage()
+    d = W_DictMultiObject(space, strategy, storage)
+    d.setitem_str("a", 3)
+    assert isinstance(d.strategy, KwargsDictStrategy)
+
+
 from pypy.objspace.std.test.test_dictmultiobject import BaseTestRDictImplementation, BaseTestDevolvedDictImplementation
 def get_impl(self):
     storage = strategy.erase(([], []))
@@ -130,4 +138,6 @@
             return args
         d = f(a=1)
         assert "KwargsDictStrategy" in self.get_strategy(d)
+        d = f()
+        assert "EmptyKwargsDictStrategy" in self.get_strategy(d)
 

From noreply at buildbot.pypy.org  Mon Apr 23 12:04:06 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 12:04:06 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: make pin return a boolean
Message-ID: <20120423100406.2D4708203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54662:05b513555e3c
Date: 2012-04-23 10:49 +0200
http://bitbucket.org/pypy/pypy/changeset/05b513555e3c/

Log:	make pin return a boolean

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -19,7 +19,7 @@
     pass
 
 def pin(obj):
-    pass
+    return False
 
 def unpin(obj):
     pass
@@ -75,15 +75,8 @@
     With other moving GCs like the MiniMark GC, it can be True for some
     time, then False for the same object, when we are sure that it won't
     move any more.
-
-    We keep it for False when uncompiled, because malloc_and_pin would actually
-    return an object. This executes more interesting paths, but also:
-
-    x = malloc_and_pin(T)
-    if x:
-       assert not can_move(x)
     """
-    return False
+    return True
 
 class CanMoveEntry(ExtRegistryEntry):
     _about_ = can_move
@@ -506,14 +499,15 @@
     _about_ = pin
 
     def compute_result_annotation(self, s_arg):
-        pass
+        from pypy.annotation.model import SomeBool
+        return SomeBool()
 
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
         v_obj, = hop.inputargs(hop.args_r[0])
         v_addr = hop.genop('cast_ptr_to_adr', [v_obj],
                            resulttype=llmemory.Address)
-        hop.genop('gc_pin', [v_addr])
+        return hop.genop('gc_pin', [v_addr], resulttype=lltype.Bool)
 
 class UnpinEntry(ExtRegistryEntry):
     _about_ = unpin
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -891,7 +891,7 @@
         pass       # marker for trackgcroot.py
 
     def op_gc_pin(self, obj):
-        self.heap.pin(obj)
+        return self.heap.pin(obj)
 
     def op_gc_unpin(self, obj):
         self.heap.unpin(obj)
diff --git a/pypy/rpython/lltypesystem/llheap.py b/pypy/rpython/lltypesystem/llheap.py
--- a/pypy/rpython/lltypesystem/llheap.py
+++ b/pypy/rpython/lltypesystem/llheap.py
@@ -38,7 +38,7 @@
     pass
 
 def pin(obj):
-    pass
+    return False
 
 def unpin(obj):
     pass
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -739,11 +739,12 @@
         # if 'buf' points inside 'data'.  This is only possible if we
         # followed the 2nd case in get_nonmovingbuffer(); in the first case,
         # 'buf' points to its own raw-malloced memory.
-        data = llstrtype(data)
-        data_start = cast_ptr_to_adr(data) + \
+        ll_data = llstrtype(data)
+        data_start = cast_ptr_to_adr(ll_data) + \
             offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
         followed_2nd_path = (buf == cast(TYPEP, data_start))
         keepalive_until_here(data)
+        keepalive_until_here(ll_data)
         if followed_2nd_path:
             rgc.unpin(data)
         else:
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -133,7 +133,7 @@
         return self.gc.can_move(addr)
 
     def pin(self, addr):
-        self.gc.pin(addr)
+        return self.gc.pin(addr)
 
     def unpin(self, addr):
         self.gc.unpin(addr)    

From noreply at buildbot.pypy.org  Mon Apr 23 12:04:07 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 12:04:07 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: try to pass more stuff
 around nonmovingbuffer
Message-ID: <20120423100407.7078A8203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54663:d3ec07c4849c
Date: 2012-04-23 12:02 +0200
http://bitbucket.org/pypy/pypy/changeset/d3ec07c4849c/

Log:	try to pass more stuff around nonmovingbuffer

diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -1481,8 +1481,7 @@
         if not final and is_dbcs_lead_byte(s[size-1]):
             size -= 1
 
-        dataptr = rffi.get_nonmovingbuffer(s)
-        try:
+        with rffi.scoped_nonmoving_unicodebuffer(s) as dataptr:
             # first get the size of the result
             usize = MultiByteToWideChar(CP_ACP, 0,
                                         dataptr, size,
@@ -1490,19 +1489,12 @@
             if usize == 0:
                 raise rwin32.lastWindowsError()
 
-            raw_buf, gc_buf = rffi.alloc_unicodebuffer(usize)
-            try:
+            with rffi.scoped_alloc_unicodebuffer(usize) as buf:
                 # do the conversion
                 if MultiByteToWideChar(CP_ACP, 0,
-                                       dataptr, size, raw_buf, usize) == 0:
+                                       dataptr, size, buf.raw, usize) == 0:
                     raise rwin32.lastWindowsError()
-
-                return (rffi.unicode_from_buffer(raw_buf, gc_buf, usize, usize),
-                        size)
-            finally:
-                rffi.keep_unicodebuffer_alive_until_here(raw_buf, gc_buf)
-        finally:
-            rffi.free_nonmovingbuffer(s, dataptr)
+                return buf.str(usize), size
 
     def unicode_encode_mbcs(p, size, errors, errorhandler=None):
         if size == 0:
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -713,24 +713,26 @@
         string is already nonmovable.  Must be followed by a
         free_nonmovingbuffer call.
         """
-        rgc.pin(data)
-        if rgc.can_move(data) or not we_are_translated():
-            count = len(data)
-            buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
-            for i in range(count):
-                buf[i] = data[i]
-            return buf
-        else:
-            ll_s = llstrtype(data)
-            data_start = cast_ptr_to_adr(ll_s) + \
-                offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
-            return cast(TYPEP, data_start)
+        pinned = False
+        if rgc.can_move(data):
+            if rgc.pin(data):
+                pinned = True
+            else:
+                count = len(data)
+                buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
+                for i in range(count):
+                    buf[i] = data[i]
+                return buf, False, True
+        ll_s = llstrtype(data)
+        data_start = cast_ptr_to_adr(ll_s) + \
+                     offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
+        return cast(TYPEP, data_start), pinned, False
     get_nonmovingbuffer._annenforceargs_ = [strtype]
 
     # (str, char*) -> None
     # Can't inline this because of the raw address manipulation.
     @jit.dont_look_inside
-    def free_nonmovingbuffer(data, buf):
+    def free_nonmovingbuffer(data, buf, pinned, is_raw):
         """
         Either free a non-moving buffer or keep the original storage alive.
         """
@@ -739,17 +741,12 @@
         # if 'buf' points inside 'data'.  This is only possible if we
         # followed the 2nd case in get_nonmovingbuffer(); in the first case,
         # 'buf' points to its own raw-malloced memory.
-        ll_data = llstrtype(data)
-        data_start = cast_ptr_to_adr(ll_data) + \
-            offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
-        followed_2nd_path = (buf == cast(TYPEP, data_start))
         keepalive_until_here(data)
-        keepalive_until_here(ll_data)
-        if followed_2nd_path:
+        if pinned:
             rgc.unpin(data)
-        else:
+        elif is_raw:
             lltype.free(buf, flavor='raw')
-    free_nonmovingbuffer._annenforceargs_ = [strtype, None]
+    free_nonmovingbuffer._annenforceargs_ = [strtype, None, bool, bool]
 
     # int -> (char*, str)
     def alloc_buffer(count):
@@ -1070,20 +1067,22 @@
     def __init__(self, data):
         self.data = data
     def __enter__(self):
-        self.buf = get_nonmovingbuffer(self.data)
+        self.buf, self.pinned, self.is_raw = get_nonmovingbuffer(self.data)
         return self.buf
     def __exit__(self, *args):
-        free_nonmovingbuffer(self.data, self.buf)
+        free_nonmovingbuffer(self.data, self.buf, self.pinned, self.is_raw)
 
 
 class scoped_nonmoving_unicodebuffer:
     def __init__(self, data):
         self.data = data
     def __enter__(self):
-        self.buf = get_nonmoving_unicodebuffer(self.data)
+        self.buf, self.pinned, self.is_raw = get_nonmoving_unicodebuffer(
+            self.data)
         return self.buf
     def __exit__(self, *args):
-        free_nonmoving_unicodebuffer(self.data, self.buf)
+        free_nonmoving_unicodebuffer(self.data, self.buf, self.pinned,
+                                     self.is_raw)
 
 class scoped_alloc_buffer:
     def __init__(self, size):
diff --git a/pypy/rpython/lltypesystem/test/test_rffi.py b/pypy/rpython/lltypesystem/test/test_rffi.py
--- a/pypy/rpython/lltypesystem/test/test_rffi.py
+++ b/pypy/rpython/lltypesystem/test/test_rffi.py
@@ -513,7 +513,7 @@
     def test_nonmovingbuffer(self):
         d = 'some cool data that should not move'
         def f():
-            buf = get_nonmovingbuffer(d)
+            buf, is_pinned, is_raw = get_nonmovingbuffer(d)
             try:
                 counter = 0
                 for i in range(len(d)):
@@ -521,7 +521,7 @@
                         counter += 1
                 return counter
             finally:
-                free_nonmovingbuffer(d, buf)
+                free_nonmovingbuffer(d, buf, is_pinned_is_raw)
         assert f() == len(d)
         fn = self.compile(f, [], gcpolicy='ref')
         assert fn() == len(d)
@@ -531,13 +531,13 @@
         def f():
             counter = 0
             for n in range(32):
-                buf = get_nonmovingbuffer(d)
+                buf, is_pinned, is_raw = get_nonmovingbuffer(d)
                 try:
                     for i in range(len(d)):
                         if buf[i] == d[i]:
                             counter += 1
                 finally:
-                    free_nonmovingbuffer(d, buf)
+                    free_nonmovingbuffer(d, buf, is_pinned, is_raw)
             return counter
         fn = self.compile(f, [], gcpolicy='semispace')
         # The semispace gc uses raw_malloc for its internal data structs
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -910,15 +910,12 @@
 
         def os_write_llimpl(fd, data):
             count = len(data)
-            buf = rffi.get_nonmovingbuffer(data)
-            try:
+            with rffi.scoped_nonmovingbuffer(data) as buf:
                 written = rffi.cast(lltype.Signed, os_write(
                     rffi.cast(rffi.INT, fd),
                     buf, rffi.cast(rffi.SIZE_T, count)))
                 if written < 0:
                     raise OSError(rposix.get_errno(), "os_write failed")
-            finally:
-                rffi.free_nonmovingbuffer(data, buf)
             return written
 
         def os_write_oofakeimpl(fd, data):

From noreply at buildbot.pypy.org  Mon Apr 23 12:32:10 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 12:32:10 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: use with more consistently
 and improve gctransform
Message-ID: <20120423103210.25AE88203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54664:b99acab54074
Date: 2012-04-23 12:18 +0200
http://bitbucket.org/pypy/pypy/changeset/b99acab54074/

Log:	use with more consistently and improve gctransform

diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py
--- a/pypy/module/_multibytecodec/c_codecs.py
+++ b/pypy/module/_multibytecodec/c_codecs.py
@@ -128,8 +128,7 @@
 def decodeex(decodebuf, stringdata, errors="strict", errorcb=None, namecb=None,
              ignore_error=0):
     inleft = len(stringdata)
-    inbuf = rffi.get_nonmovingbuffer(stringdata)
-    try:
+    with rffi.scoped_nonmovingbuffer(stringdata) as inbuf:
         if pypy_cjk_dec_init(decodebuf, inbuf, inleft) < 0:
             raise MemoryError
         while True:
@@ -141,9 +140,6 @@
         src = pypy_cjk_dec_outbuf(decodebuf)
         length = pypy_cjk_dec_outlen(decodebuf)
         return rffi.wcharpsize2unicode(src, length)
-    #
-    finally:
-        rffi.free_nonmovingbuffer(stringdata, inbuf)
 
 def multibytecodec_decerror(decodebuf, e, errors,
                             errorcb, namecb, stringdata):
@@ -172,11 +168,8 @@
         assert errorcb
         replace, end = errorcb(errors, namecb, reason,
                                stringdata, start, end)
-    inbuf = rffi.get_nonmoving_unicodebuffer(replace)
-    try:
+    with rffi.scoped_nonmoving_unicodebuffer(replace) as inbuf:
         r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, len(replace), end)
-    finally:
-        rffi.free_nonmoving_unicodebuffer(replace, inbuf)
     if r == MBERR_NOMEMORY:
         raise MemoryError
 
@@ -223,8 +216,7 @@
 def encodeex(encodebuf, unicodedata, errors="strict", errorcb=None,
              namecb=None, ignore_error=0):
     inleft = len(unicodedata)
-    inbuf = rffi.get_nonmoving_unicodebuffer(unicodedata)
-    try:
+    with rffi.scoped_nonmoving_unicodebuffer(unicodedata) as inbuf:
         if pypy_cjk_enc_init(encodebuf, inbuf, inleft) < 0:
             raise MemoryError
         if ignore_error == 0:
@@ -246,9 +238,6 @@
         src = pypy_cjk_enc_outbuf(encodebuf)
         length = pypy_cjk_enc_outlen(encodebuf)
         return rffi.charpsize2str(src, length)
-    #
-    finally:
-        rffi.free_nonmoving_unicodebuffer(unicodedata, inbuf)
 
 def multibytecodec_encerror(encodebuf, e, errors,
                             errorcb, namecb, unicodedata):
@@ -281,10 +270,7 @@
         assert errorcb
         replace, end = errorcb(errors, namecb, reason,
                                unicodedata, start, end)
-    inbuf = rffi.get_nonmovingbuffer(replace)
-    try:
+    with rffi.scoped_nonmovingbuffer(replace) as inbuf:
         r = pypy_cjk_enc_replace_on_error(encodebuf, inbuf, len(replace), end)
-    finally:
-        rffi.free_nonmovingbuffer(replace, inbuf)
     if r == MBERR_NOMEMORY:
         raise MemoryError
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -431,11 +431,8 @@
 
     count = space.len_w(w_str)
     data = space.str_w(w_str)
-    buf = rffi.get_nonmovingbuffer(data)
-    try:
+    with rffi.scoped_nonmovingbuffer(data) as buf:
         fwrite(buf, 1, count, fp)
-    finally:
-        rffi.free_nonmovingbuffer(data, buf)
     return 0
 
 
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -998,19 +998,15 @@
         """Send a data string to the socket.  For the optional flags
         argument, see the Unix manual.  Return the number of bytes
         sent; this may be less than len(data) if the network is busy."""
-        dataptr = rffi.get_nonmovingbuffer(data)
-        try:
+        with rffi.scoped_nonmovingbuffer(data) as dataptr:
             return self.send_raw(dataptr, len(data), flags)
-        finally:
-            rffi.free_nonmovingbuffer(data, dataptr)
 
     def sendall(self, data, flags=0, signal_checker=None):
         """Send a data string to the socket.  For the optional flags
         argument, see the Unix manual.  This calls send() repeatedly
         until all data is sent.  If an error occurs, it's impossible
         to tell how much data has been sent."""
-        dataptr = rffi.get_nonmovingbuffer(data)
-        try:
+        with rffi.scoped_nonmovingbuffer(data) as dataptr:
             remaining = len(data)
             p = dataptr
             while remaining > 0:
@@ -1023,8 +1019,6 @@
                         raise
                 if signal_checker:
                     signal_checker.check()
-        finally:
-            rffi.free_nonmovingbuffer(data, dataptr)
 
     def sendto(self, data, flags, address):
         """Like send(data, flags) but allows specifying the destination
@@ -1408,8 +1402,7 @@
             raise RSocketError("unknown address family")
         if len(packed) != srcsize:
             raise ValueError("packed IP wrong length for inet_ntop")
-        srcbuf = rffi.get_nonmovingbuffer(packed)
-        try:
+        with rffi.scoped_nonmovingbuffer(packed) as srcbuf:
             dstbuf = mallocbuf(dstsize)
             try:
                 res = _c.inet_ntop(family, srcbuf, dstbuf, dstsize)
@@ -1418,8 +1411,6 @@
                 return rffi.charp2str(res)
             finally:
                 lltype.free(dstbuf, flavor='raw')
-        finally:
-            rffi.free_nonmovingbuffer(packed, srcbuf)
 
 def setdefaulttimeout(timeout):
     if timeout < 0.0:
diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -1499,8 +1499,7 @@
     def unicode_encode_mbcs(p, size, errors, errorhandler=None):
         if size == 0:
             return ''
-        dataptr = rffi.get_nonmoving_unicodebuffer(p)
-        try:
+        with rffi.scoped_nonmoving_unicodebuffer(p) as dataptr:
             # first get the size of the result
             mbcssize = WideCharToMultiByte(CP_ACP, 0,
                                            dataptr, size, None, 0,
@@ -1519,5 +1518,3 @@
                 return rffi.str_from_buffer(raw_buf, gc_buf, mbcssize, mbcssize)
             finally:
                 rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
-        finally:
-            rffi.free_nonmoving_unicodebuffer(p, dataptr)
diff --git a/pypy/rlib/rzlib.py b/pypy/rlib/rzlib.py
--- a/pypy/rlib/rzlib.py
+++ b/pypy/rlib/rzlib.py
@@ -169,11 +169,8 @@
     Compute the CRC32 checksum of the string, possibly with the given
     start value, and return it as a unsigned 32 bit integer.
     """
-    bytes = rffi.get_nonmovingbuffer(string)
-    try:
+    with rffi.scoped_nonmovingbuffer(string) as bytes:
         checksum = _crc32(start, rffi.cast(Bytefp, bytes), len(string))
-    finally:
-        rffi.free_nonmovingbuffer(string, bytes)
     return checksum
 
 
@@ -184,11 +181,8 @@
     Compute the Adler-32 checksum of the string, possibly with the given
     start value, and return it as a unsigned 32 bit integer.
     """
-    bytes = rffi.get_nonmovingbuffer(string)
-    try:
+    with rffi.scoped_nonmovingbuffer(string) as bytes:
         checksum = _adler32(start, rffi.cast(Bytefp, bytes), len(string))
-    finally:
-        rffi.free_nonmovingbuffer(string, bytes)
     return checksum
 
 # ____________________________________________________________
diff --git a/pypy/rpython/lltypesystem/test/test_rffi.py b/pypy/rpython/lltypesystem/test/test_rffi.py
--- a/pypy/rpython/lltypesystem/test/test_rffi.py
+++ b/pypy/rpython/lltypesystem/test/test_rffi.py
@@ -521,7 +521,7 @@
                         counter += 1
                 return counter
             finally:
-                free_nonmovingbuffer(d, buf, is_pinned_is_raw)
+                free_nonmovingbuffer(d, buf, is_pinned, is_raw)
         assert f() == len(d)
         fn = self.compile(f, [], gcpolicy='ref')
         assert fn() == len(d)
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -180,7 +180,7 @@
         return False
 
     def pin(self, addr):
-        pass
+        return False
 
     def unpin(self, addr):
         pass
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -804,10 +804,15 @@
                 not self.header(obj).tid & GCFLAG_PINNED)
 
     def pin(self, obj):
+        if not self.is_in_nursery(obj):
+            return False # note that this should never happen since
+        # objects out of nursery should be guarded by can_move = False
+        # better check though.
         if self.pinned_objects_in_nursery >= self.max_number_of_pinned_objects:
-            return
+            return False
         self.pinned_objects_in_nursery += 1
         self.header(obj).tid |= GCFLAG_PINNED
+        return True
 
     def unpin(self, obj):
         if self.header(obj).tid & GCFLAG_PINNED:
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -439,7 +439,8 @@
                                             annmodel.SomeInteger(nonneg=True)],
                                            annmodel.s_None)
         self.pin_ptr = getfn(GCClass.pin,
-                             [s_gc, annmodel.SomeAddress()], annmodel.s_None)
+                             [s_gc, annmodel.SomeAddress()],
+                             annmodel.SomeBool())
         self.unpin_ptr = getfn(GCClass.unpin,
                                [s_gc, annmodel.SomeAddress()], annmodel.s_None)
 
@@ -743,7 +744,8 @@
 
     def gct_gc_pin(self, hop):
         op = hop.spaceop
-        hop.genop("direct_call", [self.pin_ptr, self.c_const_gc, op.args[0]])
+        hop.genop("direct_call", [self.pin_ptr, self.c_const_gc, op.args[0]],
+                  resultvar=op.result)
 
     def gct_gc_unpin(self, hop):
         op = hop.spaceop
diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -378,7 +378,10 @@
         pass
 
     def gct_gc_pin(self, hop):
-        pass
+        op = hop.spaceop
+        hop.genop("same_as",
+                  [rmodel.inputconst(lltype.Bool, False)],
+                  resultvar=op.result)
 
     def gct_gc_unpin(self, hop):
         pass

From noreply at buildbot.pypy.org  Mon Apr 23 12:32:11 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 12:32:11 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: a slight dance with
	keepalives
Message-ID: <20120423103211.693418203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54665:ad9787fabbde
Date: 2012-04-23 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/ad9787fabbde/

Log:	a slight dance with keepalives

diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -741,11 +741,11 @@
         # if 'buf' points inside 'data'.  This is only possible if we
         # followed the 2nd case in get_nonmovingbuffer(); in the first case,
         # 'buf' points to its own raw-malloced memory.
-        keepalive_until_here(data)
         if pinned:
             rgc.unpin(data)
-        elif is_raw:
+        if is_raw:
             lltype.free(buf, flavor='raw')
+        keepalive_until_here(data)
     free_nonmovingbuffer._annenforceargs_ = [strtype, None, bool, bool]
 
     # int -> (char*, str)

From noreply at buildbot.pypy.org  Mon Apr 23 12:48:25 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 12:48:25 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: fix the GC
Message-ID: <20120423104825.B7C7B8203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54666:96bd990a70e5
Date: 2012-04-23 12:47 +0200
http://bitbucket.org/pypy/pypy/changeset/96bd990a70e5/

Log:	fix the GC

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -800,8 +800,7 @@
 
     def can_move(self, obj):
         """Overrides the parent can_move()."""
-        return (self.is_in_nursery(obj) and
-                not self.header(obj).tid & GCFLAG_PINNED)
+        return not self.is_in_nursery(obj)
 
     def pin(self, obj):
         if not self.is_in_nursery(obj):
@@ -810,6 +809,8 @@
         # better check though.
         if self.pinned_objects_in_nursery >= self.max_number_of_pinned_objects:
             return False
+        if self.header(obj).tid | GCFLAG_PINNED:
+            return False
         self.pinned_objects_in_nursery += 1
         self.header(obj).tid |= GCFLAG_PINNED
         return True

From noreply at buildbot.pypy.org  Mon Apr 23 12:55:22 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 23 Apr 2012 12:55:22 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge from default
Message-ID: <20120423105522.4CAC08203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54667:9fa80f99cd1b
Date: 2012-04-23 13:54 +0300
http://bitbucket.org/pypy/pypy/changeset/9fa80f99cd1b/

Log:	merge from default

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -65,8 +65,9 @@
     check_num_args(space, w_args, 0)
     args_w = space.fixedview(w_args)
     res = generic_cpy_call(space, func_inquiry, w_self)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
     return space.wrap(bool(res))
 
 def wrap_getattr(space, w_self, w_args, func):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1836,6 +1836,13 @@
         a[a & 1 == 0] = 15
         assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
 
+    def test_array_indexing_bool_setitem_2(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a = a[::2]
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 5], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assert=plain -rf
+addopts = --assert=reinterp -rf
diff --git a/pypy/tool/pytest/pypy_test_failure_demo.py b/pypy/tool/pytest/pypy_test_failure_demo.py
--- a/pypy/tool/pytest/pypy_test_failure_demo.py
+++ b/pypy/tool/pytest/pypy_test_failure_demo.py
@@ -8,6 +8,10 @@
 def test_interp_func(space): 
     assert space.is_true(space.w_None) 
 
+def test_interp_reinterpret(space):
+    a = 1
+    assert a == 2
+
 class TestInterpTest: 
     def test_interp_method(self): 
         assert self.space.is_true(self.space.w_False) 

From noreply at buildbot.pypy.org  Mon Apr 23 13:05:31 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 13:05:31 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: oops
Message-ID: <20120423110531.687C78203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54668:046da1546e99
Date: 2012-04-23 12:52 +0200
http://bitbucket.org/pypy/pypy/changeset/046da1546e99/

Log:	oops

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -800,7 +800,7 @@
 
     def can_move(self, obj):
         """Overrides the parent can_move()."""
-        return not self.is_in_nursery(obj)
+        return self.is_in_nursery(obj)
 
     def pin(self, obj):
         if not self.is_in_nursery(obj):

From noreply at buildbot.pypy.org  Mon Apr 23 13:05:32 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 13:05:32 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: fixes
Message-ID: <20120423110532.AC2098203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54669:110160f44b5f
Date: 2012-04-23 13:01 +0200
http://bitbucket.org/pypy/pypy/changeset/110160f44b5f/

Log:	fixes

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -583,14 +583,16 @@
 
     def malloc_fixedsize_and_pin(self, typeid, size):
         r = self.malloc_fixedsize_clear(typeid, size)
-        self.pin(llmemory.cast_ptr_to_adr(r))
+        if not self.pin(llmemory.cast_ptr_to_adr(r)):
+            return lltype.nullptr(llmemory.GCREF.TO)
         return r
         
     def malloc_varsize_and_pin(self, typeid, length, size, itemsize,
                                offset_to_length):
         r = self.malloc_varsize_clear(typeid, length, size, itemsize,
                                       offset_to_length)
-        self.pin(llmemory.cast_ptr_to_adr(r))
+        if not self.pin(llmemory.cast_ptr_to_adr(r)):
+            return lltype.nullptr(llmemory.GCREF.TO)
         return r        
 
     def collect(self, gen=1):
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -4,9 +4,10 @@
 from pypy.rpython.memory import gcwrapper
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
-from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated, compute_unique_id
+from pypy.rlib.objectmodel import we_are_translated, compute_unique_id,\
+     keepalive_until_here
 from pypy.rlib import rgc
 from pypy.rlib.rstring import StringBuilder
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -596,8 +597,11 @@
         TP = lltype.GcArray(lltype.Char)
         def func():
             a = rgc.malloc_and_pin(TP, 3)
+            adr = llmemory.cast_ptr_to_adr(a)
             if a:
-                assert not rgc.can_move(a)
+                rgc.collect()
+                assert adr == llmemory.cast_ptr_to_adr(a)
+                keepalive_until_here(a)
                 return 1
             return 0
 

From noreply at buildbot.pypy.org  Mon Apr 23 13:05:33 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 13:05:33 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: grumble
Message-ID: <20120423110533.EB62D8203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54670:229c185224bc
Date: 2012-04-23 13:02 +0200
http://bitbucket.org/pypy/pypy/changeset/229c185224bc/

Log:	grumble

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -811,7 +811,7 @@
         # better check though.
         if self.pinned_objects_in_nursery >= self.max_number_of_pinned_objects:
             return False
-        if self.header(obj).tid | GCFLAG_PINNED:
+        if self.header(obj).tid & GCFLAG_PINNED:
             return False
         self.pinned_objects_in_nursery += 1
         self.header(obj).tid |= GCFLAG_PINNED

From noreply at buildbot.pypy.org  Mon Apr 23 13:22:11 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 13:22:11 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: pinned objects will remain
	in the nursery
Message-ID: <20120423112211.632DD8203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: gc-minimark-pinning
Changeset: r54671:79227902ef69
Date: 2012-04-23 13:21 +0200
http://bitbucket.org/pypy/pypy/changeset/79227902ef69/

Log:	pinned objects will remain in the nursery

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -957,11 +957,12 @@
 
     def debug_check_object(self, obj):
         # after a minor or major collection, no object should be in the nursery
-        ll_assert(not self.is_in_nursery(obj),
-                  "object in nursery after collection")
-        # similarily, all objects should have this flag:
-        ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS,
-                  "missing GCFLAG_TRACK_YOUNG_PTRS")
+        if not self.header(obj).tid & GCFLAG_PINNED:
+            ll_assert(not self.is_in_nursery(obj),
+                      "object in nursery after collection")
+            # similarily, all objects should have this flag:
+            ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS,
+                      "missing GCFLAG_TRACK_YOUNG_PTRS")
         # the GCFLAG_VISITED should not be set between collections
         ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
                   "unexpected GCFLAG_VISITED")

From noreply at buildbot.pypy.org  Mon Apr 23 13:29:02 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 13:29:02 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Test and fix.
Message-ID: <20120423112902.E27378203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54672:55e1db872e5a
Date: 2012-04-23 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/55e1db872e5a/

Log:	Test and fix.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -279,9 +279,13 @@
         def _stm_write_barrier_global(obj):
             tls = self.get_tls()
             if tls is self.main_thread_tls:
-                return obj        # not in transaction
+                # not in a transaction: the main thread writes to a global obj.
+                # In this case we turn the object local.
+                tls.main_thread_writes_to_global_obj(obj)
+                return obj
             #
-            # we need to find or make a local copy
+            # else, in a transaction: we find or make a local copy of the
+            # global object
             hdr = self.header(obj)
             if hdr.tid & GCFLAG_WAS_COPIED == 0:
                 #
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -113,6 +113,13 @@
                 if value is not self:
                     del StmGCTLS.nontranslated_dict[key]
         self.start_transaction()
+        # do something special here after we restarted the "transaction"
+        # in the main thread: we have to make sure that the write_barrier
+        # calls done in the main thread before enter/leave_transactional_mode
+        # are still pointing to local objects.  Conservatively, we mark as
+        # local all objects directly referenced from the stack.
+        self.gc.root_walker.walk_current_stack_roots(
+            StmGCTLS._remark_object_as_local, self)
 
     def start_transaction(self):
         """Start a transaction: performs any pending cleanups, and set
@@ -256,6 +263,16 @@
     def fresh_new_weakref(self, obj):
         self.local_weakrefs.append(obj)
 
+    def _remark_object_as_local(self, root):
+        self.main_thread_writes_to_global_obj(root.address[0])
+
+    def main_thread_writes_to_global_obj(self, obj):
+        hdr = self.gc.header(obj)
+        ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0,
+                  "write in main thread: unexpected GCFLAG_WAS_COPIED")
+        hdr.tid &= ~GCFLAG_GLOBAL
+        self.sharedarea_tls.add_regular(obj)
+
     # ------------------------------------------------------------
 
     def _promote_locals_to_globals(self):
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -336,7 +336,28 @@
         self.checkflags(t_adr, True, False)
         obj = self.gc.stm_writebarrier(t_adr)     # main thread, global:
         assert obj == t_adr                       # doesn't make a copy
-        #self.checkflags(obj, False, False) -- LATER check that it becomes local
+        self.checkflags(obj, False, False)        # but it becomes local
+
+    def test_relocalize_objects_after_transactional_mode(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        #
+        tr1, tr1_adr = self.malloc(SR, globl=True)
+        tr2, tr2_adr = self.malloc(SR, globl=True)
+        tr1.sr2 = tr2
+        self.gc.root_walker.current_stack = [tr1]
+        obj = self.gc.stm_writebarrier(tr1_adr)
+        assert obj == tr1_adr
+        obj = self.gc.stm_writebarrier(tr2_adr)
+        assert obj == tr2_adr
+        self.checkflags(tr1_adr, False, False)    # tr1 has become local
+        self.checkflags(tr2_adr, False, False)    # tr2 has become local
+        #
+        self.gc.enter_transactional_mode()
+        self.gc.leave_transactional_mode()
+        self.checkflags(tr2_adr, True, False)     # tr2 has become global again
+        # but tr1 is still local, because it is directly referenced from stack
+        self.checkflags(tr1_adr, False, False)
 
     def test_commit_transaction_empty(self):
         self.select_thread(1)

From noreply at buildbot.pypy.org  Mon Apr 23 13:29:04 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 13:29:04 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix test to run even if the working
 directory is different.
Message-ID: <20120423112904.2BEA48203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54673:3e95a592df1e
Date: 2012-04-23 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/3e95a592df1e/

Log:	Fix test to run even if the working directory is different.

diff --git a/pypy/translator/stm/test/test_stmgcintf.py b/pypy/translator/stm/test/test_stmgcintf.py
--- a/pypy/translator/stm/test/test_stmgcintf.py
+++ b/pypy/translator/stm/test/test_stmgcintf.py
@@ -1,14 +1,22 @@
 import os
+from pypy.tool import autopath
 from pypy.tool.udir import udir
 
 
 def test_all():
     executable = str(udir.join('test_stmgcintf'))
-    exitcode = os.system("gcc -g -o '%s' -pthread -I.. test_stmgcintf.c" % (
-        executable,))
-    assert exitcode == 0
+    prevdir = os.getcwd()
+    thisdir = os.path.join(autopath.pypydir, 'translator', 'stm', 'test')
+    try:
+        os.chdir(thisdir)
+        exitcode = os.system(
+            "gcc -g -o '%s' -pthread -I.. test_stmgcintf.c" % (
+            executable,))
+        assert exitcode == 0
+    finally:
+        os.chdir(prevdir)
     #
-    for line in open('test_stmgcintf.c'):
+    for line in open(os.path.join(thisdir, 'test_stmgcintf.c')):
         line = line.strip()
         if line.startswith('XTEST('):
             assert line.endswith(');')

From noreply at buildbot.pypy.org  Mon Apr 23 13:29:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 13:29:05 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: Fix(?) the issue of objects
 that are not found any more in
Message-ID: <20120423112905.6C3BC8203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-minimark-pinning
Changeset: r54674:27dcf8e6a03f
Date: 2012-04-23 13:28 +0200
http://bitbucket.org/pypy/pypy/changeset/27dcf8e6a03f/

Log:	Fix(?) the issue of objects that are not found any more in
	dictionaries when they are converted to ll2ctypes-style storage. The
	fix is to not use dictionaries any more.

diff --git a/pypy/rpython/lltypesystem/llarena.py b/pypy/rpython/lltypesystem/llarena.py
--- a/pypy/rpython/lltypesystem/llarena.py
+++ b/pypy/rpython/lltypesystem/llarena.py
@@ -1,4 +1,4 @@
-import array, weakref
+import array
 from pypy.rpython.lltypesystem import llmemory
 from pypy.rlib.rarithmetic import is_valid_int
 
@@ -16,8 +16,6 @@
     pass
 
 class Arena(object):
-    object_arena_location = {}     # {container: (arena, offset)}
-    old_object_arena_location = weakref.WeakKeyDictionary()
     _count_arenas = 0
 
     def __init__(self, nbytes, zero):
@@ -49,7 +47,7 @@
                 assert offset >= stop, "object overlaps cleared area"
             else:
                 obj = ptr._obj
-                _dictdel(Arena.object_arena_location, obj)
+                obj.__arena_location__[0] = False   # no longer valid
                 del self.objectptrs[offset]
                 del self.objectsizes[offset]
                 obj._free()
@@ -111,8 +109,7 @@
         self.objectptrs[offset] = objaddr.ptr
         self.objectsizes[offset] = bytes
         container = objaddr.ptr._obj
-        Arena.object_arena_location[container] = self, offset
-        Arena.old_object_arena_location[container] = self, offset
+        container.__arena_location__ = [True, self, offset]
 
     def shrink_obj(self, offset, newsize):
         oldbytes = self.objectsizes[offset]
@@ -203,12 +200,12 @@
             return None, None
         obj = other.ptr._obj
         innerobject = False
-        while obj not in Arena.object_arena_location:
+        while not getattr(obj, '__arena_location__', (False,))[0]:
             obj = obj._parentstructure()
             if obj is None:
                 return None, None     # not found in the arena
             innerobject = True
-        arena, offset = Arena.object_arena_location[obj]
+        _, arena, offset = obj.__arena_location__
         if innerobject:
             # 'obj' is really inside the object allocated from the arena,
             # so it's likely that its address "should be" a bit larger than
@@ -272,8 +269,8 @@
 def _oldobj_to_address(obj):
     obj = obj._normalizedcontainer(check=False)
     try:
-        arena, offset = Arena.old_object_arena_location[obj]
-    except KeyError:
+        _, arena, offset = obj.__arena_location__
+    except AttributeError:
         if obj._was_freed():
             msg = "taking address of %r, but it was freed"
         else:
@@ -281,16 +278,6 @@
         raise RuntimeError(msg % (obj,))
     return arena.getaddr(offset)
 
-def _dictdel(d, key):
-    # hack
-    try:
-        del d[key]
-    except KeyError:
-        items = d.items()
-        d.clear()
-        d.update(items)
-        del d[key]
-
 class RoundedUpForAllocation(llmemory.AddressOffset):
     """A size that is rounded up in order to preserve alignment of objects
     following it.  For arenas containing heterogenous objects.
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1570,7 +1570,7 @@
         return cache[tag]
     except KeyError:
         class _struct1(_struct):
-            __slots__ = flds
+            __slots__ = tag + ('__arena_location__',)
         cache[tag] = _struct1
         return _struct1
  

From noreply at buildbot.pypy.org  Mon Apr 23 13:29:47 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 13:29:47 +0200 (CEST)
Subject: [pypy-commit] pypy gc-minimark-pinning: Fix this too.
Message-ID: <20120423112947.B8B658203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gc-minimark-pinning
Changeset: r54675:3e91f895ea62
Date: 2012-04-23 13:29 +0200
http://bitbucket.org/pypy/pypy/changeset/3e91f895ea62/

Log:	Fix this too.

diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1656,7 +1656,7 @@
 class _array(_parentable):
     _kind = "array"
 
-    __slots__ = ('items',)
+    __slots__ = ('items', '__arena_location__')
 
     def __init__(self, TYPE, n, initialization=None, parent=None, parentindex=None):
         if not is_valid_int(n):

From noreply at buildbot.pypy.org  Mon Apr 23 13:53:25 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Apr 2012 13:53:25 +0200 (CEST)
Subject: [pypy-commit] pypy faster-heapcache: improve the speed of
	replace_box
Message-ID: <20120423115325.C616B8203C@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-heapcache
Changeset: r54676:38411f3cfc02
Date: 2012-04-23 13:53 +0200
http://bitbucket.org/pypy/pypy/changeset/38411f3cfc02/

Log:	improve the speed of replace_box

	after some profiling, it turns out that replace_box takes about 50%
	of the time spent in the JIT :-(.

	In general, the heap cache is quite time-intensive, maybe something
	needs to be done about that.

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -20,6 +20,7 @@
         self.dependencies = {}
         # contains frame boxes that are not virtualizables
         self.nonstandard_virtualizables = {}
+
         # heap cache
         # maps descrs to {from_box, to_box} dicts
         self.heap_cache = {}
@@ -29,6 +30,26 @@
         # cache the length of arrays
         self.length_cache = {}
 
+        # replace_box is called surprisingly often, therefore it's not efficient
+        # to go over all the dicts and fix them.
+        # instead, these two dicts are kept, and a replace_box adds an entry to
+        # each of them.
+        # every time one of the dicts heap_cache, heap_array_cache, length_cache
+        # is accessed, suitable indirections need to be performed
+
+        # this looks all very subtle, but in practice the patterns of
+        # replacements should not be that complex. Usually a box is replaced by
+        # a const, once. Also, if something goes wrong, the effect is that less
+        # caching than possible is done, which is not a huge problem.
+        self.input_indirections = {}
+        self.output_indirections = {}
+
+    def _input_indirection(self, box):
+        return self.input_indirections.get(box, box)
+
+    def _output_indirection(self, box):
+        return self.output_indirections.get(box, box)
+
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, argboxes)
         self.clear_caches(opnum, descr, argboxes)
@@ -132,14 +153,16 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
+        box = self._input_indirection(box)
         d = self.heap_cache.get(descr, None)
         if d:
             tobox = d.get(box, None)
-            if tobox:
-                return tobox
+            return self._output_indirection(tobox)
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         self.heap_cache.setdefault(descr, {})[box] = fieldbox
 
     def setfield(self, box, descr, fieldbox):
@@ -148,6 +171,8 @@
         self.heap_cache[descr] = new_d
 
     def _do_write_with_aliasing(self, d, box, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         # slightly subtle logic here
         # a write to an arbitrary box, all other boxes can alias this one
         if not d or box not in self.new_boxes:
@@ -166,6 +191,7 @@
         return new_d
 
     def getarrayitem(self, box, descr, indexbox):
+        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -173,9 +199,11 @@
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return indexcache.get(box, None)
+                return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        box = self._input_indirection(box)
+        valuebox = self._input_indirection(valuebox)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -198,25 +226,13 @@
         cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
 
     def arraylen(self, box):
-        return self.length_cache.get(box, None)
+        box = self._input_indirection(box)
+        return self._output_indirection(self.length_cache.get(box, None))
 
     def arraylen_now_known(self, box, lengthbox):
-        self.length_cache[box] = lengthbox
-
-    def _replace_box(self, d, oldbox, newbox):
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            if frombox is oldbox:
-                frombox = newbox
-            if tobox is oldbox:
-                tobox = newbox
-            new_d[frombox] = tobox
-        return new_d
+        box = self._input_indirection(box)
+        self.length_cache[box] = self._input_indirection(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        for descr, d in self.heap_cache.iteritems():
-            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
-        for descr, d in self.heap_array_cache.iteritems():
-            for index, cache in d.iteritems():
-                d[index] = self._replace_box(cache, oldbox, newbox)
-        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
+        self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox)
+        self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox)
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -2,12 +2,14 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp.history import ConstInt
 
-box1 = object()
-box2 = object()
-box3 = object()
-box4 = object()
+box1 = "box1"
+box2 = "box2"
+box3 = "box3"
+box4 = "box4"
+box5 = "box5"
 lengthbox1 = object()
 lengthbox2 = object()
+lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -276,11 +278,43 @@
         h.setfield(box1, descr2, box3)
         h.setfield(box2, descr3, box3)
         h.replace_box(box1, box4)
-        assert h.getfield(box1, descr1) is None
-        assert h.getfield(box1, descr2) is None
         assert h.getfield(box4, descr1) is box2
         assert h.getfield(box4, descr2) is box3
         assert h.getfield(box2, descr3) is box3
+        h.setfield(box4, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box4
+        assert h.getfield(box2, descr3) is box4
+
+    def test_replace_box_twice(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box5, descr1) is box2
+        assert h.getfield(box5, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+        h.setfield(box5, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box5
+        assert h.getfield(box2, descr3) is box5
 
     def test_replace_box_array(self):
         h = HeapCache()
@@ -291,9 +325,6 @@
         h.setarrayitem(box3, descr2, index2, box1)
         h.setarrayitem(box2, descr3, index2, box3)
         h.replace_box(box1, box4)
-        assert h.getarrayitem(box1, descr1, index1) is None
-        assert h.getarrayitem(box1, descr2, index1) is None
-        assert h.arraylen(box1) is None
         assert h.arraylen(box4) is lengthbox1
         assert h.getarrayitem(box4, descr1, index1) is box2
         assert h.getarrayitem(box4, descr2, index1) is box3
@@ -304,6 +335,27 @@
         h.replace_box(lengthbox1, lengthbox2)
         assert h.arraylen(box4) is lengthbox2
 
+    def test_replace_box_array_twice(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box5, descr1, index1) is box2
+        assert h.getarrayitem(box5, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box5
+        assert h.getarrayitem(box3, descr2, index2) is box5
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        h.replace_box(lengthbox2, lengthbox3)
+        assert h.arraylen(box4) is lengthbox3
+
     def test_ll_arraycopy(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)

From noreply at buildbot.pypy.org  Mon Apr 23 14:29:13 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 23 Apr 2012 14:29:13 +0200 (CEST)
Subject: [pypy-commit] pypy faster-heapcache: merge
Message-ID: <20120423122913.8E1058203C@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-heapcache
Changeset: r54677:6939635a3b69
Date: 2012-04-23 14:28 +0200
http://bitbucket.org/pypy/pypy/changeset/6939635a3b69/

Log:	merge

diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
 try:
     from thread import _local as local
 except ImportError:
-    local = object    # no threads
+    class local(object):    # no threads
+        pass
 
 class ConvMode:
     encoding = 'ascii'
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -21,7 +21,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC']
     res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
@@ -34,6 +34,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST'] # needed for VC10
     else:
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -16,7 +16,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
     res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
@@ -29,6 +29,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
                         '/EXPORT:init_testcapi']
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -229,8 +229,8 @@
                     return thing
                 elif hasattr(thing, '__name__'): # mostly types and functions
                     return thing.__name__
-                elif hasattr(thing, 'name'): # mostly ClassDescs
-                    return thing.name
+                elif hasattr(thing, 'name') and isinstance(thing.name, str):
+                    return thing.name            # mostly ClassDescs
                 elif isinstance(thing, tuple):
                     return '_'.join(map(nameof, thing))
                 else:
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/cppyy.rst
@@ -0,0 +1,333 @@
+============================
+cppyy: C++ bindings for PyPy
+============================
+
+The cppyy module provides C++ bindings for PyPy by using the reflection
+information extracted from C++ header files by means of the
+`Reflex package`_.
+For this to work, you have to both install Reflex and build PyPy from the
+reflex-support branch.
+As indicated by this being a branch, support for Reflex is still
+experimental.
+However, it is functional enough to put it in the hands of those who want
+to give it a try.
+In the medium term, cppyy will move away from Reflex and instead use
+`cling`_ as its backend, which is based on `llvm`_.
+Although that will change the logistics on the generation of reflection
+information, it will not change the python-side interface.
+
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`cling`: http://root.cern.ch/drupal/content/cling
+.. _`llvm`: http://llvm.org/
+
+
+Installation
+============
+
+For now, the easiest way of getting the latest version of Reflex, is by
+installing the ROOT package.
+Besides getting the latest version of Reflex, another advantage is that with
+the full ROOT package, you can also use your Reflex-bound code on `CPython`_.
+`Download`_ a binary or install from `source`_.
+Some Linux and Mac systems may have ROOT provided in the list of scientific
+software of their packager.
+A current, standalone version of Reflex should be provided at some point,
+once the dependencies and general packaging have been thought out.
+Also, make sure you have a version of `gccxml`_ installed, which is most
+easily provided by the packager of your system.
+If you read up on gccxml, you'll probably notice that it is no longer being
+developed and hence will not provide C++11 support.
+That's why the medium term plan is to move to `cling`_.
+
+.. _`Download`: http://root.cern.ch/drupal/content/downloading-root
+.. _`source`: http://root.cern.ch/drupal/content/installing-root-source
+.. _`gccxml`: http://www.gccxml.org
+
+Next, get the `PyPy sources`_, select the reflex-support branch, and build
+pypy-c.
+For the build to succeed, the ``$ROOTSYS`` environment variable must point to
+the location of your ROOT installation::
+
+    $ hg clone https://bitbucket.org/pypy/pypy
+    $ cd pypy
+    $ hg up reflex-support
+    $ cd pypy/translator/goal
+    $ python translate.py -O jit --gcrootfinder=shadowstack targetpypystandalone.py --withmod-cppyy
+
+This will build a ``pypy-c`` that includes the cppyy module, and through that,
+Reflex support.
+Of course, if you already have a pre-built version of the ``pypy`` interpreter,
+you can use that for the translation rather than ``python``.
+
+.. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
+
+
+Basic example
+=============
+
+Now test with a trivial example whether all packages are properly installed
+and functional.
+First, create a C++ header file with some class in it (note that all functions
+are made inline for convenience; a real-world example would of course have a
+corresponding source file)::
+
+    $ cat MyClass.h
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_myint(i) {}
+
+        int GetMyInt() { return m_myint; }
+        void SetMyInt(int i) { m_myint = i; }
+
+    public:
+       int m_myint;
+    };
+
+Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
+code::
+
+    $ genreflex MyClass.h
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+
+Now you're ready to use the bindings.
+Since the bindings are designed to look pythonistic, it should be
+straightforward::
+
+    $ pypy-c
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libMyClassDict.so")
+    <CPPLibrary object at 0xb6fd7c4c>
+    >>>> myinst = cppyy.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> myinst.SetMyInt(33)
+    >>>> print myinst.m_myint
+    33
+    >>>> myinst.m_myint = 77
+    >>>> print myinst.GetMyInt()
+    77
+    >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
+
+That's all there is to it!
+
+
+Features
+========
+
+The following is not meant to be an exhaustive list, since cppyy is still
+under active development.
+Furthermore, the intention is that every feature is as natural as possible on
+the python side, so if you find something missing in the list below, simply
+try it out.
+It is not always possible to provide exact mapping between python and C++
+(active memory management is one such case), but by and large, if the use of a
+feature does not strike you as obvious, it is more likely to simply be a bug.
+That is a strong statement to make, but also a worthy goal.
+
+* **abstract classes**: Are represented as python classes, since they are
+  needed to complete the inheritance hierarchies, but will raise an exception
+  if an attempt is made to instantiate from them.
+
+* **arrays**: Supported for builtin data types only, as used from module
+  ``array``.
+  Out-of-bounds checking is limited to those cases where the size is known at
+  compile time (and hence part of the reflection info).
+
+* **builtin data types**: Map onto the expected equivalent python types, with
+  the caveat that there may be size differences, and thus it is possible that
+  exceptions are raised if an overflow is detected.
+
+* **casting**: Is supposed to be unnecessary.
+  Object pointer returns from functions provide the most derived class known
+  in the hierarchy of the object being returned.
+  This is important to preserve object identity as well as to make casting,
+  a pure C++ feature after all, superfluous.
+
+* **classes and structs**: Get mapped onto python classes, where they can be
+  instantiated as expected.
+  If classes are inner classes or live in a namespace, their naming and
+  location will reflect that.
+
+* **data members**: Public data members are represented as python properties
+  and provide read and write access on instances as expected.
+
+* **default arguments**: C++ default arguments work as expected, but python
+  keywords are not supported.
+  It is technically possible to support keywords, but for the C++ interface,
+  the formal argument names have no meaning and are not considered part of the
+  API, hence it is not a good idea to use keywords.
+
+* **doc strings**: The doc string of a method or function contains the C++
+  arguments and return types of all overloads of that name, as applicable.
+
+* **functions**: Work as expected and live in their appropriate namespace
+  (which can be the global one, ``cppyy.gbl``).
+
+* **inheritance**: All combinations of inheritance on the C++ (single,
+  multiple, virtual) are supported in the binding.
+  However, new python classes can only use single inheritance from a bound C++
+  class.
+  Multiple inheritance would introduce two "this" pointers in the binding.
+  This is a current, not a fundamental, limitation.
+  The C++ side will not see any overridden methods on the python side, as
+  cross-inheritance is planned but not yet supported.
+
+* **methods**: Are represented as python methods and work as expected.
+  They are first class objects and can be bound to an instance.
+  Virtual C++ methods work as expected.
+  To select a specific virtual method, do like with normal python classes
+  that override methods: select it from the class that you need, rather than
+  calling the method on the instance.
+
+* **namespaces**: Are represented as python classes.
+  Namespaces are more open-ended than classes, so sometimes initial access may
+  result in updates as data and functions are looked up and constructed
+  lazily.
+  Thus the result of ``dir()`` on a namespace should not be relied upon: it
+  only shows the already accessed members. (TODO: to be fixed by implementing
+  __dir__.)
+  The global namespace is ``cppyy.gbl``.
+
+* **operator conversions**: If defined in the C++ class and a python
+  equivalent exists (i.e. all builtin integer and floating point types, as well
+  as ``bool``), it will map onto that python conversion.
+  Note that ``char*`` is mapped onto ``__str__``.
+
+* **operator overloads**: If defined in the C++ class and if a python
+  equivalent is available (not always the case, think e.g. of ``operator||``),
+  then they work as expected.
+  Special care needs to be taken for global operator overloads in C++: first,
+  make sure that they are actually reflected, especially for the global
+  overloads for ``operator==`` and ``operator!=`` of STL iterators in the case
+  of gcc.
+  Second, make sure that reflection info is loaded in the proper order.
+  I.e. that these global overloads are available before use.
+
+* **pointers**: For builtin data types, see arrays.
+  For objects, a pointer to an object and an object looks the same, unless
+  the pointer is a data member.
+  In that case, assigning to the data member will cause a copy of the pointer
+  and care should be taken about the object's life time.
+  If a pointer is a global variable, the C++ side can replace the underlying
+  object and the python side will immediately reflect that.
+
+* **static data members**: Are represented as python property objects on the
+  class and the meta-class.
+  Both read and write access is as expected.
+
+* **static methods**: Are represented as python's ``staticmethod`` objects
+  and can be called both from the class as well as from instances.
+
+* **strings**: The std::string class is considered a builtin C++ type and
+  mixes quite well with python's str.
+  Python's str can be passed where a ``const char*`` is expected, and an str
+  will be returned if the return type is ``const char*``.
+
+* **templated classes**: Are represented in a meta-class style in python.
+  This looks a little bit confusing, but conceptually is rather natural.
+  For example, given the class ``std::vector<int>``, the meta-class part would
+  be ``std.vector`` in python.
+  Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
+  create an instance of that class, do ``std.vector(int)()``.
+  Note that templates can be build up by handing actual types to the class
+  instantiation (as done in this vector example), or by passing in the list of
+  template arguments as a string.
+  The former is a lot easier to work with if you have template instantiations
+  using classes that themselves are templates (etc.) in the arguments.
+  All classes must already exist in the loaded reflection info.
+
+* **unary operators**: Are supported if a python equivalent exists, and if the
+  operator is defined in the C++ class.
+
+You can always find more detailed examples and see the full of supported
+features by looking at the tests in pypy/module/cppyy/test.
+
+If a feature or reflection info is missing, this is supposed to be handled
+gracefully.
+In fact, there are unit tests explicitly for this purpose (even as their use
+becomes less interesting over time, as the number of missing features
+decreases).
+Only when a missing feature is used, should there be an exception.
+For example, if no reflection info is available for a return type, then a
+class that has a method with that return type can still be used.
+Only that one specific method can not be used.
+
+
+The fast lane
+=============
+
+The following is an experimental feature of cppyy, and that makes it doubly
+experimental, so caveat emptor.
+With a slight modification of Reflex, it can provide function pointers for
+C++ methods, and hence allow PyPy to call those pointers directly, rather than
+calling C++ through a Reflex stub.
+This results in a rather significant speed-up.
+Mind you, the normal stub path is not exactly slow, so for now only use this
+out of curiosity or if you really need it.
+
+To install this patch of Reflex, locate the file genreflex-methptrgetter.patch
+in pypy/module/cppyy and apply it to the genreflex python scripts found in
+``$ROOTSYS/lib``::
+
+    $ cd $ROOTSYS/lib
+    $ patch -p2 < genreflex-methptrgetter.patch
+
+With this patch, ``genreflex`` will have grown the ``--with-methptrgetter``
+option.
+Use this option when running ``genreflex``, and add the
+``-Wno-pmf-conversions`` option to ``g++`` when compiling.
+The rest works the same way: the fast path will be used transparently (which
+also means that you can't actually find out whether it is in use, other than
+by running a micro-benchmark).
+
+
+CPython
+=======
+
+Most of the ideas in cppyy come originally from the `PyROOT`_ project.
+Although PyROOT does not support Reflex directly, it has an alter ego called
+"PyCintex" that, in a somewhat roundabout way, does.
+If you installed ROOT, rather than just Reflex, PyCintex should be available
+immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment
+variable.
+
+.. _`PyROOT`: http://root.cern.ch/drupal/content/pyroot
+
+There are a couple of minor differences between PyCintex and cppyy, most to do
+with naming.
+The one that you will run into directly, is that PyCintex uses a function
+called ``loadDictionary`` rather than ``load_reflection_info``.
+The reason for this is that Reflex calls the shared libraries that contain
+reflection info "dictionaries."
+However, in python, the name `dictionary` already has a well-defined meaning,
+so a more descriptive name was chosen for cppyy.
+In addition, PyCintex requires that the names of shared libraries so loaded
+start with "lib" in their name.
+The basic example above, rewritten for PyCintex thus goes like this::
+
+    $ python
+    >>> import PyCintex
+    >>> PyCintex.loadDictionary("libMyClassDict.so")
+    >>> myinst = PyCintex.gbl.MyClass(42)
+    >>> print myinst.GetMyInt()
+    42
+    >>> myinst.SetMyInt(33)
+    >>> print myinst.m_myint
+    33
+    >>> myinst.m_myint = 77
+    >>> print myinst.GetMyInt()
+    77
+    >>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
+
+Other naming differences are such things as taking an address of an object.
+In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
+made to follow the naming as in ``ctypes`` and hence use ``addressof``
+(PyROOT/PyCintex predate ``ctypes`` by several years, and the ROOT project
+follows camel-case, hence the differences).
+
+Of course, this is python, so if any of the naming is not to your liking, all
+you have to do is provide a wrapper script that you import instead of
+importing the ``cppyy`` or ``PyCintex`` modules directly.
+In that wrapper script you can rename methods exactly the way you need it.
+
+In the cling world, all these differences will be resolved.
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -23,6 +23,8 @@
 
 * Write them in RPython as mixedmodule_, using *rffi* as bindings.
 
+* Write them in C++ and bind them through Reflex_ (EXPERIMENTAL)
+
 .. _ctypes: #CTypes
 .. _\_ffi: #LibFFI
 .. _mixedmodule: #Mixed Modules
@@ -110,3 +112,34 @@
 
 XXX we should provide detailed docs about lltype and rffi, especially if we
     want people to follow that way.
+
+Reflex
+======
+
+This method is only experimental for now, and is being exercised on a branch,
+`reflex-support`_, so you will have to build PyPy yourself.
+The method works by using the `Reflex package`_ to provide reflection
+information of the C++ code, which is then used to automatically generate
+bindings at runtime, which can then be used from python.
+Full details are `available here`_.
+
+.. _`reflex-support`: cppyy.html
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`available here`: cppyy.html
+
+Pros
+----
+
+If it works, it is mostly automatic, and hence easy in use.
+The bindings can make use of direct pointers, in which case the calls are
+very fast.
+
+Cons
+----
+
+C++ is a large language, and these bindings are not yet feature-complete.
+Although missing features should do no harm if you don't use them, if you do
+need a particular feature, it may be necessary to work around it in python
+or with a C++ helper function.
+Although Reflex works on various platforms, the bindings with PyPy have only
+been tested on Linux.
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -49,7 +49,9 @@
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
         # maybe we want this shorter:
-        name = getattr(self, 'name', '?')
+        name = getattr(self, 'name', None)
+        if not isinstance(name, str):
+            name = '?'
         return "<%s %s>" % (self.__class__.__name__, name)
 
     def call_args(self, args):
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1656,15 +1656,21 @@
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
-            #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower 16 bits read there
-            #   - multiply by 4 and use it as an offset in type_info_group
-            #   - add 16 bytes, to go past the TYPE_INFO structure
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0;
+            #     this is a complete word (N=4 bytes on 32-bit, N=8 on
+            #     64-bits)
+            #   - keep the lower half of what is read there (i.e.
+            #     truncate to an unsigned 'N / 2' bytes value)
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
             loc = locs[1]
             assert isinstance(loc, ImmedLoc)
             classptr = loc.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the 16 bits in the object header
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 or 4 on a big-endian machine.
             from pypy.rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
     space.call_method(w_list, "reverse")
     return 0
 
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+    """Return a list of the objects in list containing the objects between low
+    and high.  Return NULL and set an exception if unsuccessful.  Analogous
+    to list[low:high].  Negative indices, as when slicing from Python, are not
+    supported."""
+    w_start = space.wrap(low)
+    w_stop = space.wrap(high)
+    return space.getslice(w_list, w_start, w_stop)
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyList_SetSlice(space, w_list, low, high, w_sequence):
     """Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
     This is the equivalent of the Python expression hash(o)."""
     return space.int_w(space.hash(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+    """Set a TypeError indicating that type(o) is not hashable and return -1.
+    This function receives special treatment when stored in a tp_hash slot,
+    allowing a type to explicitly indicate to the interpreter that it is not
+    hashable.
+    """
+    raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Dir(space, w_o):
     """This is equivalent to the Python expression dir(o), returning a (possibly
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
-    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc)
@@ -60,6 +60,16 @@
     args_w = space.fixedview(w_args)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
 
+def wrap_inquirypred(space, w_self, w_args, func):
+    func_inquiry = rffi.cast(inquiry, func)
+    check_num_args(space, w_args, 0)
+    args_w = space.fixedview(w_args)
+    res = generic_cpy_call(space, func_inquiry, w_self)
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
+    return space.wrap(bool(res))
+
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
     check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
         w_errors = space.wrap(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+    """Decode a string object by passing it to the codec registered
+    for encoding and return the result as Python object. encoding and
+    errors have the same meaning as the parameters of the same name in
+    the string encode() method.  The codec to be used is looked up
+    using the Python codec registry. Return NULL if an exception was
+    raised by the codec.
+
+    This function is not available in 3.x and does not have a PyBytes alias."""
+    if not PyString_Check(space, w_str):
+        PyErr_BadArgument(space)
+
+    w_encoding = w_errors = space.w_None
+    if encoding:
+        w_encoding = space.wrap(rffi.charp2str(encoding))
+    if errors:
+        w_errors = space.wrap(rffi.charp2str(errors))
+    return space.call_method(w_str, "decode", w_encoding, w_errors)
+
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
-    """Return a list of the objects in list containing the objects between low
-    and high.  Return NULL and set an exception if unsuccessful.  Analogous
-    to list[low:high].  Negative indices, as when slicing from Python, are not
-    supported.
-
-    This function used an int for low and high. This might
-    require changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([Py_ssize_t], PyObject)
 def PyLong_FromSsize_t(space, v):
     """Return a new PyLongObject object from a C Py_ssize_t, or
@@ -1606,15 +1595,6 @@
     for PyObject_Str()."""
     raise NotImplementedError
 
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
-    """Set a TypeError indicating that type(o) is not hashable and return -1.
-    This function receives special treatment when stored in a tp_hash slot,
-    allowing a type to explicitly indicate to the interpreter that it is not
-    hashable.
-    """
-    raise NotImplementedError
-
 @cpython_api([], PyFrameObject)
 def PyEval_GetFrame(space):
     """Return the current thread state's frame, which is NULL if no frame is
@@ -1737,17 +1717,6 @@
     changes in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
-    """Decode a string object by passing it to the codec registered for encoding and
-    return the result as Python object. encoding and errors have the same
-    meaning as the parameters of the same name in the string encode() method.
-    The codec to be used is looked up using the Python codec registry. Return NULL
-    if an exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Encode(space, s, size, encoding, errors):
     """Encode the char buffer of the given size by passing it to the codec
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
         w_t = api.PyList_AsTuple(w_l)
         assert space.unwrap(w_t) == (3, 2, 1)
 
+    def test_list_getslice(self, space, api):
+        w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+        w_s = api.PyList_GetSlice(w_l, 1, 5)
+        assert space.unwrap(w_s) == [2, 1]
+
 class AppTestListObject(AppTestCpythonExtensionBase):
     def test_listobject(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
+    def test_AsDecodedObject(self, space, api):
+        w_str = space.wrap('caf\xe9')
+        encoding = rffi.str2charp("latin-1")
+        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        rffi.free_charp(encoding)
+        assert space.unwrap(w_res) == u"caf\xe9"
+
     def test_eq(self, space, api):
         assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
         assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
+        
+    def test_bool(self):
+        module = self.import_extension('foo', [
+            ("newInt", "METH_VARARGS",
+             """
+                IntLikeObject *intObj;
+                long intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "i", &intval))
+                    return NULL;
+
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+                intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+                if (!intObj) {
+                    return NULL;
+                }
+
+                intObj->value = intval;
+                return (PyObject *)intObj;
+             """)],
+            """
+            typedef struct
+            {
+                PyObject_HEAD
+                int value;
+            } IntLikeObject;
+
+            static int
+            intlike_nb_nonzero(IntLikeObject *v)
+            {
+                if (v->value == -42) {
+                    PyErr_SetNone(PyExc_ValueError);
+                    return -1;
+                }
+                return v->value;
+            }
+
+            PyTypeObject IntLike_Type = {
+                PyObject_HEAD_INIT(0)
+                /*ob_size*/             0,
+                /*tp_name*/             "IntLike",
+                /*tp_basicsize*/        sizeof(IntLikeObject),
+            };
+            static PyNumberMethods intlike_as_number;
+            """)
+        assert not bool(module.newInt(0))
+        assert bool(module.newInt(1))
+        assert bool(module.newInt(-1))
+        raises(ValueError, bool, module.newInt(-42))
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,10 +453,22 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        # prefix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        # suffix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        # type checks
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+        self.raises(space, api, TypeError,
+                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
+                    2, 10, 1)
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
@@ -30,6 +31,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -10,6 +10,7 @@
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
      scalar_w, W_NDimArray, array)
+from pypy.module.micronumpy.interp_arrayops import where
 from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize, instantiate
 
@@ -35,6 +36,7 @@
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring"]
 TWO_ARG_FUNCTIONS = ["dot", 'take']
+THREE_ARG_FUNCTIONS = ['where']
 
 class FakeSpace(object):
     w_ValueError = None
@@ -445,14 +447,25 @@
             arg = self.args[1].execute(interp)
             if not isinstance(arg, BaseArray):
                 raise ArgumentNotAnArray
-            if not isinstance(arg, BaseArray):
-                raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
             else:
                 assert False # unreachable code
+        elif self.name in THREE_ARG_FUNCTIONS:
+            if len(self.args) != 3:
+                raise ArgumentMismatch
+            arg1 = self.args[1].execute(interp)
+            arg2 = self.args[2].execute(interp)
+            if not isinstance(arg1, BaseArray):
+                raise ArgumentNotAnArray
+            if not isinstance(arg2, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "where":
+                w_res = where(interp.space, arr, arg1, arg2)
+            else:
+                assert False
         else:
             raise WrongFunctionName
         if isinstance(w_res, BaseArray):
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, space, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -1583,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,16 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -270,3 +270,13 @@
         b -> 2
         """)
         assert interp.results[0].value == 3
+
+    def test_where(self):
+        interp = self.run('''
+        a = [1, 0, 3, 0]
+        b = [1, 1, 1, 1]
+        c = [0, 0, 0, 0]
+        d = where(a, b, c)
+        d -> 1
+        ''')
+        assert interp.results[0].value == 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
@@ -1831,6 +1830,19 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
+    def test_array_indexing_bool_setitem_multidim(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
+
+    def test_array_indexing_bool_setitem_2(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a = a[::2]
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 5], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,6 @@
 
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_ufunc_instance(self):
         from _numpypy import add, ufunc
@@ -149,7 +148,11 @@
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+        # Since comparisons with nnan and nan all return false,
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
+
 
     def test_fmin(self):
         from _numpypy import fmin
@@ -165,7 +168,9 @@
         assert math.isnan(fmin(0, nan))
         assert math.isnan(fmin(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
     def test_fmod(self):
         from _numpypy import fmod
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -596,7 +596,7 @@
         import sys
         size = 0x14FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
@@ -618,7 +618,7 @@
         import sys
         size = 0x17FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_exception.py b/pypy/module/pypyjit/test_pypy_c/test_exception.py
--- a/pypy/module/pypyjit/test_pypy_c/test_exception.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_exception.py
@@ -91,3 +91,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_continue_in_finally(self):
+        # check that 'continue' inside a try:finally: block is correctly
+        # detected as closing a loop
+        py.test.skip("is this case important?")
+        def f(n):
+            i = 0
+            while 1:
+                try:
+                    if i < n:
+                        continue
+                finally:
+                    i += 1
+                return i
+
+        log = self.run(f, [2000])
+        assert log.result == 2001
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i40 = int_add_ovf(i31, 1)
+            guard_no_overflow(descr=...)
+            i41 = int_lt(i40, i33)
+            guard_true(i41, descr=...)
+            --TICK--
+            jump(..., descr=...)
+        """)
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -378,7 +378,10 @@
     __slots__ = ()
 
     def __repr__(self):
-        s = '%s(%s)' % (self.__class__.__name__, getattr(self, 'name', ''))
+        name = getattr(self, 'name', '')
+        if not isinstance(name, str):
+            name = ''
+        s = '%s(%s)' % (self.__class__.__name__, name)
         w_cls = getattr(self, 'w__class__', None)
         if w_cls is not None and w_cls is not self:
             s += ' instance of %s' % self.w__class__
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,12 +295,11 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assert=plain -rf
+addopts = --assert=reinterp -rf
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -40,7 +40,7 @@
 # In that case, do 5 bits at a time.  The potential drawback is that
 # a table of 2**5 intermediate results is computed.
 
-## FIVEARY_CUTOFF = 8   disabled for now
+FIVEARY_CUTOFF = 8
 
 
 def _mask_digit(x):
@@ -456,7 +456,7 @@
 
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
-        if 1:   ## b.numdigits() <= FIVEARY_CUTOFF:
+        if b.numdigits() <= FIVEARY_CUTOFF:
             # Left-to-right binary exponentiation (HAC Algorithm 14.79)
             # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
             i = b.numdigits() - 1
@@ -469,30 +469,51 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 i -= 1
-##        else:
-##            This code is disabled for now, because it assumes that
-##            SHIFT is a multiple of 5.  It could be fixed but it looks
-##            like it's more troubles than benefits...
-##
-##            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-##            # This is only useful in the case where c != None.
-##            # z still holds 1L
-##            table = [z] * 32
-##            table[0] = z
-##            for i in range(1, 32):
-##                table[i] = _help_mult(table[i-1], a, c)
-##            i = b.numdigits() - 1
-##            while i >= 0:
-##                bi = b.digit(i)
-##                j = SHIFT - 5
-##                while j >= 0:
-##                    index = (bi >> j) & 0x1f
-##                    for k in range(5):
-##                        z = _help_mult(z, z, c)
-##                    if index:
-##                        z = _help_mult(z, table[index], c)
-##                    j -= 5
-##                i -= 1
+        else:
+            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
+            # This is only useful in the case where c != None.
+            # z still holds 1L
+            table = [z] * 32
+            table[0] = z
+            for i in range(1, 32):
+                table[i] = _help_mult(table[i-1], a, c)
+            i = b.numdigits()
+            # Note that here SHIFT is not a multiple of 5.  The difficulty
+            # is to extract 5 bits at a time from 'b', starting from the
+            # most significant digits, so that at the end of the algorithm
+            # it falls exactly to zero.
+            # m  = max number of bits = i * SHIFT
+            # m+ = m rounded up to the next multiple of 5
+            # j  = (m+) % SHIFT = (m+) - (i * SHIFT)
+            # (computed without doing "i * SHIFT", which might overflow)
+            j = i % 5
+            if j != 0:
+                j = 5 - j
+            if not we_are_translated():
+                assert j == (i*SHIFT+4)//5*5 - i*SHIFT
+            #
+            accum = r_uint(0)
+            while True:
+                j -= 5
+                if j >= 0:
+                    index = (accum >> j) & 0x1f
+                else:
+                    # 'accum' does not have enough digit.
+                    # must get the next digit from 'b' in order to complete
+                    i -= 1
+                    if i < 0:
+                        break    # done
+                    bi = b.udigit(i)
+                    index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f
+                    accum = bi
+                    j += SHIFT
+                #
+                for k in range(5):
+                    z = _help_mult(z, z, c)
+                if index:
+                    z = _help_mult(z, table[index], c)
+            #
+            assert j == -5
 
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -6,6 +6,7 @@
 import sys, os
 
 link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -27,6 +28,7 @@
         # amount of troubles due to symbol versions
         # and 0.9.8/1.0.0
         link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
     else:
         libraries += ['ssl', 'crypto']
 
@@ -41,6 +43,7 @@
 eci = ExternalCompilationInfo(
     libraries = libraries,
     link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint.py
@@ -379,6 +379,18 @@
         for n, expected in [(37, 9), (1291, 931), (67889, 39464)]:
             v = two.pow(t, rbigint.fromint(n))
             assert v.toint() == expected
+        #
+        # more tests, comparing against CPython's answer
+        enabled = sample(range(5*32), 10)
+        for i in range(5*32):
+            t = t.mul(two)      # add one random bit
+            if random() >= 0.5:
+                t = t.add(rbigint.fromint(1))
+            if i not in enabled:
+                continue    # don't take forever
+            n = randint(1, sys.maxint)
+            v = two.pow(t, rbigint.fromint(n))
+            assert v.toint() == pow(2, t.tolong(), n)
 
     def test_pow_lln(self):
         x = 10L
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1168,8 +1168,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno
diff --git a/pypy/tool/pytest/pypy_test_failure_demo.py b/pypy/tool/pytest/pypy_test_failure_demo.py
--- a/pypy/tool/pytest/pypy_test_failure_demo.py
+++ b/pypy/tool/pytest/pypy_test_failure_demo.py
@@ -8,6 +8,10 @@
 def test_interp_func(space): 
     assert space.is_true(space.w_None) 
 
+def test_interp_reinterpret(space):
+    a = 1
+    assert a == 2
+
 class TestInterpTest: 
     def test_interp_method(self): 
         assert self.space.is_true(self.space.w_False) 
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct
diff --git a/pypy/translator/c/src/exception.h b/pypy/translator/c/src/exception.h
--- a/pypy/translator/c/src/exception.h
+++ b/pypy/translator/c/src/exception.h
@@ -43,6 +43,16 @@
           filename, lineno, functionname);
 }
 #endif
+#else   /* !DO_LOG_EXC: define the function anyway, so that we can shut
+           off the prints of a debug_exc by remaking only testing_1.o */
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname);
+#ifndef PYPY_NOT_MAIN_FILE
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname)
+{
+}
+#endif
 #endif  /* DO_LOG_EXC */
 
 /* Hint: functions and macros not defined here, like RPyRaiseException,
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Mon Apr 23 16:00:54 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 23 Apr 2012 16:00:54 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: Merged default in.
Message-ID: <20120423140054.E72578203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54678:1cdf2585f691
Date: 2012-04-23 08:29 -0400
http://bitbucket.org/pypy/pypy/changeset/1cdf2585f691/

Log:	Merged default in.

diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
 try:
     from thread import _local as local
 except ImportError:
-    local = object    # no threads
+    class local(object):    # no threads
+        pass
 
 class ConvMode:
     encoding = 'ascii'
diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -21,7 +21,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC']
     res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
@@ -34,6 +34,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST'] # needed for VC10
     else:
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -16,7 +16,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
     res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
@@ -29,6 +29,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
                         '/EXPORT:init_testcapi']
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -229,8 +229,8 @@
                     return thing
                 elif hasattr(thing, '__name__'): # mostly types and functions
                     return thing.__name__
-                elif hasattr(thing, 'name'): # mostly ClassDescs
-                    return thing.name
+                elif hasattr(thing, 'name') and isinstance(thing.name, str):
+                    return thing.name            # mostly ClassDescs
                 elif isinstance(thing, tuple):
                     return '_'.join(map(nameof, thing))
                 else:
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/cppyy.rst
@@ -0,0 +1,333 @@
+============================
+cppyy: C++ bindings for PyPy
+============================
+
+The cppyy module provides C++ bindings for PyPy by using the reflection
+information extracted from C++ header files by means of the
+`Reflex package`_.
+For this to work, you have to both install Reflex and build PyPy from the
+reflex-support branch.
+As indicated by this being a branch, support for Reflex is still
+experimental.
+However, it is functional enough to put it in the hands of those who want
+to give it a try.
+In the medium term, cppyy will move away from Reflex and instead use
+`cling`_ as its backend, which is based on `llvm`_.
+Although that will change the logistics on the generation of reflection
+information, it will not change the python-side interface.
+
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`cling`: http://root.cern.ch/drupal/content/cling
+.. _`llvm`: http://llvm.org/
+
+
+Installation
+============
+
+For now, the easiest way of getting the latest version of Reflex, is by
+installing the ROOT package.
+Besides getting the latest version of Reflex, another advantage is that with
+the full ROOT package, you can also use your Reflex-bound code on `CPython`_.
+`Download`_ a binary or install from `source`_.
+Some Linux and Mac systems may have ROOT provided in the list of scientific
+software of their packager.
+A current, standalone version of Reflex should be provided at some point,
+once the dependencies and general packaging have been thought out.
+Also, make sure you have a version of `gccxml`_ installed, which is most
+easily provided by the packager of your system.
+If you read up on gccxml, you'll probably notice that it is no longer being
+developed and hence will not provide C++11 support.
+That's why the medium term plan is to move to `cling`_.
+
+.. _`Download`: http://root.cern.ch/drupal/content/downloading-root
+.. _`source`: http://root.cern.ch/drupal/content/installing-root-source
+.. _`gccxml`: http://www.gccxml.org
+
+Next, get the `PyPy sources`_, select the reflex-support branch, and build
+pypy-c.
+For the build to succeed, the ``$ROOTSYS`` environment variable must point to
+the location of your ROOT installation::
+
+    $ hg clone https://bitbucket.org/pypy/pypy
+    $ cd pypy
+    $ hg up reflex-support
+    $ cd pypy/translator/goal
+    $ python translate.py -O jit --gcrootfinder=shadowstack targetpypystandalone.py --withmod-cppyy
+
+This will build a ``pypy-c`` that includes the cppyy module, and through that,
+Reflex support.
+Of course, if you already have a pre-built version of the ``pypy`` interpreter,
+you can use that for the translation rather than ``python``.
+
+.. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
+
+
+Basic example
+=============
+
+Now test with a trivial example whether all packages are properly installed
+and functional.
+First, create a C++ header file with some class in it (note that all functions
+are made inline for convenience; a real-world example would of course have a
+corresponding source file)::
+
+    $ cat MyClass.h
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_myint(i) {}
+
+        int GetMyInt() { return m_myint; }
+        void SetMyInt(int i) { m_myint = i; }
+
+    public:
+       int m_myint;
+    };
+
+Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
+code::
+
+    $ genreflex MyClass.h
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+
+Now you're ready to use the bindings.
+Since the bindings are designed to look pythonistic, it should be
+straightforward::
+
+    $ pypy-c
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libMyClassDict.so")
+    <CPPLibrary object at 0xb6fd7c4c>
+    >>>> myinst = cppyy.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> myinst.SetMyInt(33)
+    >>>> print myinst.m_myint
+    33
+    >>>> myinst.m_myint = 77
+    >>>> print myinst.GetMyInt()
+    77
+    >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
+
+That's all there is to it!
+
+
+Features
+========
+
+The following is not meant to be an exhaustive list, since cppyy is still
+under active development.
+Furthermore, the intention is that every feature is as natural as possible on
+the python side, so if you find something missing in the list below, simply
+try it out.
+It is not always possible to provide exact mapping between python and C++
+(active memory management is one such case), but by and large, if the use of a
+feature does not strike you as obvious, it is more likely to simply be a bug.
+That is a strong statement to make, but also a worthy goal.
+
+* **abstract classes**: Are represented as python classes, since they are
+  needed to complete the inheritance hierarchies, but will raise an exception
+  if an attempt is made to instantiate from them.
+
+* **arrays**: Supported for builtin data types only, as used from module
+  ``array``.
+  Out-of-bounds checking is limited to those cases where the size is known at
+  compile time (and hence part of the reflection info).
+
+* **builtin data types**: Map onto the expected equivalent python types, with
+  the caveat that there may be size differences, and thus it is possible that
+  exceptions are raised if an overflow is detected.
+
+* **casting**: Is supposed to be unnecessary.
+  Object pointer returns from functions provide the most derived class known
+  in the hierarchy of the object being returned.
+  This is important to preserve object identity as well as to make casting,
+  a pure C++ feature after all, superfluous.
+
+* **classes and structs**: Get mapped onto python classes, where they can be
+  instantiated as expected.
+  If classes are inner classes or live in a namespace, their naming and
+  location will reflect that.
+
+* **data members**: Public data members are represented as python properties
+  and provide read and write access on instances as expected.
+
+* **default arguments**: C++ default arguments work as expected, but python
+  keywords are not supported.
+  It is technically possible to support keywords, but for the C++ interface,
+  the formal argument names have no meaning and are not considered part of the
+  API, hence it is not a good idea to use keywords.
+
+* **doc strings**: The doc string of a method or function contains the C++
+  arguments and return types of all overloads of that name, as applicable.
+
+* **functions**: Work as expected and live in their appropriate namespace
+  (which can be the global one, ``cppyy.gbl``).
+
+* **inheritance**: All combinations of inheritance on the C++ (single,
+  multiple, virtual) are supported in the binding.
+  However, new python classes can only use single inheritance from a bound C++
+  class.
+  Multiple inheritance would introduce two "this" pointers in the binding.
+  This is a current, not a fundamental, limitation.
+  The C++ side will not see any overridden methods on the python side, as
+  cross-inheritance is planned but not yet supported.
+
+* **methods**: Are represented as python methods and work as expected.
+  They are first class objects and can be bound to an instance.
+  Virtual C++ methods work as expected.
+  To select a specific virtual method, do like with normal python classes
+  that override methods: select it from the class that you need, rather than
+  calling the method on the instance.
+
+* **namespaces**: Are represented as python classes.
+  Namespaces are more open-ended than classes, so sometimes initial access may
+  result in updates as data and functions are looked up and constructed
+  lazily.
+  Thus the result of ``dir()`` on a namespace should not be relied upon: it
+  only shows the already accessed members. (TODO: to be fixed by implementing
+  __dir__.)
+  The global namespace is ``cppyy.gbl``.
+
+* **operator conversions**: If defined in the C++ class and a python
+  equivalent exists (i.e. all builtin integer and floating point types, as well
+  as ``bool``), it will map onto that python conversion.
+  Note that ``char*`` is mapped onto ``__str__``.
+
+* **operator overloads**: If defined in the C++ class and if a python
+  equivalent is available (not always the case, think e.g. of ``operator||``),
+  then they work as expected.
+  Special care needs to be taken for global operator overloads in C++: first,
+  make sure that they are actually reflected, especially for the global
+  overloads for ``operator==`` and ``operator!=`` of STL iterators in the case
+  of gcc.
+  Second, make sure that reflection info is loaded in the proper order.
+  I.e. that these global overloads are available before use.
+
+* **pointers**: For builtin data types, see arrays.
+  For objects, a pointer to an object and an object looks the same, unless
+  the pointer is a data member.
+  In that case, assigning to the data member will cause a copy of the pointer
+  and care should be taken about the object's life time.
+  If a pointer is a global variable, the C++ side can replace the underlying
+  object and the python side will immediately reflect that.
+
+* **static data members**: Are represented as python property objects on the
+  class and the meta-class.
+  Both read and write access is as expected.
+
+* **static methods**: Are represented as python's ``staticmethod`` objects
+  and can be called both from the class as well as from instances.
+
+* **strings**: The std::string class is considered a builtin C++ type and
+  mixes quite well with python's str.
+  Python's str can be passed where a ``const char*`` is expected, and an str
+  will be returned if the return type is ``const char*``.
+
+* **templated classes**: Are represented in a meta-class style in python.
+  This looks a little bit confusing, but conceptually is rather natural.
+  For example, given the class ``std::vector<int>``, the meta-class part would
+  be ``std.vector`` in python.
+  Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
+  create an instance of that class, do ``std.vector(int)()``.
+  Note that templates can be build up by handing actual types to the class
+  instantiation (as done in this vector example), or by passing in the list of
+  template arguments as a string.
+  The former is a lot easier to work with if you have template instantiations
+  using classes that themselves are templates (etc.) in the arguments.
+  All classes must already exist in the loaded reflection info.
+
+* **unary operators**: Are supported if a python equivalent exists, and if the
+  operator is defined in the C++ class.
+
+You can always find more detailed examples and see the full of supported
+features by looking at the tests in pypy/module/cppyy/test.
+
+If a feature or reflection info is missing, this is supposed to be handled
+gracefully.
+In fact, there are unit tests explicitly for this purpose (even as their use
+becomes less interesting over time, as the number of missing features
+decreases).
+Only when a missing feature is used, should there be an exception.
+For example, if no reflection info is available for a return type, then a
+class that has a method with that return type can still be used.
+Only that one specific method can not be used.
+
+
+The fast lane
+=============
+
+The following is an experimental feature of cppyy, and that makes it doubly
+experimental, so caveat emptor.
+With a slight modification of Reflex, it can provide function pointers for
+C++ methods, and hence allow PyPy to call those pointers directly, rather than
+calling C++ through a Reflex stub.
+This results in a rather significant speed-up.
+Mind you, the normal stub path is not exactly slow, so for now only use this
+out of curiosity or if you really need it.
+
+To install this patch of Reflex, locate the file genreflex-methptrgetter.patch
+in pypy/module/cppyy and apply it to the genreflex python scripts found in
+``$ROOTSYS/lib``::
+
+    $ cd $ROOTSYS/lib
+    $ patch -p2 < genreflex-methptrgetter.patch
+
+With this patch, ``genreflex`` will have grown the ``--with-methptrgetter``
+option.
+Use this option when running ``genreflex``, and add the
+``-Wno-pmf-conversions`` option to ``g++`` when compiling.
+The rest works the same way: the fast path will be used transparently (which
+also means that you can't actually find out whether it is in use, other than
+by running a micro-benchmark).
+
+
+CPython
+=======
+
+Most of the ideas in cppyy come originally from the `PyROOT`_ project.
+Although PyROOT does not support Reflex directly, it has an alter ego called
+"PyCintex" that, in a somewhat roundabout way, does.
+If you installed ROOT, rather than just Reflex, PyCintex should be available
+immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment
+variable.
+
+.. _`PyROOT`: http://root.cern.ch/drupal/content/pyroot
+
+There are a couple of minor differences between PyCintex and cppyy, most to do
+with naming.
+The one that you will run into directly, is that PyCintex uses a function
+called ``loadDictionary`` rather than ``load_reflection_info``.
+The reason for this is that Reflex calls the shared libraries that contain
+reflection info "dictionaries."
+However, in python, the name `dictionary` already has a well-defined meaning,
+so a more descriptive name was chosen for cppyy.
+In addition, PyCintex requires that the names of shared libraries so loaded
+start with "lib" in their name.
+The basic example above, rewritten for PyCintex thus goes like this::
+
+    $ python
+    >>> import PyCintex
+    >>> PyCintex.loadDictionary("libMyClassDict.so")
+    >>> myinst = PyCintex.gbl.MyClass(42)
+    >>> print myinst.GetMyInt()
+    42
+    >>> myinst.SetMyInt(33)
+    >>> print myinst.m_myint
+    33
+    >>> myinst.m_myint = 77
+    >>> print myinst.GetMyInt()
+    77
+    >>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
+
+Other naming differences are such things as taking an address of an object.
+In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
+made to follow the naming as in ``ctypes`` and hence use ``addressof``
+(PyROOT/PyCintex predate ``ctypes`` by several years, and the ROOT project
+follows camel-case, hence the differences).
+
+Of course, this is python, so if any of the naming is not to your liking, all
+you have to do is provide a wrapper script that you import instead of
+importing the ``cppyy`` or ``PyCintex`` modules directly.
+In that wrapper script you can rename methods exactly the way you need it.
+
+In the cling world, all these differences will be resolved.
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -23,6 +23,8 @@
 
 * Write them in RPython as mixedmodule_, using *rffi* as bindings.
 
+* Write them in C++ and bind them through Reflex_ (EXPERIMENTAL)
+
 .. _ctypes: #CTypes
 .. _\_ffi: #LibFFI
 .. _mixedmodule: #Mixed Modules
@@ -110,3 +112,34 @@
 
 XXX we should provide detailed docs about lltype and rffi, especially if we
     want people to follow that way.
+
+Reflex
+======
+
+This method is only experimental for now, and is being exercised on a branch,
+`reflex-support`_, so you will have to build PyPy yourself.
+The method works by using the `Reflex package`_ to provide reflection
+information of the C++ code, which is then used to automatically generate
+bindings at runtime, which can then be used from python.
+Full details are `available here`_.
+
+.. _`reflex-support`: cppyy.html
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`available here`: cppyy.html
+
+Pros
+----
+
+If it works, it is mostly automatic, and hence easy in use.
+The bindings can make use of direct pointers, in which case the calls are
+very fast.
+
+Cons
+----
+
+C++ is a large language, and these bindings are not yet feature-complete.
+Although missing features should do no harm if you don't use them, if you do
+need a particular feature, it may be necessary to work around it in python
+or with a C++ helper function.
+Although Reflex works on various platforms, the bindings with PyPy have only
+been tested on Linux.
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -49,7 +49,9 @@
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
         # maybe we want this shorter:
-        name = getattr(self, 'name', '?')
+        name = getattr(self, 'name', None)
+        if not isinstance(name, str):
+            name = '?'
         return "<%s %s>" % (self.__class__.__name__, name)
 
     def call_args(self, args):
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1656,15 +1656,21 @@
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
-            #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower 16 bits read there
-            #   - multiply by 4 and use it as an offset in type_info_group
-            #   - add 16 bytes, to go past the TYPE_INFO structure
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0;
+            #     this is a complete word (N=4 bytes on 32-bit, N=8 on
+            #     64-bits)
+            #   - keep the lower half of what is read there (i.e.
+            #     truncate to an unsigned 'N / 2' bytes value)
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
             loc = locs[1]
             assert isinstance(loc, ImmedLoc)
             classptr = loc.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the 16 bits in the object header
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 or 4 on a big-endian machine.
             from pypy.rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
     space.call_method(w_list, "reverse")
     return 0
 
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+    """Return a list of the objects in list containing the objects between low
+    and high.  Return NULL and set an exception if unsuccessful.  Analogous
+    to list[low:high].  Negative indices, as when slicing from Python, are not
+    supported."""
+    w_start = space.wrap(low)
+    w_stop = space.wrap(high)
+    return space.getslice(w_list, w_start, w_stop)
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyList_SetSlice(space, w_list, low, high, w_sequence):
     """Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
     This is the equivalent of the Python expression hash(o)."""
     return space.int_w(space.hash(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+    """Set a TypeError indicating that type(o) is not hashable and return -1.
+    This function receives special treatment when stored in a tp_hash slot,
+    allowing a type to explicitly indicate to the interpreter that it is not
+    hashable.
+    """
+    raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Dir(space, w_o):
     """This is equivalent to the Python expression dir(o), returning a (possibly
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
-    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc)
@@ -60,6 +60,16 @@
     args_w = space.fixedview(w_args)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
 
+def wrap_inquirypred(space, w_self, w_args, func):
+    func_inquiry = rffi.cast(inquiry, func)
+    check_num_args(space, w_args, 0)
+    args_w = space.fixedview(w_args)
+    res = generic_cpy_call(space, func_inquiry, w_self)
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
+    return space.wrap(bool(res))
+
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
     check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
         w_errors = space.wrap(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+    """Decode a string object by passing it to the codec registered
+    for encoding and return the result as Python object. encoding and
+    errors have the same meaning as the parameters of the same name in
+    the string encode() method.  The codec to be used is looked up
+    using the Python codec registry. Return NULL if an exception was
+    raised by the codec.
+
+    This function is not available in 3.x and does not have a PyBytes alias."""
+    if not PyString_Check(space, w_str):
+        PyErr_BadArgument(space)
+
+    w_encoding = w_errors = space.w_None
+    if encoding:
+        w_encoding = space.wrap(rffi.charp2str(encoding))
+    if errors:
+        w_errors = space.wrap(rffi.charp2str(errors))
+    return space.call_method(w_str, "decode", w_encoding, w_errors)
+
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
-    """Return a list of the objects in list containing the objects between low
-    and high.  Return NULL and set an exception if unsuccessful.  Analogous
-    to list[low:high].  Negative indices, as when slicing from Python, are not
-    supported.
-
-    This function used an int for low and high. This might
-    require changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([Py_ssize_t], PyObject)
 def PyLong_FromSsize_t(space, v):
     """Return a new PyLongObject object from a C Py_ssize_t, or
@@ -1606,15 +1595,6 @@
     for PyObject_Str()."""
     raise NotImplementedError
 
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
-    """Set a TypeError indicating that type(o) is not hashable and return -1.
-    This function receives special treatment when stored in a tp_hash slot,
-    allowing a type to explicitly indicate to the interpreter that it is not
-    hashable.
-    """
-    raise NotImplementedError
-
 @cpython_api([], PyFrameObject)
 def PyEval_GetFrame(space):
     """Return the current thread state's frame, which is NULL if no frame is
@@ -1737,17 +1717,6 @@
     changes in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
-    """Decode a string object by passing it to the codec registered for encoding and
-    return the result as Python object. encoding and errors have the same
-    meaning as the parameters of the same name in the string encode() method.
-    The codec to be used is looked up using the Python codec registry. Return NULL
-    if an exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Encode(space, s, size, encoding, errors):
     """Encode the char buffer of the given size by passing it to the codec
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
         w_t = api.PyList_AsTuple(w_l)
         assert space.unwrap(w_t) == (3, 2, 1)
 
+    def test_list_getslice(self, space, api):
+        w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+        w_s = api.PyList_GetSlice(w_l, 1, 5)
+        assert space.unwrap(w_s) == [2, 1]
+
 class AppTestListObject(AppTestCpythonExtensionBase):
     def test_listobject(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
+    def test_AsDecodedObject(self, space, api):
+        w_str = space.wrap('caf\xe9')
+        encoding = rffi.str2charp("latin-1")
+        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        rffi.free_charp(encoding)
+        assert space.unwrap(w_res) == u"caf\xe9"
+
     def test_eq(self, space, api):
         assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
         assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
+        
+    def test_bool(self):
+        module = self.import_extension('foo', [
+            ("newInt", "METH_VARARGS",
+             """
+                IntLikeObject *intObj;
+                long intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "i", &intval))
+                    return NULL;
+
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+                intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+                if (!intObj) {
+                    return NULL;
+                }
+
+                intObj->value = intval;
+                return (PyObject *)intObj;
+             """)],
+            """
+            typedef struct
+            {
+                PyObject_HEAD
+                int value;
+            } IntLikeObject;
+
+            static int
+            intlike_nb_nonzero(IntLikeObject *v)
+            {
+                if (v->value == -42) {
+                    PyErr_SetNone(PyExc_ValueError);
+                    return -1;
+                }
+                return v->value;
+            }
+
+            PyTypeObject IntLike_Type = {
+                PyObject_HEAD_INIT(0)
+                /*ob_size*/             0,
+                /*tp_name*/             "IntLike",
+                /*tp_basicsize*/        sizeof(IntLikeObject),
+            };
+            static PyNumberMethods intlike_as_number;
+            """)
+        assert not bool(module.newInt(0))
+        assert bool(module.newInt(1))
+        assert bool(module.newInt(-1))
+        raises(ValueError, bool, module.newInt(-42))
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,10 +453,22 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        # prefix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        # suffix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        # type checks
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+        self.raises(space, api, TypeError,
+                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
+                    2, 10, 1)
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
@@ -30,6 +31,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -10,6 +10,7 @@
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
      scalar_w, W_NDimArray, array)
+from pypy.module.micronumpy.interp_arrayops import where
 from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize, instantiate
 
@@ -35,6 +36,7 @@
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring"]
 TWO_ARG_FUNCTIONS = ["dot", 'take']
+THREE_ARG_FUNCTIONS = ['where']
 
 class FakeSpace(object):
     w_ValueError = None
@@ -445,14 +447,25 @@
             arg = self.args[1].execute(interp)
             if not isinstance(arg, BaseArray):
                 raise ArgumentNotAnArray
-            if not isinstance(arg, BaseArray):
-                raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
             else:
                 assert False # unreachable code
+        elif self.name in THREE_ARG_FUNCTIONS:
+            if len(self.args) != 3:
+                raise ArgumentMismatch
+            arg1 = self.args[1].execute(interp)
+            arg2 = self.args[2].execute(interp)
+            if not isinstance(arg1, BaseArray):
+                raise ArgumentNotAnArray
+            if not isinstance(arg2, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "where":
+                w_res = where(interp.space, arr, arg1, arg2)
+            else:
+                assert False
         else:
             raise WrongFunctionName
         if isinstance(w_res, BaseArray):
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, space, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -1583,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,16 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -270,3 +270,13 @@
         b -> 2
         """)
         assert interp.results[0].value == 3
+
+    def test_where(self):
+        interp = self.run('''
+        a = [1, 0, 3, 0]
+        b = [1, 1, 1, 1]
+        c = [0, 0, 0, 0]
+        d = where(a, b, c)
+        d -> 1
+        ''')
+        assert interp.results[0].value == 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
@@ -1831,6 +1830,19 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
+    def test_array_indexing_bool_setitem_multidim(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
+
+    def test_array_indexing_bool_setitem_2(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a = a[::2]
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 5], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,6 @@
 
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_ufunc_instance(self):
         from _numpypy import add, ufunc
@@ -149,7 +148,11 @@
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+        # Since comparisons with nnan and nan all return false,
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
+
 
     def test_fmin(self):
         from _numpypy import fmin
@@ -165,7 +168,9 @@
         assert math.isnan(fmin(0, nan))
         assert math.isnan(fmin(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
     def test_fmod(self):
         from _numpypy import fmod
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -596,7 +596,7 @@
         import sys
         size = 0x14FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
@@ -618,7 +618,7 @@
         import sys
         size = 0x17FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_exception.py b/pypy/module/pypyjit/test_pypy_c/test_exception.py
--- a/pypy/module/pypyjit/test_pypy_c/test_exception.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_exception.py
@@ -91,3 +91,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_continue_in_finally(self):
+        # check that 'continue' inside a try:finally: block is correctly
+        # detected as closing a loop
+        py.test.skip("is this case important?")
+        def f(n):
+            i = 0
+            while 1:
+                try:
+                    if i < n:
+                        continue
+                finally:
+                    i += 1
+                return i
+
+        log = self.run(f, [2000])
+        assert log.result == 2001
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i40 = int_add_ovf(i31, 1)
+            guard_no_overflow(descr=...)
+            i41 = int_lt(i40, i33)
+            guard_true(i41, descr=...)
+            --TICK--
+            jump(..., descr=...)
+        """)
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -366,7 +366,10 @@
     __slots__ = ()
 
     def __repr__(self):
-        s = '%s(%s)' % (self.__class__.__name__, getattr(self, 'name', ''))
+        name = getattr(self, 'name', '')
+        if not isinstance(name, str):
+            name = ''
+        s = '%s(%s)' % (self.__class__.__name__, name)
         w_cls = getattr(self, 'w__class__', None)
         if w_cls is not None and w_cls is not self:
             s += ' instance of %s' % self.w__class__
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,12 +295,11 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,31 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        res = 'bar' in x
+        assert res is True
+        #
+        class MyError(Exception):
+            pass
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise MyError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(MyError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assert=plain -rf
+addopts = --assert=reinterp -rf
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -40,7 +40,7 @@
 # In that case, do 5 bits at a time.  The potential drawback is that
 # a table of 2**5 intermediate results is computed.
 
-## FIVEARY_CUTOFF = 8   disabled for now
+FIVEARY_CUTOFF = 8
 
 
 def _mask_digit(x):
@@ -456,7 +456,7 @@
 
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
-        if 1:   ## b.numdigits() <= FIVEARY_CUTOFF:
+        if b.numdigits() <= FIVEARY_CUTOFF:
             # Left-to-right binary exponentiation (HAC Algorithm 14.79)
             # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
             i = b.numdigits() - 1
@@ -469,30 +469,51 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 i -= 1
-##        else:
-##            This code is disabled for now, because it assumes that
-##            SHIFT is a multiple of 5.  It could be fixed but it looks
-##            like it's more troubles than benefits...
-##
-##            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-##            # This is only useful in the case where c != None.
-##            # z still holds 1L
-##            table = [z] * 32
-##            table[0] = z
-##            for i in range(1, 32):
-##                table[i] = _help_mult(table[i-1], a, c)
-##            i = b.numdigits() - 1
-##            while i >= 0:
-##                bi = b.digit(i)
-##                j = SHIFT - 5
-##                while j >= 0:
-##                    index = (bi >> j) & 0x1f
-##                    for k in range(5):
-##                        z = _help_mult(z, z, c)
-##                    if index:
-##                        z = _help_mult(z, table[index], c)
-##                    j -= 5
-##                i -= 1
+        else:
+            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
+            # This is only useful in the case where c != None.
+            # z still holds 1L
+            table = [z] * 32
+            table[0] = z
+            for i in range(1, 32):
+                table[i] = _help_mult(table[i-1], a, c)
+            i = b.numdigits()
+            # Note that here SHIFT is not a multiple of 5.  The difficulty
+            # is to extract 5 bits at a time from 'b', starting from the
+            # most significant digits, so that at the end of the algorithm
+            # it falls exactly to zero.
+            # m  = max number of bits = i * SHIFT
+            # m+ = m rounded up to the next multiple of 5
+            # j  = (m+) % SHIFT = (m+) - (i * SHIFT)
+            # (computed without doing "i * SHIFT", which might overflow)
+            j = i % 5
+            if j != 0:
+                j = 5 - j
+            if not we_are_translated():
+                assert j == (i*SHIFT+4)//5*5 - i*SHIFT
+            #
+            accum = r_uint(0)
+            while True:
+                j -= 5
+                if j >= 0:
+                    index = (accum >> j) & 0x1f
+                else:
+                    # 'accum' does not have enough digit.
+                    # must get the next digit from 'b' in order to complete
+                    i -= 1
+                    if i < 0:
+                        break    # done
+                    bi = b.udigit(i)
+                    index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f
+                    accum = bi
+                    j += SHIFT
+                #
+                for k in range(5):
+                    z = _help_mult(z, z, c)
+                if index:
+                    z = _help_mult(z, table[index], c)
+            #
+            assert j == -5
 
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -6,6 +6,7 @@
 import sys, os
 
 link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -27,6 +28,7 @@
         # amount of troubles due to symbol versions
         # and 0.9.8/1.0.0
         link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
     else:
         libraries += ['ssl', 'crypto']
 
@@ -41,6 +43,7 @@
 eci = ExternalCompilationInfo(
     libraries = libraries,
     link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint.py
@@ -379,6 +379,18 @@
         for n, expected in [(37, 9), (1291, 931), (67889, 39464)]:
             v = two.pow(t, rbigint.fromint(n))
             assert v.toint() == expected
+        #
+        # more tests, comparing against CPython's answer
+        enabled = sample(range(5*32), 10)
+        for i in range(5*32):
+            t = t.mul(two)      # add one random bit
+            if random() >= 0.5:
+                t = t.add(rbigint.fromint(1))
+            if i not in enabled:
+                continue    # don't take forever
+            n = randint(1, sys.maxint)
+            v = two.pow(t, rbigint.fromint(n))
+            assert v.toint() == pow(2, t.tolong(), n)
 
     def test_pow_lln(self):
         x = 10L
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1168,8 +1168,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno
diff --git a/pypy/tool/pytest/pypy_test_failure_demo.py b/pypy/tool/pytest/pypy_test_failure_demo.py
--- a/pypy/tool/pytest/pypy_test_failure_demo.py
+++ b/pypy/tool/pytest/pypy_test_failure_demo.py
@@ -8,6 +8,10 @@
 def test_interp_func(space): 
     assert space.is_true(space.w_None) 
 
+def test_interp_reinterpret(space):
+    a = 1
+    assert a == 2
+
 class TestInterpTest: 
     def test_interp_method(self): 
         assert self.space.is_true(self.space.w_False) 
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct
diff --git a/pypy/translator/c/src/exception.h b/pypy/translator/c/src/exception.h
--- a/pypy/translator/c/src/exception.h
+++ b/pypy/translator/c/src/exception.h
@@ -43,6 +43,16 @@
           filename, lineno, functionname);
 }
 #endif
+#else   /* !DO_LOG_EXC: define the function anyway, so that we can shut
+           off the prints of a debug_exc by remaking only testing_1.o */
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname);
+#ifndef PYPY_NOT_MAIN_FILE
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname)
+{
+}
+#endif
 #endif  /* DO_LOG_EXC */
 
 /* Hint: functions and macros not defined here, like RPyRaiseException,
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Mon Apr 23 17:50:47 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 17:50:47 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: progress in
 supporting prebuilt untyped storage in the C backend
Message-ID: <20120423155047.6ED788203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54679:21c67be4174e
Date: 2012-04-23 17:49 +0200
http://bitbucket.org/pypy/pypy/changeset/21c67be4174e/

Log:	progress in supporting prebuilt untyped storage in the C backend

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -127,6 +127,7 @@
 UNTYPEDSTORAGE = lltype.GcStruct("untypedstorage",
     ("shape", lltype.Ptr(STR)),
     ("data", lltype.Array(llmemory.Address)),
+    hints={'untyped_storage': True},
     rtti=True,
 )
 
@@ -298,3 +299,16 @@
     @classmethod
     def ll_getlength(cls, arr):
         return len(arr.shape.chars)
+
+def ll_enumerate_elements(storage):
+    for i, elem in enumerate(storage.shape.chars):
+        if elem in [INSTANCE, STRING, UNICODE]:
+            yield storage.data[i].ptr
+        elif elem == INT:
+            yield rffi.cast(lltype.Signed, storage.data[i])
+        elif elem == FLOAT:
+            yield rffi.cast(lltype.Float, storage.data[i])
+        elif elem == BOOL:
+            yield rffi.cast(lltype.Bool, storage.data[i])
+        else:
+            assert False
diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -184,5 +184,19 @@
         res = self.interpret(f, [0])
         assert res == 21
 
+    def test_enumerate_elements(self):
+        def f():
+            storage = rerased_raw.UntypedStorage("sibf")
+            storage.setint(1, 13)
+            storage.setstr(0, "abc")
+            storage.setfloat(3, 3.5)
+            storage.setbool(2, True)
+            return storage
+
+        llres = self.interpret(f, [])
+        lst = list(rerased_raw.ll_enumerate_elements(llres))
+        assert hlstr(lst[0]) == "abc"
+        assert lst[1:] == [13, True, 3.5]
+
 class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
     pass
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -13,6 +13,7 @@
 from pypy.rlib import exports
 from pypy.rlib.rfloat import isfinite
 from pypy.rlib.rstackovf import _StackOverflow
+from pypy.rlib.rerased_raw import xxx
 from pypy.translator.c import extfunc
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from py.builtin import BaseException
@@ -576,6 +577,14 @@
 
     def enum_dependencies(self):
         T = self.getTYPE()
+        if T._hints.get('untyped_storage', False):
+            arrayfld = T._arrayfld
+            shapefld = [fld for fld in T._flds if fld != arrayfld][0]
+            shape = getattr(self.obj, shapefld)
+            yield shape
+            for elem in shape.chars:
+                import pdb
+                pdb.set_trace()
         for name in T._names:
             yield getattr(self.obj, name)
 

From noreply at buildbot.pypy.org  Mon Apr 23 17:59:33 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 17:59:33 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: use nto the pointer
	but the struct
Message-ID: <20120423155933.06CA48203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54680:d019b749b73a
Date: 2012-04-23 17:59 +0200
http://bitbucket.org/pypy/pypy/changeset/d019b749b73a/

Log:	use nto the pointer but the struct

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -303,12 +303,12 @@
 def ll_enumerate_elements(storage):
     for i, elem in enumerate(storage.shape.chars):
         if elem in [INSTANCE, STRING, UNICODE]:
-            yield storage.data[i].ptr
+            yield storage.data.items[i].ptr
         elif elem == INT:
-            yield rffi.cast(lltype.Signed, storage.data[i])
+            yield rffi.cast(lltype.Signed, storage.data.items[i])
         elif elem == FLOAT:
-            yield rffi.cast(lltype.Float, storage.data[i])
+            yield rffi.cast(lltype.Float, storage.data.items[i])
         elif elem == BOOL:
-            yield rffi.cast(lltype.Bool, storage.data[i])
+            yield rffi.cast(lltype.Bool, storage.data.items[i])
         else:
             assert False
diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -193,7 +193,7 @@
             storage.setbool(2, True)
             return storage
 
-        llres = self.interpret(f, [])
+        llres = self.interpret(f, [])._obj
         lst = list(rerased_raw.ll_enumerate_elements(llres))
         assert hlstr(lst[0]) == "abc"
         assert lst[1:] == [13, True, 3.5]
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -13,7 +13,7 @@
 from pypy.rlib import exports
 from pypy.rlib.rfloat import isfinite
 from pypy.rlib.rstackovf import _StackOverflow
-from pypy.rlib.rerased_raw import xxx
+from pypy.rlib.rerased_raw import ll_enumerate_elements
 from pypy.translator.c import extfunc
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from py.builtin import BaseException
@@ -582,9 +582,8 @@
             shapefld = [fld for fld in T._flds if fld != arrayfld][0]
             shape = getattr(self.obj, shapefld)
             yield shape
-            for elem in shape.chars:
-                import pdb
-                pdb.set_trace()
+            for elem in ll_enumerate_elements(self.obj):
+                yield elem
         for name in T._names:
             yield getattr(self.obj, name)
 
diff --git a/pypy/translator/c/test/test_untyped_storage.py b/pypy/translator/c/test/test_untyped_storage.py
--- a/pypy/translator/c/test/test_untyped_storage.py
+++ b/pypy/translator/c/test/test_untyped_storage.py
@@ -3,6 +3,9 @@
 from pypy.translator.c.test.test_genc import compile
 
 class CCompiledMixin(object):
+    # for individual tests see
+    # ===> ../../../rlib/test/test_rerased_raw.py
+    
     def interpret(self, f, args):
         fn = compile(f, [type(arg) for arg in args], gcpolicy='boehm')
         return fn(*args)

From noreply at buildbot.pypy.org  Mon Apr 23 18:01:17 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 18:01:17 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: fix enum_dependencies
Message-ID: <20120423160117.521FA8203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54681:6fbb57921a2d
Date: 2012-04-23 18:00 +0200
http://bitbucket.org/pypy/pypy/changeset/6fbb57921a2d/

Log:	fix enum_dependencies

diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -584,8 +584,9 @@
             yield shape
             for elem in ll_enumerate_elements(self.obj):
                 yield elem
-        for name in T._names:
-            yield getattr(self.obj, name)
+        else:
+            for name in T._names:
+                yield getattr(self.obj, name)
 
     def getlength(self):
         T = self.getTYPE()

From noreply at buildbot.pypy.org  Mon Apr 23 18:03:25 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 23 Apr 2012 18:03:25 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: progress
Message-ID: <20120423160325.D593E8203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54682:c50fe5be8265
Date: 2012-04-23 12:00 -0400
http://bitbucket.org/pypy/pypy/changeset/c50fe5be8265/

Log:	progress

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -6,6 +6,7 @@
 
 from pypy.annotation import model as annmodel
 from pypy.annotation.bookkeeper import getbookkeeper
+from pypy.rlib import longlong2float
 from pypy.rpython.annlowlevel import (hlstr, llstr, llhelper,
     cast_instance_to_base_ptr)
 from pypy.rpython.rclass import getinstancerepr
@@ -307,7 +308,7 @@
         elif elem == INT:
             yield rffi.cast(lltype.Signed, storage.data[i])
         elif elem == FLOAT:
-            yield rffi.cast(lltype.Float, storage.data[i])
+            yield longlong2float.longlong2float(storage.data[i])
         elif elem == BOOL:
             yield rffi.cast(lltype.Bool, storage.data[i])
         else:

From noreply at buildbot.pypy.org  Mon Apr 23 18:03:27 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 23 Apr 2012 18:03:27 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: fix
Message-ID: <20120423160327.3B80E8203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54683:2bee4cac6100
Date: 2012-04-23 12:02 -0400
http://bitbucket.org/pypy/pypy/changeset/2bee4cac6100/

Log:	fix

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -308,7 +308,7 @@
         elif elem == INT:
             yield rffi.cast(lltype.Signed, storage.data[i])
         elif elem == FLOAT:
-            yield longlong2float.longlong2float(storage.data[i])
+            yield longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data[i]))
         elif elem == BOOL:
             yield rffi.cast(lltype.Bool, storage.data[i])
         else:

From noreply at buildbot.pypy.org  Mon Apr 23 18:03:28 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 23 Apr 2012 18:03:28 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: merged
Message-ID: <20120423160328.786AC8203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54684:0bdcf2965cc7
Date: 2012-04-23 12:03 -0400
http://bitbucket.org/pypy/pypy/changeset/0bdcf2965cc7/

Log:	merged

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -304,12 +304,12 @@
 def ll_enumerate_elements(storage):
     for i, elem in enumerate(storage.shape.chars):
         if elem in [INSTANCE, STRING, UNICODE]:
-            yield storage.data[i].ptr
+            yield storage.data.items[i].ptr
         elif elem == INT:
-            yield rffi.cast(lltype.Signed, storage.data[i])
+            yield rffi.cast(lltype.Signed, storage.data.items[i])
         elif elem == FLOAT:
             yield longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data[i]))
         elif elem == BOOL:
-            yield rffi.cast(lltype.Bool, storage.data[i])
+            yield rffi.cast(lltype.Bool, storage.data.items[i])
         else:
             assert False
diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -193,7 +193,7 @@
             storage.setbool(2, True)
             return storage
 
-        llres = self.interpret(f, [])
+        llres = self.interpret(f, [])._obj
         lst = list(rerased_raw.ll_enumerate_elements(llres))
         assert hlstr(lst[0]) == "abc"
         assert lst[1:] == [13, True, 3.5]
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -13,7 +13,7 @@
 from pypy.rlib import exports
 from pypy.rlib.rfloat import isfinite
 from pypy.rlib.rstackovf import _StackOverflow
-from pypy.rlib.rerased_raw import xxx
+from pypy.rlib.rerased_raw import ll_enumerate_elements
 from pypy.translator.c import extfunc
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from py.builtin import BaseException
@@ -582,11 +582,11 @@
             shapefld = [fld for fld in T._flds if fld != arrayfld][0]
             shape = getattr(self.obj, shapefld)
             yield shape
-            for elem in shape.chars:
-                import pdb
-                pdb.set_trace()
-        for name in T._names:
-            yield getattr(self.obj, name)
+            for elem in ll_enumerate_elements(self.obj):
+                yield elem
+        else:
+            for name in T._names:
+                yield getattr(self.obj, name)
 
     def getlength(self):
         T = self.getTYPE()
diff --git a/pypy/translator/c/test/test_untyped_storage.py b/pypy/translator/c/test/test_untyped_storage.py
--- a/pypy/translator/c/test/test_untyped_storage.py
+++ b/pypy/translator/c/test/test_untyped_storage.py
@@ -3,6 +3,9 @@
 from pypy.translator.c.test.test_genc import compile
 
 class CCompiledMixin(object):
+    # for individual tests see
+    # ===> ../../../rlib/test/test_rerased_raw.py
+    
     def interpret(self, f, args):
         fn = compile(f, [type(arg) for arg in args], gcpolicy='boehm')
         return fn(*args)

From noreply at buildbot.pypy.org  Mon Apr 23 18:05:09 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 23 Apr 2012 18:05:09 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: fix
Message-ID: <20120423160509.F2A8C8203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54685:b0aa1fc69a03
Date: 2012-04-23 12:04 -0400
http://bitbucket.org/pypy/pypy/changeset/b0aa1fc69a03/

Log:	fix

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -308,7 +308,7 @@
         elif elem == INT:
             yield rffi.cast(lltype.Signed, storage.data.items[i])
         elif elem == FLOAT:
-            yield longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data[i]))
+            yield longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data.items[i]))
         elif elem == BOOL:
             yield rffi.cast(lltype.Bool, storage.data.items[i])
         else:

From noreply at buildbot.pypy.org  Mon Apr 23 18:31:43 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 18:31:43 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: return offsets as well
Message-ID: <20120423163143.C0ED48203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54686:32c12242fbb2
Date: 2012-04-23 18:31 +0200
http://bitbucket.org/pypy/pypy/changeset/32c12242fbb2/

Log:	return offsets as well

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -24,6 +24,9 @@
 STRING = "s"
 UNICODE = "u"
 
+# all the same size for now
+WORD = rffi.sizeof(llmemory.Address)
+
 class UntypedStorage(object):
     def __init__(self, shape):
         self.storage = [None] * len(shape)
@@ -304,12 +307,12 @@
 def ll_enumerate_elements(storage):
     for i, elem in enumerate(storage.shape.chars):
         if elem in [INSTANCE, STRING, UNICODE]:
-            yield storage.data.items[i].ptr
+            yield WORD * i, storage.data.items[i].ptr
         elif elem == INT:
-            yield rffi.cast(lltype.Signed, storage.data.items[i])
+            yield WORD * i, rffi.cast(lltype.Signed, storage.data.items[i])
         elif elem == FLOAT:
-            yield longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data.items[i]))
+            yield WORD * i, longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data.items[i]))
         elif elem == BOOL:
-            yield rffi.cast(lltype.Bool, storage.data.items[i])
+            yield WORD * i, rffi.cast(lltype.Bool, storage.data.items[i])
         else:
             assert False
diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -195,8 +195,10 @@
 
         llres = self.interpret(f, [])._obj
         lst = list(rerased_raw.ll_enumerate_elements(llres))
-        assert hlstr(lst[0]) == "abc"
-        assert lst[1:] == [13, True, 3.5]
+        assert hlstr(lst[0][1]) == "abc"
+        assert lst[0][0] == 0
+        WORD = rerased_raw.WORD
+        assert lst[1:] == [(WORD, 13), (WORD * 2, True), (WORD * 3, 3.5)]
 
 class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
     pass

From noreply at buildbot.pypy.org  Mon Apr 23 18:39:42 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 18:39:42 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: in progress
Message-ID: <20120423163942.5DA9E8203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54687:761ffdf601ea
Date: 2012-04-23 18:39 +0200
http://bitbucket.org/pypy/pypy/changeset/761ffdf601ea/

Log:	in progress

diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -582,7 +582,7 @@
             shapefld = [fld for fld in T._flds if fld != arrayfld][0]
             shape = getattr(self.obj, shapefld)
             yield shape
-            for elem in ll_enumerate_elements(self.obj):
+            for _, elem in ll_enumerate_elements(self.obj):
                 yield elem
         else:
             for name in T._names:
@@ -629,16 +629,28 @@
         else:
             padding_drop = []
 
-        for name, value in data:
-            if name in padding_drop:
-                continue
-            c_expr = defnode.access_expr(self.name, name)
-            lines = generic_initializationexpr(self.db, value, c_expr,
-                                               decoration + name)
+        if T._hints.get('untyped_storage'):
+            arrayfld = T._arrayfld
+            shapefld = [fld for fld in T._flds if fld != arrayfld]
+            c_expr = defnode.access_expr(self.name, 'shape')
+            lines = generic_initializationexpr(self.db, self.obj.shape,
+                                               c_expr, decoration + 'shape')
             for line in lines:
-                yield '\t' + line
-            if not lines[0].startswith('/*'):
-                is_empty = False
+                yield "\t" + line
+            for i, obj in enumerate(ll_enumerate_elements(self.obj)):
+                xxx
+            is_empty = False
+        else:
+            for name, value in data:
+                if name in padding_drop:
+                    continue
+                c_expr = defnode.access_expr(self.name, name)
+                lines = generic_initializationexpr(self.db, value, c_expr,
+                                                   decoration + name)
+                for line in lines:
+                    yield '\t' + line
+                if not lines[0].startswith('/*'):
+                    is_empty = False
         if is_empty:
             yield '\t%s' % '0,'
         yield '}'

From noreply at buildbot.pypy.org  Mon Apr 23 19:02:29 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 23 Apr 2012 19:02:29 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: more random progress
Message-ID: <20120423170229.F37D58203C@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54688:01fb6e09fb86
Date: 2012-04-23 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/01fb6e09fb86/

Log:	more random progress

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -24,9 +24,6 @@
 STRING = "s"
 UNICODE = "u"
 
-# all the same size for now
-WORD = rffi.sizeof(llmemory.Address)
-
 class UntypedStorage(object):
     def __init__(self, shape):
         self.storage = [None] * len(shape)
@@ -307,12 +304,12 @@
 def ll_enumerate_elements(storage):
     for i, elem in enumerate(storage.shape.chars):
         if elem in [INSTANCE, STRING, UNICODE]:
-            yield WORD * i, storage.data.items[i].ptr
+            yield i, storage.data.items[i].ptr
         elif elem == INT:
-            yield WORD * i, rffi.cast(lltype.Signed, storage.data.items[i])
+            yield i, rffi.cast(lltype.Signed, storage.data.items[i])
         elif elem == FLOAT:
-            yield WORD * i, longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data.items[i]))
+            yield i, longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data.items[i]))
         elif elem == BOOL:
-            yield WORD * i, rffi.cast(lltype.Bool, storage.data.items[i])
+            yield i, rffi.cast(lltype.Bool, storage.data.items[i])
         else:
             assert False
diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -197,8 +197,7 @@
         lst = list(rerased_raw.ll_enumerate_elements(llres))
         assert hlstr(lst[0][1]) == "abc"
         assert lst[0][0] == 0
-        WORD = rerased_raw.WORD
-        assert lst[1:] == [(WORD, 13), (WORD * 2, True), (WORD * 3, 3.5)]
+        assert lst[1:] == [(1, 13), (2, True), (3, 3.5)]
 
 class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
     pass
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -631,14 +631,19 @@
 
         if T._hints.get('untyped_storage'):
             arrayfld = T._arrayfld
-            shapefld = [fld for fld in T._flds if fld != arrayfld]
             c_expr = defnode.access_expr(self.name, 'shape')
             lines = generic_initializationexpr(self.db, self.obj.shape,
                                                c_expr, decoration + 'shape')
             for line in lines:
                 yield "\t" + line
-            for i, obj in enumerate(ll_enumerate_elements(self.obj)):
-                xxx
+            arraydef = self.db.gettypedefnode(getattr(T, arrayfld))
+            for i, obj in ll_enumerate_elements(self.obj):
+                c_expr = '(void*)' + arraydef.access_expr(
+                    self.name + '.' + arrayfld, i)
+                lines = generic_initializationexpr(self.db, obj, c_expr,
+                                                   '%s[%d]' % (name, i))
+                for line in lines:
+                    yield '\t' + line
             is_empty = False
         else:
             for name, value in data:

From noreply at buildbot.pypy.org  Mon Apr 23 19:55:40 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 23 Apr 2012 19:55:40 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Start for float support in PPC
	backend.
Message-ID: <20120423175540.866068203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54689:00899aeb3dbb
Date: 2012-04-23 13:54 -0400
http://bitbucket.org/pypy/pypy/changeset/00899aeb3dbb/

Log:	Start for float support in PPC backend.

diff --git a/pypy/jit/backend/ppc/helper/assembler.py b/pypy/jit/backend/ppc/helper/assembler.py
--- a/pypy/jit/backend/ppc/helper/assembler.py
+++ b/pypy/jit/backend/ppc/helper/assembler.py
@@ -6,12 +6,15 @@
 import pypy.jit.backend.ppc.register as r
 from pypy.rpython.lltypesystem import rffi, lltype
 
-def gen_emit_cmp_op(condition, signed=True):
+def gen_emit_cmp_op(condition, signed=True, fp=False):
     def f(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         # do the comparison
-        self.mc.cmp_op(0, l0.value, l1.value,
-                       imm=l1.is_imm(), signed=signed)
+        if fp == True:
+            self.mc.fcmpu(0, l0.value, l1.value)
+        else:
+            self.mc.cmp_op(0, l0.value, l1.value,
+                           imm=l1.is_imm(), signed=signed)
         # After the comparison, place the result
         # in the first bit of the CR
         if condition == c.LT or condition == c.U_LT:
diff --git a/pypy/jit/backend/ppc/helper/regalloc.py b/pypy/jit/backend/ppc/helper/regalloc.py
--- a/pypy/jit/backend/ppc/helper/regalloc.py
+++ b/pypy/jit/backend/ppc/helper/regalloc.py
@@ -1,5 +1,4 @@
-from pypy.jit.metainterp.history import ConstInt
-from pypy.jit.metainterp.history import Box
+from pypy.jit.metainterp.history import ConstInt, Box, FLOAT
 
 IMM_SIZE = 2 ** 15 - 1
 
@@ -84,3 +83,41 @@
         self.possibly_free_var(op.result)
         return [reg1, reg2, res]
     return f
+
+def prepare_float_op(name=None, base=True, float_result=True, guard=False):
+    if guard:
+        def f(self, op, guard_op):
+            locs = []
+            loc1 = self._ensure_value_is_boxed(op.getarg(0))
+            locs.append(loc1)
+            if base:
+                loc2 = self._ensure_value_is_boxed(op.getarg(1))
+                locs.append(loc2)
+            self.possibly_free_vars_for_op(op)
+            self.free_temp_vars()
+            if guard_op is None:
+                res = self.force_allocate_reg(op.result)
+                assert float_result == (op.result.type == FLOAT)
+                locs.append(res)
+                return locs
+            else:
+                args = self._prepare_guard(guard_op, locs)
+                return args
+    else:
+        def f(self, op):
+            locs = []
+            loc1 = self._ensure_value_is_boxed(op.getarg(0))
+            locs.append(loc1)
+            if base:
+                loc2 = self._ensure_value_is_boxed(op.getarg(1))
+                locs.append(loc2)
+            self.possibly_free_vars_for_op(op)
+            self.free_temp_vars()
+            res = self.force_allocate_reg(op.result)
+            assert float_result == (op.result.type == FLOAT)
+            locs.append(res)
+            return locs
+    if name:
+        f.__name__ = name
+    return f
+
diff --git a/pypy/jit/backend/ppc/locations.py b/pypy/jit/backend/ppc/locations.py
--- a/pypy/jit/backend/ppc/locations.py
+++ b/pypy/jit/backend/ppc/locations.py
@@ -4,8 +4,10 @@
 # XXX import from arch.py, currently we have a circular import
 if sys.maxint == (2**31 - 1):
     WORD = 4
+    FWORD = 8
 else:
     WORD = 8
+    FWORD = 8
 DWORD = 2 * WORD
 
 class AssemblerLocation(object):
@@ -49,7 +51,7 @@
 class FPRegisterLocation(RegisterLocation):
     _immutable_ = True
     type = FLOAT 
-    width = DWORD
+    width = FWORD
 
     def __repr__(self):
         return 'fp%d' % self.value
@@ -83,6 +85,28 @@
     def as_key(self):
         return self.value + 40
 
+class ConstFloatLoc(AssemblerLocation):
+    """This class represents an imm float value which is stored in memory at
+    the address stored in the field value"""
+    _immutable_ = True
+    width = FWORD
+    type = FLOAT
+
+    def __init__(self, value):
+        self.value = value
+
+    def getint(self):
+        return self.value
+
+    def __repr__(self):
+        return "imm_float(stored at %d)" % (self.value)
+
+    def is_imm_float(self):
+        return True
+
+    def as_key(self):
+        return self.value
+
 class StackLocation(AssemblerLocation):
     _immutable_ = True
 
diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -174,6 +174,53 @@
         l0, res = arglocs
         self.mc.not_(res.value, l0.value)
 
+class FloatOpAssembler(object):
+    _mixin_ = True
+
+    def emit_float_add(self, op, argsloc, regalloc):
+        l0, l1, res = arglocs
+        self.mc.fadd(res.value, l0.value, l1.value)
+
+    def emit_float_sub(self, op, argsloc, regalloc):
+        l0, l1, res = arglocs
+        self.mc.fsub(res.value, l0.value, l1.value)
+
+    def emit_float_mul(self, op, argsloc, regalloc):
+        l0, l1, res = arglocs
+        self.mc.fmul(res.value, l0.value, l1.value)
+
+    def emit_float_truediv(self, op, argsloc, regalloc):
+        l0, l1, res = arglocs
+        self.mc.fdiv(res.value, l0.value, l1.value)
+
+    def emit_float_neg(self, op, argsloc, regalloc):
+        l0, res = arglocs
+        self.mc.fneg(res.value, l0.value)
+
+    def emit_float_abs(self, op, argsloc, regalloc):
+        l0, res = arglocs
+        self.mc.fabs(res.value, l0.value)
+
+    def emit_float_sqrt(self, op, argsloc, regalloc):
+        l0, res = arglocs
+        self.mc.fsqrt(res.value, l0.value)
+
+    emit_float_le = gen_emit_cmp_op(c.LE, fp=True)
+    emit_float_lt = gen_emit_cmp_op(c.LT, fp=True)
+    emit_float_gt = gen_emit_cmp_op(c.GT, fp=True)
+    emit_float_ge = gen_emit_cmp_op(c.GE, fp=True)
+    emit_float_eq = gen_emit_cmp_op(c.EQ, fp=True)
+    emit_float_ne = gen_emit_cmp_op(c.NE, fp=True)
+
+    def emit_op_cast_float_to_int(self, op, arglocs, regalloc):
+        l0, temp_loc, res = arglocs
+        self.mc.fctidz(temp_loc.value, l0.value)
+        self.mc.mfprgpr(res.value, temp_loc.value)
+
+    def emit_op_cast_int_to_float(self, op, arglocs, regalloc):
+        l0, temp_loc, res = arglocs
+        self.mc.mgprfpr(temp_loc.value, l0.value)
+        self.mc.fcfid(res.value, temp_loc.value)
 
 class GuardOpAssembler(object):
 
@@ -1254,7 +1301,7 @@
                   MiscOpAssembler, FieldOpAssembler,
                   ArrayOpAssembler, StrOpAssembler,
                   UnicodeOpAssembler, ForceOpAssembler,
-                  AllocOpAssembler):
+                  AllocOpAssembler, FloatOpAssembler):
 
     def nop(self):
         self.mc.ori(0, 0, 0)
diff --git a/pypy/jit/backend/ppc/regalloc.py b/pypy/jit/backend/ppc/regalloc.py
--- a/pypy/jit/backend/ppc/regalloc.py
+++ b/pypy/jit/backend/ppc/regalloc.py
@@ -8,7 +8,8 @@
                                                   prepare_unary_int_op,
                                                   prepare_binary_int_op,
                                                   prepare_binary_int_op_with_imm,
-                                                  prepare_unary_cmp)
+                                                  prepare_unary_cmp,
+                                                  prepare_float_op)
 from pypy.jit.metainterp.history import (Const, ConstInt, ConstPtr,
                                          Box, BoxPtr,
                                          INT, REF, FLOAT)
@@ -41,6 +42,57 @@
     def __repr__(self):
         return "<TempPtr at %s>" % (id(self),)
 
+class TempFloat(TempBox):
+    type = FLOAT
+
+    def __repr__(self):
+        return "<TempFloat at %s>" % (id(self),)
+
+
+class FPRegisterManager(RegisterManager):
+    all_regs              = r.ALL_FLOAT_REGS
+    box_types             = [FLOAT]
+    save_around_call_regs = r.VOLATILES_FLOAT
+
+    def convert_to_imm(self, c):
+        adr = self.assembler.datablockwrapper.malloc_aligned(8, 8)
+        x = c.getfloatstorage()
+        rffi.cast(rffi.CArrayPtr(long.FLOATSTORAGE), adr)[0] = x
+        return locations.ConstFloatLoc(adr)
+
+    def __init__(self, longevity, frame_manager=None, assembler=None):
+        RegisterManager.__init__(self, longevity, frame_manager, assembler)
+
+    def after_call(self, v):
+        """ Adjust registers according to the result of the call,
+        which is in variable v.
+        """
+        self._check_type(v)
+        r = self.force_allocate_reg(v)
+        return r
+
+    def ensure_value_is_boxed(self, thing, forbidden_vars=[]):
+        loc = None
+        if isinstance(thing, Const):
+            assert isinstance(thing, ConstFloat)
+            loc = self.get_scratch_reg(FLOAT, self.temp_boxes + forbidden_vars)
+            immvalue = self.convert_to_imm(thing)
+            self.assembler.load(loc, immvalue)
+        else:
+            loc = self.make_sure_var_in_reg(thing,
+                            forbidden_vars=self.temp_boxes + forbidden_vars)
+        return loc
+
+    def get_scratch_reg(self, type=FLOAT, forbidden_vars=[],
+                                                        selected_reg=None):
+        assert type == FLOAT  # for now
+        box = TempFloat()
+        self.temp_boxes.append(box)
+        reg = self.force_allocate_reg(box, forbidden_vars=forbidden_vars,
+                                                    selected_reg=selected_reg)
+        return reg
+
+
 class PPCRegisterManager(RegisterManager):
     all_regs              = r.MANAGED_REGS
     box_types             = None       # or a list of acceptable types
@@ -131,21 +183,15 @@
     @staticmethod
     def frame_pos(loc, type):
         num_words = PPCFrameManager.frame_size(type)
-        if type == FLOAT:
-            assert 0, "not implemented yet"
         return locations.StackLocation(loc, num_words=num_words, type=type)
 
     @staticmethod
     def frame_size(type):
-        if type == FLOAT:
-            assert 0, "TODO"
         return 1
 
     @staticmethod
     def get_loc_index(loc):
         assert loc.is_stack()
-        if loc.type == FLOAT:
-            assert 0, "not implemented yet"
         return loc.position
 
 class Regalloc(object):
@@ -164,6 +210,7 @@
         self.last_real_usage = last_real_usage
         fm = self.frame_manager
         asm = self.assembler
+        self.fprm = FPRegisterManager(longevity, fm, asm)
         self.rm = PPCRegisterManager(longevity, fm, asm)
 
     def prepare_loop(self, inputargs, operations):
@@ -182,12 +229,11 @@
         cur_frame_pos = -self.assembler.OFFSET_STACK_ARGS // WORD + 1
         for box in inputargs:
             assert isinstance(box, Box)
-            # handle inputargs in argument registers
-            if box.type == FLOAT and arg_index % 2 != 0:
-                assert 0, "not implemented yet"
             if arg_index < n_register_args:
                 if box.type == FLOAT:
-                    assert 0, "not implemented yet"
+                    loc = r.PARAM_FPREGS[arg_index]
+                    self.try_allocate_reg(box, selected_reg=loc)
+                    arg_index += 1
                 else:
                     loc = r.PARAM_REGS[arg_index]
                     self.try_allocate_reg(box, selected_reg=loc)
@@ -195,7 +241,13 @@
             else:
                 # treat stack args as stack locations with a negative offset
                 if box.type == FLOAT:
-                    assert 0, "not implemented yet"
+                    if IS_PPC_32:
+                        cur_frame_pos -= 2
+                    else:
+                        cur_frame_pos -= 1
+                    if count % 2 != 0: # Stack argument alignment
+                        cur_frame_pos -= 1
+                        count = 0
                 else:
                     cur_frame_pos -= 1
                     count += 1
@@ -255,21 +307,24 @@
 
     def _check_invariants(self):
         self.rm._check_invariants()
+        self.fprm._check_invariants()
 
     def loc(self, var):
         if var.type == FLOAT:
-            assert 0, "not implemented yet"
-        return self.rm.loc(var)
+            return self.fprm.loc(var)
+        else:
+            return self.rm.loc(var)
 
     def position(self):
         return self.rm.position
 
     def next_instruction(self):
         self.rm.next_instruction()
+        self.fprm.next_instruction()
 
     def force_spill_var(self, var):
         if var.type == FLOAT:
-            assert 0, "not implemented yet"
+            self.fprm.force_spill_var(var)
         else:
             self.rm.force_spill_var(var)
 
@@ -278,30 +333,32 @@
 
     def after_call(self, v):
         if v.type == FLOAT:
-            assert 0, "not implemented yet"
+            return self.fprm.after_call(v)
         else:
             return self.rm.after_call(v)
 
     def call_result_location(self, v):
         if v.type == FLOAT:
-            assert 0, "not implemented yet"
+            return self.fprm.call_result_location(v)
         else:
             return self.rm.call_result_location(v)
 
     def _ensure_value_is_boxed(self, thing, forbidden_vars=[]):
         if thing.type == FLOAT:
-            assert 0, "not implemented yet"
+            return self.fprm.ensure_value_is_boxed(thing, forbidden_vars)
         else:
             return self.rm.ensure_value_is_boxed(thing, forbidden_vars)
 
     def get_scratch_reg(self, type, forbidden_vars=[], selected_reg=None):
         if type == FLOAT:
-            assert 0, "not implemented yet"
+            return self.fprm.get_scratch_reg(type, forbidden_vars,
+                                                                selected_reg)
         else:
             return self.rm.get_scratch_reg(type, forbidden_vars, selected_reg)
 
     def free_temp_vars(self):
         self.rm.free_temp_vars()
+        self.fprm.free_temp_vars()
 
     def make_sure_var_in_reg(self, var, forbidden_vars=[],
                              selected_reg=None, need_lower_byte=False):
@@ -315,11 +372,12 @@
         if isinstance(value, ConstInt):
             return self.rm.convert_to_imm(value)
         else:
-            assert 0, "not implemented yet"
+            assert isinstance(value, ConstFloat)
+            return self.fprm.convert_to_imm(value)
 
     def _sync_var(self, v):
         if v.type == FLOAT:
-            assert 0, "not implemented yet"
+            self.fprm._sync_var(v)
         else:
             self.rm._sync_var(v)
 
@@ -373,6 +431,65 @@
     prepare_int_is_true = prepare_unary_cmp()
     prepare_int_is_zero = prepare_unary_cmp()
 
+    prepare_float_add = prepare_float_op(name='prepare_float_add')
+    prepare_float_sub = prepare_float_op(name='prepare_float_sub')
+    prepare_float_mul = prepare_float_op(name='prepare_float_mul')
+    prepare_float_truediv = prepare_float_op(name='prepare_float_truediv')
+
+    prepare_float_lt = prepare_float_op(float_result=False,
+                                        name='prepare_op_float_lt')
+    prepare_float_le = prepare_float_op(float_result=False,
+                                        name='prepare_op_float_le')
+    prepare_float_eq = prepare_float_op(float_result=False,
+                                        name='prepare_op_float_eq')
+    prepare_float_ne = prepare_float_op(float_result=False,
+                                        name='prepare_op_float_ne')
+    prepare_float_gt = prepare_float_op(float_result=False,
+                                        name='prepare_op_float_gt')
+    prepare_float_ge = prepare_float_op(float_result=False,
+                                        name='prepare_op_float_ge')
+    prepare_float_neg = prepare_float_op(base=False,
+                                         name='prepare_op_float_neg')
+    prepare_float_abs = prepare_float_op(base=False,
+                                         name='prepare_op_float_abs')
+
+    prepare_guard_float_lt = prepare_float_op(guard=True,
+                            float_result=False, name='prepare_guard_float_lt')
+    prepare_guard_float_le = prepare_float_op(guard=True,
+                            float_result=False, name='prepare_guard_float_le')
+    prepare_guard_float_eq = prepare_float_op(guard=True,
+                            float_result=False, name='prepare_guard_float_eq')
+    prepare_guard_float_ne = prepare_float_op(guard=True,
+                            float_result=False, name='prepare_guard_float_ne')
+    prepare_guard_float_gt = prepare_float_op(guard=True,
+                            float_result=False, name='prepare_guard_float_gt')
+    prepare_guard_float_ge = prepare_float_op(guard=True,
+                            float_result=False, name='prepare_guard_float_ge')
+
+    def prepare_math_sqrt(self, op):
+        loc = self._ensure_value_is_boxed(op.getarg(1))
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
+        res = self.fprm.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [loc, res]
+
+    def prepare_cast_float_to_int(self, op):
+        loc1 = self._ensure_value_is_boxed(op.getarg(0))
+        temp_loc = self.get_scratch_reg(FLOAT)
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
+        res = self.rm.force_allocate_reg(op.result)
+        return [loc1, temp_loc, res]
+
+    def prepare_cast_int_to_float(self, op):
+        loc1 = self._ensure_value_is_boxed(op.getarg(0))
+        temp_loc = self.get_scratch_reg(FLOAT)
+        self.possibly_free_vars_for_op(op)
+        self.free_temp_vars()
+        res = self.fprm.force_allocate_reg(op.result)
+        return [loc1, temp_loc, res]
+
     def prepare_finish(self, op):
         args = [None] * (op.numargs() + 1)
         for i in range(op.numargs()):
@@ -776,6 +893,7 @@
         for i in range(op.numargs()):
             args.append(self.loc(op.getarg(i)))
         # spill variables that need to be saved around calls
+        self.fprm.before_call(save_all_regs=save_all_regs)
         if not save_all_regs:
             gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
diff --git a/pypy/jit/backend/ppc/register.py b/pypy/jit/backend/ppc/register.py
--- a/pypy/jit/backend/ppc/register.py
+++ b/pypy/jit/backend/ppc/register.py
@@ -19,7 +19,7 @@
 
 NONVOLATILES_FLOAT  = [f14, f15, f16, f17, f18, f19, f20, f21, f22, f23,
                     f24, f25, f26, f27, f28, f29, f30, f31]
-
+VOLATILES_FLOAT  = [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13]
 
 SCRATCH = r0
 SP      = r1
@@ -33,6 +33,7 @@
                 r27, r28, r29, r30]
 
 PARAM_REGS = [r3, r4, r5, r6, r7, r8, r9, r10]
+PARAM_FPREGS = [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13]
 
 def get_managed_reg_index(reg):
     if reg > r13.value:
diff --git a/pypy/jit/backend/ppc/runner.py b/pypy/jit/backend/ppc/runner.py
--- a/pypy/jit/backend/ppc/runner.py
+++ b/pypy/jit/backend/ppc/runner.py
@@ -27,7 +27,7 @@
                                translate_support_code, gcdescr)
 
         # floats are not supported yet
-        self.supports_floats = False
+        self.supports_floats = True
 
     def setup(self):
         self.asm = AssemblerPPC(self)

From noreply at buildbot.pypy.org  Mon Apr 23 20:31:50 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 23 Apr 2012 20:31:50 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Add float support to
 count_reg_args. Add float support to regalloc_mov. Fix arglocs typo.
Message-ID: <20120423183150.93B138203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54690:215a02bf68e1
Date: 2012-04-23 14:31 -0400
http://bitbucket.org/pypy/pypy/changeset/215a02bf68e1/

Log:	Add float support to count_reg_args. Add float support to
	regalloc_mov. Fix arglocs typo.

diff --git a/pypy/jit/backend/ppc/helper/assembler.py b/pypy/jit/backend/ppc/helper/assembler.py
--- a/pypy/jit/backend/ppc/helper/assembler.py
+++ b/pypy/jit/backend/ppc/helper/assembler.py
@@ -61,7 +61,8 @@
     count = 0
     for x in range(min(len(args), MAX_REG_PARAMS)):
         if args[x].type == FLOAT:
-            assert 0, "not implemented yet"
+            count += 1
+            words += 1
         else:
             count += 1
             words += 1
diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -177,31 +177,31 @@
 class FloatOpAssembler(object):
     _mixin_ = True
 
-    def emit_float_add(self, op, argsloc, regalloc):
+    def emit_float_add(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         self.mc.fadd(res.value, l0.value, l1.value)
 
-    def emit_float_sub(self, op, argsloc, regalloc):
+    def emit_float_sub(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         self.mc.fsub(res.value, l0.value, l1.value)
 
-    def emit_float_mul(self, op, argsloc, regalloc):
+    def emit_float_mul(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         self.mc.fmul(res.value, l0.value, l1.value)
 
-    def emit_float_truediv(self, op, argsloc, regalloc):
+    def emit_float_truediv(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         self.mc.fdiv(res.value, l0.value, l1.value)
 
-    def emit_float_neg(self, op, argsloc, regalloc):
+    def emit_float_neg(self, op, arglocs, regalloc):
         l0, res = arglocs
         self.mc.fneg(res.value, l0.value)
 
-    def emit_float_abs(self, op, argsloc, regalloc):
+    def emit_float_abs(self, op, arglocs, regalloc):
         l0, res = arglocs
         self.mc.fabs(res.value, l0.value)
 
-    def emit_float_sqrt(self, op, argsloc, regalloc):
+    def emit_float_sqrt(self, op, arglocs, regalloc):
         l0, res = arglocs
         self.mc.fsqrt(res.value, l0.value)
 
diff --git a/pypy/jit/backend/ppc/ppc_assembler.py b/pypy/jit/backend/ppc/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppc_assembler.py
@@ -1168,6 +1168,17 @@
                     self.mc.load(r.SCRATCH.value, r.SPP.value, offset)
                     self.mc.store(r.SCRATCH.value, r.SPP.value, target_offset)
                 return
+            elif loc.is_fp_reg():
+                assert prev_loc.type == FLOAT, 'source not float location'
+                with scratch_reg(self.mc):
+                    offset = prev_loc.value
+                    if not _check_imm_arg(offset):
+                        self.mc.load_imm(r.SCRATCH, offset)
+                        self.mc.sub(r.SCRATCH.value, r.SPP.value, r.SCRATCH.value)
+                    else:
+                        self.mc.sub(r.SCRATCH.value, r.SPP.value, r.SCRARTCH.value)
+                    self.mc.lfdx(loc.value, 0, r.SCRATCH.value)
+                return
             assert 0, "not supported location"
         elif prev_loc.is_reg():
             reg = prev_loc.as_key()
@@ -1182,6 +1193,35 @@
                 self.mc.store(reg, r.SPP.value, offset)
                 return
             assert 0, "not supported location"
+        elif prev_loc.is_imm_float():
+            if loc.is_fp_reg():
+                with scratch_reg(self.mc):
+                    self.mc.load_imm(r.SCRATCH, value)
+                    self.mc.lfdx(loc, 0, r.SCRATCH)
+                return
+            elif loc.is_stack():
+                with scratch_reg(self.mc):
+                    offset = loc.value
+                    self.mc.load_imm(r.SCRATCH, value)
+                    self.mc.store(r.SCRATCH.value, r.SPP.value, offset)
+                return
+            assert 0, "not supported location"
+        elif prev_loc.is_fp_reg():
+            if loc.is_fp_reg():
+                self.mc.fmr(loc.value, prev_loc.value)
+                return
+            elif loc.is_stack():
+                assert loc.type == FLOAT, "target not float location"
+                with scratch_reg(self.mc):
+                    offset = loc.value
+                    if not _check_imm_arg(offset):
+                        self.mc.load_imm(r.SCRATCH, offset)
+                        self.mc.sub(r.SCRATCH.value, r.SPP.value, offset)
+                    else:
+                        self.mc.sub(r.SCRATCH.value, r.SPP.value, r.SCRATCH.value)
+                    self.mc.stfdx(prev_loc.value, 0, r.SCRATCH.value)
+                return
+            assert 0, "not supported location"
         assert 0, "not supported location"
     mov_loc_loc = regalloc_mov
 
diff --git a/pypy/jit/backend/ppc/regalloc.py b/pypy/jit/backend/ppc/regalloc.py
--- a/pypy/jit/backend/ppc/regalloc.py
+++ b/pypy/jit/backend/ppc/regalloc.py
@@ -1,6 +1,7 @@
 from pypy.jit.backend.llsupport.regalloc import (RegisterManager, FrameManager,
                                                  TempBox, compute_vars_longevity)
 from pypy.jit.backend.ppc.arch import (WORD, MY_COPY_OF_REGS, IS_PPC_32)
+from pypy.jit.codewriter import longlong
 from pypy.jit.backend.ppc.jump import remap_frame_layout
 from pypy.jit.backend.ppc.locations import imm
 from pypy.jit.backend.ppc.helper.regalloc import (_check_imm_arg,
@@ -10,7 +11,7 @@
                                                   prepare_binary_int_op_with_imm,
                                                   prepare_unary_cmp,
                                                   prepare_float_op)
-from pypy.jit.metainterp.history import (Const, ConstInt, ConstPtr,
+from pypy.jit.metainterp.history import (Const, ConstInt, ConstFloat, ConstPtr,
                                          Box, BoxPtr,
                                          INT, REF, FLOAT)
 from pypy.jit.metainterp.history import JitCellToken, TargetToken
@@ -57,7 +58,7 @@
     def convert_to_imm(self, c):
         adr = self.assembler.datablockwrapper.malloc_aligned(8, 8)
         x = c.getfloatstorage()
-        rffi.cast(rffi.CArrayPtr(long.FLOATSTORAGE), adr)[0] = x
+        rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0] = x
         return locations.ConstFloatLoc(adr)
 
     def __init__(self, longevity, frame_manager=None, assembler=None):

From noreply at buildbot.pypy.org  Mon Apr 23 21:05:00 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 23 Apr 2012 21:05:00 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: fix tests: 7 is a valid signal
 enum on some platforms, rework bad expression
Message-ID: <20120423190500.D351C8203C@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54691:37a21c90d685
Date: 2012-04-23 22:04 +0300
http://bitbucket.org/pypy/pypy/changeset/37a21c90d685/

Log:	fix tests: 7 is a valid signal enum on some platforms, rework bad
	expression

diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py
--- a/pypy/module/signal/test/test_signal.py
+++ b/pypy/module/signal/test/test_signal.py
@@ -52,9 +52,9 @@
         if not hasattr(os, 'kill') or not hasattr(os, 'getpid'):
             skip("requires os.kill() and os.getpid()")
         signal = self.signal   # the signal module to test
-        try:
-            signum = signal.USR1
-        except:
+        if hasattr(signal,'SIGUSR1'):
+            signum = signal.SIGUSR1
+        else:
             signum = signal.CTRL_BREAK_EVENT
 
         received = []
@@ -154,7 +154,7 @@
 
         raises(ValueError, getsignal, 4444)
         raises(ValueError, signal, 4444, lambda *args: None)
-        raises(ValueError, signal, 7, lambda *args: None)
+        raises(ValueError, signal, 42, lambda *args: None)
 
     def test_alarm(self):
         try:

From noreply at buildbot.pypy.org  Mon Apr 23 21:11:09 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 21:11:09 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Complete the tests and the code.
Message-ID: <20120423191109.7A0C28203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54692:3e6d963c1ab2
Date: 2012-04-23 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/3e6d963c1ab2/

Log:	Complete the tests and the code.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -47,6 +47,10 @@
         # --- the LOCAL objects which are weakrefs.  They are also listed
         #     in the appropriate place, like sharedarea_tls, if needed.
         self.local_weakrefs = self.AddressStack()
+        # --- main thread only: this is the list of GLOBAL objects that
+        #     have been turned into LOCAL objects
+        if in_main_thread:
+            self.main_thread_was_global_objects = NULL
         #
         self._register_with_C_code()
 
@@ -105,6 +109,16 @@
         """Called on the main thread, just before spawning the other
         threads."""
         self.stop_transaction()
+        #
+        # We must also mark the following objects as GLOBAL again
+        obj = self.main_thread_was_global_objects
+        self.main_thread_was_global_objects = NULL
+        while obj:
+            hdr = self.gc.header(obj)
+            hdr.tid |= GCFLAG_GLOBAL
+            obj = hdr.version
+        if not we_are_translated():
+            del self.main_thread_was_global_objects   # don't use any more
 
     def leave_transactional_mode(self):
         """Restart using the main thread for mallocs."""
@@ -113,11 +127,16 @@
                 if value is not self:
                     del StmGCTLS.nontranslated_dict[key]
         self.start_transaction()
-        # do something special here after we restarted the "transaction"
-        # in the main thread: we have to make sure that the write_barrier
-        # calls done in the main thread before enter/leave_transactional_mode
-        # are still pointing to local objects.  Conservatively, we mark as
-        # local all objects directly referenced from the stack.
+        #
+        # Do something special here after we restarted the "transaction"
+        # in the main thread.  At this point, *all* objects are GLOBAL.
+        # The write_barrier will ensure that any write makes the written-to
+        # objects LOCAL again.  However, it is possible that the write
+        # barrier was called before the enter/leave_transactional_mode()
+        # and will not be called again before writing.  But such objects
+        # are right now directly in the stack.  So to fix this issue, we
+        # conservatively mark as local all objects directly from the stack.
+        self.main_thread_was_global_objects = NULL
         self.gc.root_walker.walk_current_stack_roots(
             StmGCTLS._remark_object_as_local, self)
 
@@ -195,6 +214,8 @@
         # objects.
         if not self.in_main_thread:
             self.collect_roots_from_tldict()
+        else:
+            self.collect_from_main_thread_was_global_objects()
         #
         # Now repeatedly follow objects until 'pending' is empty.
         self.collect_flush_pending()
@@ -264,14 +285,19 @@
         self.local_weakrefs.append(obj)
 
     def _remark_object_as_local(self, root):
-        self.main_thread_writes_to_global_obj(root.address[0])
+        obj = root.address[0]
+        hdr = self.gc.header(obj)
+        if hdr.tid & GCFLAG_GLOBAL:
+            self.main_thread_writes_to_global_obj(obj)
 
     def main_thread_writes_to_global_obj(self, obj):
         hdr = self.gc.header(obj)
         ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0,
                   "write in main thread: unexpected GCFLAG_WAS_COPIED")
         hdr.tid &= ~GCFLAG_GLOBAL
-        self.sharedarea_tls.add_regular(obj)
+        # add the object into this linked list
+        hdr.version = self.main_thread_was_global_objects
+        self.main_thread_was_global_objects = obj
 
     # ------------------------------------------------------------
 
@@ -474,6 +500,19 @@
         #
         self.trace_and_drag_out_of_nursery(localobj)
 
+    def collect_from_main_thread_was_global_objects(self):
+        # NB. all objects in the 'main_thread_was_global_objects' list are
+        # currently immortal (because they were once GLOBAL)
+        obj = self.main_thread_was_global_objects
+        while obj:
+            hdr = self.gc.header(obj)
+            ll_assert(hdr.tid & GCFLAG_GLOBAL == 0,
+                      "unexpected GLOBAL in main_thread_was_global_objects")
+            if hdr.tid & GCFLAG_VISITED == 0:
+                hdr.tid |= GCFLAG_VISITED
+                self.pending.append(obj)
+            obj = hdr.version
+
     def collect_flush_pending(self):
         # Follow the objects in the 'pending' stack and move the
         # young objects they point to out of the nursery.
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -342,23 +342,56 @@
         from pypy.rpython.memory.gc.test import test_stmtls
         self.gc.root_walker = test_stmtls.FakeRootWalker()
         #
-        tr1, tr1_adr = self.malloc(SR, globl=True)
+        tr1, tr1_adr = self.malloc(SR, globl=True)   # three prebuilt objects
         tr2, tr2_adr = self.malloc(SR, globl=True)
+        tr3, tr3_adr = self.malloc(SR, globl=True)
         tr1.sr2 = tr2
         self.gc.root_walker.current_stack = [tr1]
         obj = self.gc.stm_writebarrier(tr1_adr)
         assert obj == tr1_adr
         obj = self.gc.stm_writebarrier(tr2_adr)
         assert obj == tr2_adr
+        obj = self.gc.stm_writebarrier(tr3_adr)
+        assert obj == tr3_adr
         self.checkflags(tr1_adr, False, False)    # tr1 has become local
         self.checkflags(tr2_adr, False, False)    # tr2 has become local
+        self.checkflags(tr3_adr, False, False)    # tr3 has become local
         #
         self.gc.enter_transactional_mode()
         self.gc.leave_transactional_mode()
         self.checkflags(tr2_adr, True, False)     # tr2 has become global again
+        self.checkflags(tr3_adr, True, False)     # tr3 has become global again
         # but tr1 is still local, because it is directly referenced from stack
         self.checkflags(tr1_adr, False, False)
 
+    def test_non_prebuilt_relocalize_after_transactional_mode(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        #
+        tr1, tr1_adr = self.malloc(SR)            # local
+        tr2, tr2_adr = self.malloc(SR)            # local
+        tr1.sr2 = tr2
+        self.gc.root_walker.current_stack = [tr1]
+        self.gc.enter_transactional_mode()
+        self.checkflags(tr1_adr, True, False)     # tr1 has become global
+        self.checkflags(tr2_adr, True, False)     # tr2 has become global
+        self.gc.leave_transactional_mode()
+        self.checkflags(tr2_adr, True, False)     # tr2 is still global
+        self.checkflags(tr1_adr, False, False)    # tr1 is back to a local
+
+    def test_collect_from_main_thread_was_global_objects(self):
+        tr1, tr1_adr = self.malloc(SR, globl=True)  # a global prebuilt object
+        tr2, tr2_adr = self.malloc(SR, globl=False) # tr2 is a local
+        self.checkflags(tr2_adr, False, False)      # check that tr2 is a local
+        obj = self.gc.stm_writebarrier(tr1_adr)
+        assert obj == tr1_adr                       # tr1 is now local
+        tr1.sr2 = tr2
+        self.gc.collect(0)
+        tr2 = tr1.sr2       # reload, because it moved
+        tr2_adr = llmemory.cast_ptr_to_adr(tr2)
+        self.checkflags(tr2_adr, False, False)  # tr2 is still alive and local
+        self.checkflags(tr1_adr, False, False)  # tr1 is still local
+
     def test_commit_transaction_empty(self):
         self.select_thread(1)
         s, s_adr = self.malloc(S)

From noreply at buildbot.pypy.org  Mon Apr 23 21:11:10 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 21:11:10 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix.
Message-ID: <20120423191110.B68C08203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54693:8f4784f89e0c
Date: 2012-04-23 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/8f4784f89e0c/

Log:	Fix.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -113,10 +113,7 @@
         # We must also mark the following objects as GLOBAL again
         obj = self.main_thread_was_global_objects
         self.main_thread_was_global_objects = NULL
-        while obj:
-            hdr = self.gc.header(obj)
-            hdr.tid |= GCFLAG_GLOBAL
-            obj = hdr.version
+        self._promote_list_to_globals(obj)
         if not we_are_translated():
             del self.main_thread_was_global_objects   # don't use any more
 
@@ -308,6 +305,9 @@
         obj = self.sharedarea_tls.chained_list
         self.sharedarea_tls.chained_list = NULL
         #
+        self._promote_list_to_globals(obj)
+
+    def _promote_list_to_globals(self, obj):
         while obj:
             hdr = self.gc.header(obj)
             obj = hdr.version
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -80,6 +80,7 @@
 
 class InitialTransaction(rstm.Transaction):
     def run(self):
+        debug_print("InitialTransaction.run", self.retry_counter)
         ll_assert(self.retry_counter == 0, "no reason to abort-and-retry here")
         ll_assert(rstm.thread_id() != 0, "thread_id == 0")
         scheduled = []

From noreply at buildbot.pypy.org  Mon Apr 23 21:11:11 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 21:11:11 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Documentation, documentation,
 documentation.  It helps a lot
Message-ID: <20120423191111.F37598203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54694:c4fe5a7ab290
Date: 2012-04-23 21:10 +0200
http://bitbucket.org/pypy/pypy/changeset/c4fe5a7ab290/

Log:	Documentation, documentation, documentation. It helps a lot to make
	my confused mind a bit clearer, and it should trigger a few
	simplifications too.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -28,7 +28,7 @@
 #       - the non-small raw-malloced objects
 #
 #   - The GLOBAL objects are all located in the shared area.
-#     All global objects are non-movable.
+#     All GLOBAL objects are non-movable.
 #
 #   - The LOCAL objects might be YOUNG or OLD depending on whether they
 #     already survived a collection.  YOUNG LOCAL objects are either in
@@ -38,11 +38,112 @@
 #     is not actually generational (slow when running long transactions
 #     or before running transactions at all).
 #
+#   - A few details are different depending on the running mode:
+#     either "transactional" or "non-transactional".  The transactional
+#     mode is where we have multiple threads, in a transaction.run()
+#     call.  The non-transactional mode has got only the main thread.
+#
+# GC Flags on objects:
+#
+#   - GCFLAG_GLOBAL: identifies GLOBAL objects.  All prebuilt objects
+#     start as GLOBAL; conversely, all freshly allocated objects start
+#     as LOCAL.  But they may switch between the two; see below.
+#     All objects that are or have been GLOBAL are immortal for now
+#     (global_collect() will be done later).
+#
+#   - GCFLAG_WAS_COPIED: means that the object is either a LOCAL COPY
+#     or, if GLOBAL, then it has or had at least one LOCAL COPY.  Used
+#     in transactional mode only; see below.
+#
+#   - GCFLAG_VISITED: used during collections to flag objects found to be
+#     surviving.  Between collections, it must be set on LOCAL COPY objects
+#     and only on them.
+#
+#   - GCFLAG_HAS_SHADOW: set on nursery objects whose id() or identityhash()
+#     was taken.  Means that we already have a corresponding object allocated
+#     outside the nursery.
+#
+#   - GCFLAG_FIXED_HASH: only on some prebuilt objects.  For identityhash().
+#
+# When the mutator (= the program outside the GC) wants to write to an
+# object, stm_writebarrier() does something special on GLOBAL objects:
+#
+#   - In non-transactional mode, the write barrier turns the object LOCAL
+#     and add it in the list 'main_thread_tls.mt_global_turned_local'.
+#     This list contains all previously-GLOBAL objects that have been
+#     modified.  Objects turned LOCAL are changed back to GLOBAL and
+#     removed from 'mt_global_turned_local' by the next collection,
+#     unless they are also found in the stack (the reason being that if
+#     they are in the stack and stm_writebarrier() has already been
+#     called, then it might not be called a second time if they are
+#     changed again after collection).
+#
+#   - In transactional mode, the write barrier creates a LOCAL COPY of
+#     the object and returns it (or, if already created by the same
+#     transaction, finds it again).  The list of LOCAL COPY objects has
+#     a role similar to 'mt_global_turned_local', but is maintained by C
+#     code (see tldict_lookup()).
+#
+# Invariant: between two transactions, all objects visible from the current
+# thread are always GLOBAL.  In particular:
+#
+#   - The LOCAL object of a thread are not visible at all from other threads.
+#     This means that in transactional mode there is *no* pointer from a
+#     GLOBAL object directly to a LOCAL object.
+#
+#   - At the end of enter_transactional_mode(), and at the beginning of
+#     leave_transactional_mode(), *all* objects everywhere are GLOBAL.
+#
+# Collection: for now we have only local_collection(), which ignores all
+# GLOBAL objects.
+#
+#   - In non-transactional mode, we use 'mt_global_turned_local' as a list
+#     of roots, together with the stack.  By construction, all objects that
+#     are still GLOBAL can be ignored, because they cannot point to a LOCAL
+#     object (except to a 'mt_global_turned_local' object).
+#
+#   - In transactional mode, we similarly use the list maintained by C code
+#     of the LOCAL COPY objects of the current transaction, together with
+#     the stack.  Again, GLOBAL objects can be ignored because they have no
+#     pointer to any LOCAL object at all in that mode.
+#
+#   - A special case is the end-of-transaction collection, done by the same
+#     local_collection() with a twist: all pointers to a LOCAL COPY object
+#     are replaced with copies to the corresponding GLOBAL original.  When
+#     it is done, we mark all surviving LOCAL objects as GLOBAL too, and we
+#     are back to the situation where this thread sees only GLOBAL objects.
+#     What we leave to the C code to do "as a finishing touch" is to copy
+#     transactionally the content of the LOCAL COPY objects back over the
+#     GLOBAL originals; before this is done, the transaction can be aborted
+#     at any point with no visible side-effect on any object that other
+#     threads can see.
+#
+# All objects have an address-sized 'version' field in their header.  On
+# GLOBAL objects, it is used as a version by C code to handle STM (it must
+# be set to 0 when the object first turns GLOBAL).   On the LOCAL objects,
+# though, it is abused here in the GC:
+#
+#   - if GCFLAG_WAS_COPIED, it points to the GLOBAL original.
+#
+#   - if GCFLAG_HAS_SHADOW, to the shadow object outside the nursery.
+#     (It is not used on any other nursery object.)
+#
+#   - it contains the 'next' object of the 'mt_global_turned_local' list.
+#
+#   - it contains the 'next' object of the 'sharedarea_tls.chained_list'
+#     list, which describes all LOCAL objects malloced outside the nursery
+#     (excluding the ones that were GLOBAL at some point).
+#
+#   - for nursery objects, during collection, if they are copied outside
+#     the nursery, they grow GCFLAG_VISITED and their 'version' points
+#     to the fresh copy.
+#
+
 GCFLAG_GLOBAL     = first_gcflag << 0     # keep in sync with et.c
 GCFLAG_WAS_COPIED = first_gcflag << 1     # keep in sync with et.c
-GCFLAG_HAS_SHADOW = first_gcflag << 2
-GCFLAG_FIXED_HASH = first_gcflag << 3
-GCFLAG_VISITED    = first_gcflag << 4
+GCFLAG_VISITED    = first_gcflag << 2
+GCFLAG_HAS_SHADOW = first_gcflag << 3
+GCFLAG_FIXED_HASH = first_gcflag << 4
 
 
 def always_inline(fn):
@@ -320,8 +421,8 @@
             # The raw copy done above does not include the header fields.
             hdr = self.header(obj)
             localhdr = self.header(localobj)
-            GCFLAGS = (GCFLAG_GLOBAL | GCFLAG_WAS_COPIED)
-            ll_assert(hdr.tid & GCFLAGS == GCFLAGS,
+            GCFLAGS = (GCFLAG_GLOBAL | GCFLAG_WAS_COPIED | GCFLAG_VISITED)
+            ll_assert(hdr.tid & GCFLAGS == (GCFLAG_GLOBAL | GCFLAG_WAS_COPIED),
                       "stm_write: bogus flags on source object")
             #
             # Remove the GCFLAG_GLOBAL from the copy, and add GCFLAG_VISITED
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -50,7 +50,7 @@
         # --- main thread only: this is the list of GLOBAL objects that
         #     have been turned into LOCAL objects
         if in_main_thread:
-            self.main_thread_was_global_objects = NULL
+            self.mt_global_turned_local = NULL
         #
         self._register_with_C_code()
 
@@ -111,11 +111,11 @@
         self.stop_transaction()
         #
         # We must also mark the following objects as GLOBAL again
-        obj = self.main_thread_was_global_objects
-        self.main_thread_was_global_objects = NULL
+        obj = self.mt_global_turned_local
+        self.mt_global_turned_local = NULL
         self._promote_list_to_globals(obj)
         if not we_are_translated():
-            del self.main_thread_was_global_objects   # don't use any more
+            del self.mt_global_turned_local   # don't use any more
 
     def leave_transactional_mode(self):
         """Restart using the main thread for mallocs."""
@@ -133,7 +133,7 @@
         # and will not be called again before writing.  But such objects
         # are right now directly in the stack.  So to fix this issue, we
         # conservatively mark as local all objects directly from the stack.
-        self.main_thread_was_global_objects = NULL
+        self.mt_global_turned_local = NULL
         self.gc.root_walker.walk_current_stack_roots(
             StmGCTLS._remark_object_as_local, self)
 
@@ -212,7 +212,7 @@
         if not self.in_main_thread:
             self.collect_roots_from_tldict()
         else:
-            self.collect_from_main_thread_was_global_objects()
+            self.collect_from_mt_global_turned_local()
         #
         # Now repeatedly follow objects until 'pending' is empty.
         self.collect_flush_pending()
@@ -293,8 +293,8 @@
                   "write in main thread: unexpected GCFLAG_WAS_COPIED")
         hdr.tid &= ~GCFLAG_GLOBAL
         # add the object into this linked list
-        hdr.version = self.main_thread_was_global_objects
-        self.main_thread_was_global_objects = obj
+        hdr.version = self.mt_global_turned_local
+        self.mt_global_turned_local = obj
 
     # ------------------------------------------------------------
 
@@ -500,14 +500,14 @@
         #
         self.trace_and_drag_out_of_nursery(localobj)
 
-    def collect_from_main_thread_was_global_objects(self):
-        # NB. all objects in the 'main_thread_was_global_objects' list are
+    def collect_from_mt_global_turned_local(self):
+        # NB. all objects in the 'mt_global_turned_local' list are
         # currently immortal (because they were once GLOBAL)
-        obj = self.main_thread_was_global_objects
+        obj = self.mt_global_turned_local
         while obj:
             hdr = self.gc.header(obj)
             ll_assert(hdr.tid & GCFLAG_GLOBAL == 0,
-                      "unexpected GLOBAL in main_thread_was_global_objects")
+                      "unexpected GLOBAL in mt_global_turned_local")
             if hdr.tid & GCFLAG_VISITED == 0:
                 hdr.tid |= GCFLAG_VISITED
                 self.pending.append(obj)

From noreply at buildbot.pypy.org  Mon Apr 23 21:19:26 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 23 Apr 2012 21:19:26 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Add float support to emit_finish.
Message-ID: <20120423191926.5D5428203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54695:a98d6daa8fec
Date: 2012-04-23 15:19 -0400
http://bitbucket.org/pypy/pypy/changeset/a98d6daa8fec/

Log:	Add float support to emit_finish.

diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -346,12 +346,20 @@
                 with scratch_reg(self.mc):
                     self.mc.load_imm(r.SCRATCH, adr)
                     self.mc.storex(loc.value, 0, r.SCRATCH.value)
-            elif loc.is_vfp_reg():
+            elif loc.is_fp_reg():
                 assert box.type == FLOAT
-                assert 0, "not implemented yet"
+                adr = self.fail_boxes_float.get_addr_for_num(i)
+                with scratch_reg(self.mc):
+                    self.mc.load_imm(r.SCRATCH, adr)
+                    self.mc.stfdx(loc.value, 0, r.SCRATCH.value)
             elif loc.is_stack() or loc.is_imm() or loc.is_imm_float():
                 if box.type == FLOAT:
-                    assert 0, "not implemented yet"
+                    adr = self.fail_boxes_float.get_addr_for_num(i)
+                    self.mc.stfd(r.f0.value, r.SPP.value, 0)
+                    self.mov_loc_loc(loc, r.f0.value)
+                    self.mc.load_imm(r.SCRATCH, adr)
+                    self.mc.stfdx(r.f0.value, 0, r.SCRATCH.value)
+                    self.mc.lfd(r.f0.value, r.SPP.value, 0)
                 elif box.type == REF or box.type == INT:
                     if box.type == REF:
                         adr = self.fail_boxes_ptr.get_addr_for_num(i)
@@ -459,7 +467,8 @@
             for i in range(reg_args, n_args):
                 arg = arglocs[i]
                 if arg.type == FLOAT:
-                    assert 0, "not implemented yet"
+                    count += 1
+                    n += WORD
                 else:
                     count += 1
                     n += WORD
diff --git a/pypy/jit/backend/ppc/ppc_assembler.py b/pypy/jit/backend/ppc/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppc_assembler.py
@@ -20,6 +20,7 @@
 from pypy.jit.backend.model import CompiledLoopToken
 from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.codewriter import longlong
 from pypy.jit.metainterp.history import (INT, REF, FLOAT)
 from pypy.jit.backend.x86.support import values_array
 from pypy.rlib.debug import (debug_print, debug_start, debug_stop,
@@ -83,6 +84,8 @@
     def __init__(self, cpu, failargs_limit=1000):
         self.cpu = cpu
         self.fail_boxes_int = values_array(lltype.Signed, failargs_limit)
+        self.fail_boxes_float = values_array(longlong.FLOATSTORAGE,
+                                                            failargs_limit)
         self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit)
         self.mc = None
         self.datablockwrapper = None
@@ -1371,11 +1374,14 @@
             self.mc.store(r.SCRATCH.value, r.SPP.value, self.FORCE_INDEX_AREA)
             
     def load(self, loc, value):
-        assert loc.is_reg() and value.is_imm()
+        assert (loc.is_reg() and value.is_imm()
+                or loc.is_fp_reg() and value.is_imm_float())
         if value.is_imm():
             self.mc.load_imm(loc, value.getint())
         elif value.is_imm_float():
-            assert 0, "not implemented yet"
+            with scratch_reg(self.mc):
+                self.mc.load_imm(r.SCRATCH, value.getint())
+                self.mc.lfdx(loc.value, 0, r.SCRATCH.value)
 
 def notimplemented_op(self, op, arglocs, regalloc):
     print "[PPC/asm] %s not implemented" % op.getopname()

From noreply at buildbot.pypy.org  Mon Apr 23 21:20:46 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 21:20:46 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Mark a point that is not done.
Message-ID: <20120423192046.817028203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54696:bfaf482a7edb
Date: 2012-04-23 21:17 +0200
http://bitbucket.org/pypy/pypy/changeset/bfaf482a7edb/

Log:	Mark a point that is not done.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -71,7 +71,7 @@
 #   - In non-transactional mode, the write barrier turns the object LOCAL
 #     and add it in the list 'main_thread_tls.mt_global_turned_local'.
 #     This list contains all previously-GLOBAL objects that have been
-#     modified.  Objects turned LOCAL are changed back to GLOBAL and
+#     modified.  [XXX:TODO]Objects turned LOCAL are changed back to GLOBAL and
 #     removed from 'mt_global_turned_local' by the next collection,
 #     unless they are also found in the stack (the reason being that if
 #     they are in the stack and stm_writebarrier() has already been
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -133,6 +133,8 @@
         # and will not be called again before writing.  But such objects
         # are right now directly in the stack.  So to fix this issue, we
         # conservatively mark as local all objects directly from the stack.
+        # XXX TODO: do the same thing after each local_collection() by
+        # the main thread.
         self.mt_global_turned_local = NULL
         self.gc.root_walker.walk_current_stack_roots(
             StmGCTLS._remark_object_as_local, self)

From noreply at buildbot.pypy.org  Mon Apr 23 21:33:40 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 23 Apr 2012 21:33:40 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Tweak to make mt_global_turned_local and
 the tldict more similar.
Message-ID: <20120423193340.033898203C@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54697:ae1257934487
Date: 2012-04-23 21:32 +0200
http://bitbucket.org/pypy/pypy/changeset/ae1257934487/

Log:	Tweak to make mt_global_turned_local and the tldict more similar.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -56,8 +56,9 @@
 #     in transactional mode only; see below.
 #
 #   - GCFLAG_VISITED: used during collections to flag objects found to be
-#     surviving.  Between collections, it must be set on LOCAL COPY objects
-#     and only on them.
+#     surviving.  Between collections, it must be set on the LOCAL COPY
+#     objects or the ones from 'mt_global_turned_local' (see below), and
+#     only on them.
 #
 #   - GCFLAG_HAS_SHADOW: set on nursery objects whose id() or identityhash()
 #     was taken.  Means that we already have a corresponding object allocated
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -113,7 +113,13 @@
         # We must also mark the following objects as GLOBAL again
         obj = self.mt_global_turned_local
         self.mt_global_turned_local = NULL
-        self._promote_list_to_globals(obj)
+        while obj:
+            hdr = self.gc.header(obj)
+            obj = hdr.version
+            ll_assert(hdr.tid & GCFLAG_GLOBAL == 0, "already GLOBAL [2]")
+            ll_assert(hdr.tid & GCFLAG_VISITED != 0, "missing VISITED [2]")
+            hdr.version = NULL
+            hdr.tid += GCFLAG_GLOBAL - GCFLAG_VISITED
         if not we_are_translated():
             del self.mt_global_turned_local   # don't use any more
 
@@ -232,7 +238,7 @@
         # don't have GCFLAG_VISITED.  As the newly allocated nursery
         # objects don't have it either, at the start of the next
         # collection, the only LOCAL objects that have it are the ones
-        # in 'copied_local_objects'.
+        # in 'mt_global_turned_local' or the C tldict with GCFLAG_WAS_COPIED.
         #
         # All live nursery objects are out, and the rest dies.  Fill
         # the whole nursery with zero and reset the current nursery pointer.
@@ -291,9 +297,11 @@
 
     def main_thread_writes_to_global_obj(self, obj):
         hdr = self.gc.header(obj)
-        ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0,
-                  "write in main thread: unexpected GCFLAG_WAS_COPIED")
-        hdr.tid &= ~GCFLAG_GLOBAL
+        ll_assert(hdr.tid & (GCFLAG_WAS_COPIED|GCFLAG_VISITED) == 0,
+                  "write in main thread: unexpected GCFLAG_WAS_COPIED"
+                  " or GCFLAG_VISITED")
+        # remove GCFLAG_GLOBAL, and add GCFLAG_VISITED
+        hdr.tid += (GCFLAG_VISITED - GCFLAG_GLOBAL)
         # add the object into this linked list
         hdr.version = self.mt_global_turned_local
         self.mt_global_turned_local = obj
@@ -307,13 +315,11 @@
         obj = self.sharedarea_tls.chained_list
         self.sharedarea_tls.chained_list = NULL
         #
-        self._promote_list_to_globals(obj)
-
-    def _promote_list_to_globals(self, obj):
         while obj:
             hdr = self.gc.header(obj)
             obj = hdr.version
-            ll_assert(not (hdr.tid & GCFLAG_GLOBAL), "already GLOBAL [1]")
+            ll_assert(hdr.tid & GCFLAG_GLOBAL == 0, "already GLOBAL [1]")
+            ll_assert(hdr.tid & GCFLAG_VISITED == 0, "unexpected VISITED [1]")
             hdr.version = NULL
             hdr.tid |= GCFLAG_GLOBAL
 
@@ -509,10 +515,10 @@
         while obj:
             hdr = self.gc.header(obj)
             ll_assert(hdr.tid & GCFLAG_GLOBAL == 0,
-                      "unexpected GLOBAL in mt_global_turned_local")
-            if hdr.tid & GCFLAG_VISITED == 0:
-                hdr.tid |= GCFLAG_VISITED
-                self.pending.append(obj)
+                      "unexpected GCFLAG_GLOBAL in mt_global_turned_local")
+            ll_assert(hdr.tid & GCFLAG_VISITED != 0,
+                      "missing GCFLAG_VISIBLE in mt_global_turned_local")
+            self.trace_and_drag_out_of_nursery(obj)
             obj = hdr.version
 
     def collect_flush_pending(self):

From noreply at buildbot.pypy.org  Mon Apr 23 23:35:39 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 23 Apr 2012 23:35:39 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Use is_fp_reg() in
 AssemblerLocation. Start emit_call float support.
Message-ID: <20120423213539.A432C8203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54698:08ea7f321551
Date: 2012-04-23 17:35 -0400
http://bitbucket.org/pypy/pypy/changeset/08ea7f321551/

Log:	Use is_fp_reg() in AssemblerLocation. Start emit_call float support.

diff --git a/pypy/jit/backend/ppc/locations.py b/pypy/jit/backend/ppc/locations.py
--- a/pypy/jit/backend/ppc/locations.py
+++ b/pypy/jit/backend/ppc/locations.py
@@ -23,7 +23,7 @@
     def is_reg(self):
         return False
 
-    def is_vfp_reg(self):
+    def is_fp_reg(self):
         return False
 
     def is_imm_float(self):
diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -503,17 +503,18 @@
         for i in range(reg_args):
             arg = arglocs[i]
             if arg.type == FLOAT and count % 2 != 0:
-                assert 0, "not implemented yet"
+                num += 1
+                count = 0
             reg = r.PARAM_REGS[num]
 
             if arg.type == FLOAT:
-                assert 0, "not implemented yet"
+                float_locs.append((arg, reg))
             else:
                 non_float_locs.append(arg)
                 non_float_regs.append(reg)
 
             if arg.type == FLOAT:
-                assert 0, "not implemented yet"
+                num += 1
             else:
                 num += 1
                 count += 1

From noreply at buildbot.pypy.org  Mon Apr 23 23:59:27 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 23 Apr 2012 23:59:27 +0200 (CEST)
Subject: [pypy-commit] pypy default: (sbehnel) cpyext: Inline the fast path
 of Py_INCREF and Py_DECREF, 
Message-ID: <20120423215927.601528203C@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54699:0feb5408579b
Date: 2012-04-23 23:56 +0200
http://bitbucket.org/pypy/pypy/changeset/0feb5408579b/

Log:	(sbehnel) cpyext: Inline the fast path of Py_INCREF and Py_DECREF,
	to avoid costly function calls.

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -38,10 +38,19 @@
 	PyObject_VAR_HEAD
 } PyVarObject;
 
+#ifndef PYPY_DEBUG_REFCOUNT
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)ob))
 #define Py_DECREF(ob)   (Py_DecRef((PyObject *)ob))
 #define Py_XINCREF(ob)  (Py_IncRef((PyObject *)ob))
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)ob))
+#else
+#define Py_INCREF(ob)   (((PyObject *)ob)->ob_refcnt++)
+#define Py_DECREF(ob)  ((((PyObject *)ob)->ob_refcnt > 1) ? \
+			((PyObject *)ob)->ob_refcnt-- : (Py_DecRef((PyObject *)ob)))
+
+#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
+#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
+#endif
 
 #define Py_CLEAR(op)				\
         do {                            	\

From noreply at buildbot.pypy.org  Mon Apr 23 23:59:28 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 23 Apr 2012 23:59:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Consider utf16 surrogates when encoding
 to raw_unicode_escape, 
Message-ID: <20120423215928.BB8B28203C@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54700:7fc6072593dd
Date: 2012-04-23 23:54 +0200
http://bitbucket.org/pypy/pypy/changeset/7fc6072593dd/

Log:	Consider utf16 surrogates when encoding to raw_unicode_escape, like
	the unicode_escape, but in both case this must be done only in
	narrow unicode build!

diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -1234,7 +1234,7 @@
             pos += 1
             continue
 
-        if 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
             # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
             pos += 1
             oc2 = ord(s[pos])
@@ -1350,6 +1350,20 @@
     pos = 0
     while pos < size:
         oc = ord(s[pos])
+
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+            # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
+            pos += 1
+            oc2 = ord(s[pos])
+
+            if 0xDC00 <= oc2 <= 0xDFFF:
+                ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000
+                raw_unicode_escape_helper(result, ucs)
+                pos += 1
+                continue
+            # Fall through: isolated surrogates are copied as-is
+            pos -= 1
+
         if oc < 0x100:
             result.append(chr(oc))
         else:
diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py
--- a/pypy/rlib/test/test_runicode.py
+++ b/pypy/rlib/test/test_runicode.py
@@ -728,3 +728,18 @@
 
         res = interpret(f, [0x10140])
         assert res == 0x10140
+
+    def test_encode_surrogate_pair(self):
+        u = runicode.UNICHR(0xD800) + runicode.UNICHR(0xDC00)
+        if runicode.MAXUNICODE < 65536:
+            # Narrow unicode build, consider utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+        else:
+            # Wide unicode build, don't merge utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'

From noreply at buildbot.pypy.org  Tue Apr 24 02:14:52 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 02:14:52 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: Write out the
	gcheader.
Message-ID: <20120424001452.0730C8203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54701:39913c0e6849
Date: 2012-04-23 20:14 -0400
http://bitbucket.org/pypy/pypy/changeset/39913c0e6849/

Log:	Write out the gcheader.

diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -483,7 +483,7 @@
 
 class ContainerNode(object):
     if USESLOTS:      # keep the number of slots down!
-        __slots__ = """db obj 
+        __slots__ = """db obj
                        typename implementationtypename
                         name
                         _funccodegen_owner
@@ -631,11 +631,17 @@
 
         if T._hints.get('untyped_storage'):
             arrayfld = T._arrayfld
+            c_expr = defnode.access_expr(self.name, "gcheader")
+            lines = generic_initializationexpr(self.db, gc_init, c_expr, decoration + "gcheader")
+            for line in lines:
+                yield '\t' + line
+
             c_expr = defnode.access_expr(self.name, 'shape')
             lines = generic_initializationexpr(self.db, self.obj.shape,
                                                c_expr, decoration + 'shape')
             for line in lines:
                 yield "\t" + line
+            # XXX: write out length of the array.
             arraydef = self.db.gettypedefnode(getattr(T, arrayfld))
             for i, obj in ll_enumerate_elements(self.obj):
                 c_expr = '(void*)' + arraydef.access_expr(
@@ -1069,7 +1075,7 @@
                 return 'PyExc_RuntimeError'
         raise Exception("don't know how to simply render py object: %r" %
                         (value, ))
-    
+
     def forward_declaration(self):
         return []
 

From noreply at buildbot.pypy.org  Tue Apr 24 02:20:00 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 02:20:00 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: working!
Message-ID: <20120424002000.4E4778203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54702:80f3aa6cedeb
Date: 2012-04-23 20:19 -0400
http://bitbucket.org/pypy/pypy/changeset/80f3aa6cedeb/

Log:	working!

diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -642,6 +642,7 @@
             for line in lines:
                 yield "\t" + line
             # XXX: write out length of the array.
+            yield '\t%s, ' % len(self.obj.data.items)
             arraydef = self.db.gettypedefnode(getattr(T, arrayfld))
             for i, obj in ll_enumerate_elements(self.obj):
                 c_expr = '(void*)' + arraydef.access_expr(

From noreply at buildbot.pypy.org  Tue Apr 24 02:37:00 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue, 24 Apr 2012 02:37:00 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: merge default into branch
Message-ID: <20120424003700.A3C678203C@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54703:4fbf169b5eb2
Date: 2012-04-23 14:35 -0700
http://bitbucket.org/pypy/pypy/changeset/4fbf169b5eb2/

Log:	merge default into branch

diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
--- a/_pytest/assertion/oldinterpret.py
+++ b/_pytest/assertion/oldinterpret.py
@@ -1,8 +1,7 @@
 import py
 import sys, inspect
 from compiler import parse, ast, pycodegen
-from _pytest.assertion.util import format_explanation
-from _pytest.assertion.reinterpret import BuiltinAssertionError
+from _pytest.assertion.util import format_explanation, BuiltinAssertionError
 
 passthroughex = py.builtin._sysex
 
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
--- a/_pytest/assertion/reinterpret.py
+++ b/_pytest/assertion/reinterpret.py
@@ -1,7 +1,6 @@
 import sys
 import py
-
-BuiltinAssertionError = py.builtin.builtins.AssertionError
+from _pytest.assertion.util import BuiltinAssertionError
 
 class AssertionError(BuiltinAssertionError):
     def __init__(self, *args):
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
--- a/_pytest/assertion/util.py
+++ b/_pytest/assertion/util.py
@@ -2,6 +2,7 @@
 
 import py
 
+BuiltinAssertionError = py.builtin.builtins.AssertionError
 
 # The _reprcompare attribute on the util module is used by the new assertion
 # interpretation code and assertion rewriter to detect this plugin was
diff --git a/lib_pypy/_ctypes/builtin.py b/lib_pypy/_ctypes/builtin.py
--- a/lib_pypy/_ctypes/builtin.py
+++ b/lib_pypy/_ctypes/builtin.py
@@ -3,7 +3,8 @@
 try:
     from thread import _local as local
 except ImportError:
-    local = object    # no threads
+    class local(object):    # no threads
+        pass
 
 class ConvMode:
     encoding = 'ascii'
diff --git a/lib_pypy/numpypy/core/fromnumeric.py b/lib_pypy/numpypy/core/fromnumeric.py
--- a/lib_pypy/numpypy/core/fromnumeric.py
+++ b/lib_pypy/numpypy/core/fromnumeric.py
@@ -411,7 +411,8 @@
             [3, 7]]])
 
     """
-    raise NotImplementedError('Waiting on interp level method')
+    swapaxes = a.swapaxes
+    return swapaxes(axis1, axis2)
 
 
 def transpose(a, axes=None):
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -229,8 +229,8 @@
                     return thing
                 elif hasattr(thing, '__name__'): # mostly types and functions
                     return thing.__name__
-                elif hasattr(thing, 'name'): # mostly ClassDescs
-                    return thing.name
+                elif hasattr(thing, 'name') and isinstance(thing.name, str):
+                    return thing.name            # mostly ClassDescs
                 elif isinstance(thing, tuple):
                     return '_'.join(map(nameof, thing))
                 else:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -320,10 +320,14 @@
                    default=False),
         BoolOption("getattributeshortcut",
                    "track types that override __getattribute__",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
         BoolOption("newshortcut",
                    "cache and shortcut calling __new__ from builtin types",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
 
         BoolOption("logspaceoptypes",
                    "a instrumentation option: before exit, print the types seen by "
@@ -337,7 +341,9 @@
                    requires=[("objspace.std.builtinshortcut", True)]),
         BoolOption("withidentitydict",
                    "track types that override __hash__, __eq__ or __cmp__ and use a special dict strategy for those which do not",
-                   default=False),
+                   default=False,
+                   # weakrefs needed, because of get_subclasses()
+                   requires=[("translation.rweakref", True)]),
      ]),
 ])
 
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/cppyy.rst
@@ -0,0 +1,333 @@
+============================
+cppyy: C++ bindings for PyPy
+============================
+
+The cppyy module provides C++ bindings for PyPy by using the reflection
+information extracted from C++ header files by means of the
+`Reflex package`_.
+For this to work, you have to both install Reflex and build PyPy from the
+reflex-support branch.
+As indicated by this being a branch, support for Reflex is still
+experimental.
+However, it is functional enough to put it in the hands of those who want
+to give it a try.
+In the medium term, cppyy will move away from Reflex and instead use
+`cling`_ as its backend, which is based on `llvm`_.
+Although that will change the logistics on the generation of reflection
+information, it will not change the python-side interface.
+
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`cling`: http://root.cern.ch/drupal/content/cling
+.. _`llvm`: http://llvm.org/
+
+
+Installation
+============
+
+For now, the easiest way of getting the latest version of Reflex, is by
+installing the ROOT package.
+Besides getting the latest version of Reflex, another advantage is that with
+the full ROOT package, you can also use your Reflex-bound code on `CPython`_.
+`Download`_ a binary or install from `source`_.
+Some Linux and Mac systems may have ROOT provided in the list of scientific
+software of their packager.
+A current, standalone version of Reflex should be provided at some point,
+once the dependencies and general packaging have been thought out.
+Also, make sure you have a version of `gccxml`_ installed, which is most
+easily provided by the packager of your system.
+If you read up on gccxml, you'll probably notice that it is no longer being
+developed and hence will not provide C++11 support.
+That's why the medium term plan is to move to `cling`_.
+
+.. _`Download`: http://root.cern.ch/drupal/content/downloading-root
+.. _`source`: http://root.cern.ch/drupal/content/installing-root-source
+.. _`gccxml`: http://www.gccxml.org
+
+Next, get the `PyPy sources`_, select the reflex-support branch, and build
+pypy-c.
+For the build to succeed, the ``$ROOTSYS`` environment variable must point to
+the location of your ROOT installation::
+
+    $ hg clone https://bitbucket.org/pypy/pypy
+    $ cd pypy
+    $ hg up reflex-support
+    $ cd pypy/translator/goal
+    $ python translate.py -O jit --gcrootfinder=shadowstack targetpypystandalone.py --withmod-cppyy
+
+This will build a ``pypy-c`` that includes the cppyy module, and through that,
+Reflex support.
+Of course, if you already have a pre-built version of the ``pypy`` interpreter,
+you can use that for the translation rather than ``python``.
+
+.. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
+
+
+Basic example
+=============
+
+Now test with a trivial example whether all packages are properly installed
+and functional.
+First, create a C++ header file with some class in it (note that all functions
+are made inline for convenience; a real-world example would of course have a
+corresponding source file)::
+
+    $ cat MyClass.h
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_myint(i) {}
+
+        int GetMyInt() { return m_myint; }
+        void SetMyInt(int i) { m_myint = i; }
+
+    public:
+       int m_myint;
+    };
+
+Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
+code::
+
+    $ genreflex MyClass.h
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+
+Now you're ready to use the bindings.
+Since the bindings are designed to look pythonistic, it should be
+straightforward::
+
+    $ pypy-c
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libMyClassDict.so")
+    <CPPLibrary object at 0xb6fd7c4c>
+    >>>> myinst = cppyy.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> myinst.SetMyInt(33)
+    >>>> print myinst.m_myint
+    33
+    >>>> myinst.m_myint = 77
+    >>>> print myinst.GetMyInt()
+    77
+    >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
+
+That's all there is to it!
+
+
+Features
+========
+
+The following is not meant to be an exhaustive list, since cppyy is still
+under active development.
+Furthermore, the intention is that every feature is as natural as possible on
+the python side, so if you find something missing in the list below, simply
+try it out.
+It is not always possible to provide exact mapping between python and C++
+(active memory management is one such case), but by and large, if the use of a
+feature does not strike you as obvious, it is more likely to simply be a bug.
+That is a strong statement to make, but also a worthy goal.
+
+* **abstract classes**: Are represented as python classes, since they are
+  needed to complete the inheritance hierarchies, but will raise an exception
+  if an attempt is made to instantiate from them.
+
+* **arrays**: Supported for builtin data types only, as used from module
+  ``array``.
+  Out-of-bounds checking is limited to those cases where the size is known at
+  compile time (and hence part of the reflection info).
+
+* **builtin data types**: Map onto the expected equivalent python types, with
+  the caveat that there may be size differences, and thus it is possible that
+  exceptions are raised if an overflow is detected.
+
+* **casting**: Is supposed to be unnecessary.
+  Object pointer returns from functions provide the most derived class known
+  in the hierarchy of the object being returned.
+  This is important to preserve object identity as well as to make casting,
+  a pure C++ feature after all, superfluous.
+
+* **classes and structs**: Get mapped onto python classes, where they can be
+  instantiated as expected.
+  If classes are inner classes or live in a namespace, their naming and
+  location will reflect that.
+
+* **data members**: Public data members are represented as python properties
+  and provide read and write access on instances as expected.
+
+* **default arguments**: C++ default arguments work as expected, but python
+  keywords are not supported.
+  It is technically possible to support keywords, but for the C++ interface,
+  the formal argument names have no meaning and are not considered part of the
+  API, hence it is not a good idea to use keywords.
+
+* **doc strings**: The doc string of a method or function contains the C++
+  arguments and return types of all overloads of that name, as applicable.
+
+* **functions**: Work as expected and live in their appropriate namespace
+  (which can be the global one, ``cppyy.gbl``).
+
+* **inheritance**: All combinations of inheritance on the C++ (single,
+  multiple, virtual) are supported in the binding.
+  However, new python classes can only use single inheritance from a bound C++
+  class.
+  Multiple inheritance would introduce two "this" pointers in the binding.
+  This is a current, not a fundamental, limitation.
+  The C++ side will not see any overridden methods on the python side, as
+  cross-inheritance is planned but not yet supported.
+
+* **methods**: Are represented as python methods and work as expected.
+  They are first class objects and can be bound to an instance.
+  Virtual C++ methods work as expected.
+  To select a specific virtual method, do like with normal python classes
+  that override methods: select it from the class that you need, rather than
+  calling the method on the instance.
+
+* **namespaces**: Are represented as python classes.
+  Namespaces are more open-ended than classes, so sometimes initial access may
+  result in updates as data and functions are looked up and constructed
+  lazily.
+  Thus the result of ``dir()`` on a namespace should not be relied upon: it
+  only shows the already accessed members. (TODO: to be fixed by implementing
+  __dir__.)
+  The global namespace is ``cppyy.gbl``.
+
+* **operator conversions**: If defined in the C++ class and a python
+  equivalent exists (i.e. all builtin integer and floating point types, as well
+  as ``bool``), it will map onto that python conversion.
+  Note that ``char*`` is mapped onto ``__str__``.
+
+* **operator overloads**: If defined in the C++ class and if a python
+  equivalent is available (not always the case, think e.g. of ``operator||``),
+  then they work as expected.
+  Special care needs to be taken for global operator overloads in C++: first,
+  make sure that they are actually reflected, especially for the global
+  overloads for ``operator==`` and ``operator!=`` of STL iterators in the case
+  of gcc.
+  Second, make sure that reflection info is loaded in the proper order.
+  I.e. that these global overloads are available before use.
+
+* **pointers**: For builtin data types, see arrays.
+  For objects, a pointer to an object and an object looks the same, unless
+  the pointer is a data member.
+  In that case, assigning to the data member will cause a copy of the pointer
+  and care should be taken about the object's life time.
+  If a pointer is a global variable, the C++ side can replace the underlying
+  object and the python side will immediately reflect that.
+
+* **static data members**: Are represented as python property objects on the
+  class and the meta-class.
+  Both read and write access is as expected.
+
+* **static methods**: Are represented as python's ``staticmethod`` objects
+  and can be called both from the class as well as from instances.
+
+* **strings**: The std::string class is considered a builtin C++ type and
+  mixes quite well with python's str.
+  Python's str can be passed where a ``const char*`` is expected, and an str
+  will be returned if the return type is ``const char*``.
+
+* **templated classes**: Are represented in a meta-class style in python.
+  This looks a little bit confusing, but conceptually is rather natural.
+  For example, given the class ``std::vector<int>``, the meta-class part would
+  be ``std.vector`` in python.
+  Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
+  create an instance of that class, do ``std.vector(int)()``.
+  Note that templates can be build up by handing actual types to the class
+  instantiation (as done in this vector example), or by passing in the list of
+  template arguments as a string.
+  The former is a lot easier to work with if you have template instantiations
+  using classes that themselves are templates (etc.) in the arguments.
+  All classes must already exist in the loaded reflection info.
+
+* **unary operators**: Are supported if a python equivalent exists, and if the
+  operator is defined in the C++ class.
+
+You can always find more detailed examples and see the full of supported
+features by looking at the tests in pypy/module/cppyy/test.
+
+If a feature or reflection info is missing, this is supposed to be handled
+gracefully.
+In fact, there are unit tests explicitly for this purpose (even as their use
+becomes less interesting over time, as the number of missing features
+decreases).
+Only when a missing feature is used, should there be an exception.
+For example, if no reflection info is available for a return type, then a
+class that has a method with that return type can still be used.
+Only that one specific method can not be used.
+
+
+The fast lane
+=============
+
+The following is an experimental feature of cppyy, and that makes it doubly
+experimental, so caveat emptor.
+With a slight modification of Reflex, it can provide function pointers for
+C++ methods, and hence allow PyPy to call those pointers directly, rather than
+calling C++ through a Reflex stub.
+This results in a rather significant speed-up.
+Mind you, the normal stub path is not exactly slow, so for now only use this
+out of curiosity or if you really need it.
+
+To install this patch of Reflex, locate the file genreflex-methptrgetter.patch
+in pypy/module/cppyy and apply it to the genreflex python scripts found in
+``$ROOTSYS/lib``::
+
+    $ cd $ROOTSYS/lib
+    $ patch -p2 < genreflex-methptrgetter.patch
+
+With this patch, ``genreflex`` will have grown the ``--with-methptrgetter``
+option.
+Use this option when running ``genreflex``, and add the
+``-Wno-pmf-conversions`` option to ``g++`` when compiling.
+The rest works the same way: the fast path will be used transparently (which
+also means that you can't actually find out whether it is in use, other than
+by running a micro-benchmark).
+
+
+CPython
+=======
+
+Most of the ideas in cppyy come originally from the `PyROOT`_ project.
+Although PyROOT does not support Reflex directly, it has an alter ego called
+"PyCintex" that, in a somewhat roundabout way, does.
+If you installed ROOT, rather than just Reflex, PyCintex should be available
+immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment
+variable.
+
+.. _`PyROOT`: http://root.cern.ch/drupal/content/pyroot
+
+There are a couple of minor differences between PyCintex and cppyy, most to do
+with naming.
+The one that you will run into directly, is that PyCintex uses a function
+called ``loadDictionary`` rather than ``load_reflection_info``.
+The reason for this is that Reflex calls the shared libraries that contain
+reflection info "dictionaries."
+However, in python, the name `dictionary` already has a well-defined meaning,
+so a more descriptive name was chosen for cppyy.
+In addition, PyCintex requires that the names of shared libraries so loaded
+start with "lib" in their name.
+The basic example above, rewritten for PyCintex thus goes like this::
+
+    $ python
+    >>> import PyCintex
+    >>> PyCintex.loadDictionary("libMyClassDict.so")
+    >>> myinst = PyCintex.gbl.MyClass(42)
+    >>> print myinst.GetMyInt()
+    42
+    >>> myinst.SetMyInt(33)
+    >>> print myinst.m_myint
+    33
+    >>> myinst.m_myint = 77
+    >>> print myinst.GetMyInt()
+    77
+    >>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
+
+Other naming differences are such things as taking an address of an object.
+In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
+made to follow the naming as in ``ctypes`` and hence use ``addressof``
+(PyROOT/PyCintex predate ``ctypes`` by several years, and the ROOT project
+follows camel-case, hence the differences).
+
+Of course, this is python, so if any of the naming is not to your liking, all
+you have to do is provide a wrapper script that you import instead of
+importing the ``cppyy`` or ``PyCintex`` modules directly.
+In that wrapper script you can rename methods exactly the way you need it.
+
+In the cling world, all these differences will be resolved.
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -23,6 +23,8 @@
 
 * Write them in RPython as mixedmodule_, using *rffi* as bindings.
 
+* Write them in C++ and bind them through Reflex_ (EXPERIMENTAL)
+
 .. _ctypes: #CTypes
 .. _\_ffi: #LibFFI
 .. _mixedmodule: #Mixed Modules
@@ -110,3 +112,34 @@
 
 XXX we should provide detailed docs about lltype and rffi, especially if we
     want people to follow that way.
+
+Reflex
+======
+
+This method is only experimental for now, and is being exercised on a branch,
+`reflex-support`_, so you will have to build PyPy yourself.
+The method works by using the `Reflex package`_ to provide reflection
+information of the C++ code, which is then used to automatically generate
+bindings at runtime, which can then be used from python.
+Full details are `available here`_.
+
+.. _`reflex-support`: cppyy.html
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`available here`: cppyy.html
+
+Pros
+----
+
+If it works, it is mostly automatic, and hence easy in use.
+The bindings can make use of direct pointers, in which case the calls are
+very fast.
+
+Cons
+----
+
+C++ is a large language, and these bindings are not yet feature-complete.
+Although missing features should do no harm if you don't use them, if you do
+need a particular feature, it may be necessary to work around it in python
+or with a C++ helper function.
+Although Reflex works on various platforms, the bindings with PyPy have only
+been tested on Linux.
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -85,6 +85,10 @@
     Collects the arguments of a function call.
 
     Instances should be considered immutable.
+
+    Some parts of this class are written in a slightly convoluted style to help
+    the JIT. It is really crucial to get this right, because Python's argument
+    semantics are complex, but calls occur everywhere.
     """
 
     ###  Construction  ###
@@ -171,7 +175,13 @@
         space = self.space
         keywords, values_w = space.view_as_kwargs(w_starstararg)
         if keywords is not None: # this path also taken for empty dicts
-            self._add_keywordargs_no_unwrapping(keywords, values_w)
+            if self.keywords is None:
+                self.keywords = keywords[:] # copy to make non-resizable
+                self.keywords_w = values_w[:]
+            else:
+                self._check_not_duplicate_kwargs(keywords, values_w)
+                self.keywords = self.keywords + keywords
+                self.keywords_w = self.keywords_w + values_w
             return not jit.isconstant(len(self.keywords))
         if space.isinstance_w(w_starstararg, space.w_dict):
             keys_w = space.unpackiterable(w_starstararg)
@@ -229,22 +239,16 @@
     @jit.look_inside_iff(lambda self, keywords, keywords_w:
             jit.isconstant(len(keywords) and
             jit.isconstant(self.keywords)))
-    def _add_keywordargs_no_unwrapping(self, keywords, keywords_w):
-        if self.keywords is None:
-            self.keywords = keywords[:] # copy to make non-resizable
-            self.keywords_w = keywords_w[:]
-        else:
-            # looks quadratic, but the JIT should remove all of it nicely.
-            # Also, all the lists should be small
-            for key in keywords:
-                for otherkey in self.keywords:
-                    if otherkey == key:
-                        raise operationerrfmt(self.space.w_TypeError,
-                                              "got multiple values "
-                                              "for keyword argument "
-                                              "'%s'", key)
-            self.keywords = self.keywords + keywords
-            self.keywords_w = self.keywords_w + keywords_w
+    def _check_not_duplicate_kwargs(self, keywords, keywords_w):
+        # looks quadratic, but the JIT should remove all of it nicely.
+        # Also, all the lists should be small
+        for key in keywords:
+            for otherkey in self.keywords:
+                if otherkey == key:
+                    raise operationerrfmt(self.space.w_TypeError,
+                                          "got multiple values "
+                                          "for keyword argument "
+                                          "'%s'", key)
 
     def fixedunpack(self, argcount):
         """The simplest argument parsing: get the 'argcount' arguments,
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -49,7 +49,9 @@
     def __repr__(self):
         # return "function %s.%s" % (self.space, self.name)
         # maybe we want this shorter:
-        name = getattr(self, 'name', '?')
+        name = getattr(self, 'name', None)
+        if not isinstance(name, str):
+            name = '?'
         return "<%s %s>" % (self.__class__.__name__, name)
 
     def call_args(self, args):
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1656,15 +1656,21 @@
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
-            #   - read the typeid from mem(locs[0]), i.e. at offset 0
-            #   - keep the lower 16 bits read there
-            #   - multiply by 4 and use it as an offset in type_info_group
-            #   - add 16 bytes, to go past the TYPE_INFO structure
+            #   - read the typeid from mem(locs[0]), i.e. at offset 0;
+            #     this is a complete word (N=4 bytes on 32-bit, N=8 on
+            #     64-bits)
+            #   - keep the lower half of what is read there (i.e.
+            #     truncate to an unsigned 'N / 2' bytes value)
+            #   - multiply by 4 (on 32-bits only) and use it as an
+            #     offset in type_info_group
+            #   - add 16/32 bytes, to go past the TYPE_INFO structure
             loc = locs[1]
             assert isinstance(loc, ImmedLoc)
             classptr = loc.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the 16 bits in the object header
+            # from reading the half-word in the object header.  Note that
+            # this half-word is at offset 0 on a little-endian machine;
+            # it would be at offset 2 or 4 on a big-endian machine.
             from pypy.rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1223,7 +1223,7 @@
     def run_one_step(self):
         # Execute the frame forward.  This method contains a loop that leaves
         # whenever the 'opcode_implementations' (which is one of the 'opimpl_'
-        # methods) returns True.  This is the case when the current frame
+        # methods) raises ChangeFrame.  This is the case when the current frame
         # changes, due to a call or a return.
         try:
             staticdata = self.metainterp.staticdata
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -341,9 +341,13 @@
 
     def add(self, w_iobase):
         assert w_iobase.streamholder is None
-        holder = StreamHolder(w_iobase)
-        w_iobase.streamholder = holder
-        self.streams[holder] = None
+        if rweakref.has_weakref_support():
+            holder = StreamHolder(w_iobase)
+            w_iobase.streamholder = holder
+            self.streams[holder] = None
+        #else:
+        #   no support for weakrefs, so ignore and we
+        #   will not get autoflushing
 
     def remove(self, w_iobase):
         holder = w_iobase.streamholder
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -185,6 +185,7 @@
             if subentry is not None:
                 subentry._stop(tt, it)
 
+ at jit.elidable_promote()
 def create_spec(space, w_arg):
     if isinstance(w_arg, Method):
         w_function = w_arg.w_function
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -22,7 +22,7 @@
     cannot be iterated."""
     return space.iter(w_obj)
 
- at cpython_api([PyObject], PyObject, error=CANNOT_FAIL)
+ at cpython_api([PyObject], PyObject)
 def PyIter_Next(space, w_obj):
     """Return the next value from the iteration o.  If the object is an
     iterator, this retrieves the next value from the iteration, and returns
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -314,7 +314,10 @@
     """This function simulates the effect of a SIGINT signal arriving --- the
     next time PyErr_CheckSignals() is called, KeyboardInterrupt will be raised.
     It may be called without holding the interpreter lock."""
-    space.check_signal_action.set_interrupt()
+    if space.check_signal_action is not None:
+        space.check_signal_action.set_interrupt()
+    #else:
+    #   no 'signal' module present, ignore...  We can't return an error here
 
 @cpython_api([PyObjectP, PyObjectP, PyObjectP], lltype.Void)
 def PyErr_GetExcInfo(space, ptype, pvalue, ptraceback):
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -15,3 +15,8 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
         assert api.PyIter_Next(w_iter) is None
         assert not api.PyErr_Occurred()
+
+    def test_iternext_error(self,space, api):
+        assert api.PyIter_Next(space.w_None) is None
+        assert api.PyErr_Occurred() is space.w_TypeError
+        api.PyErr_Clear()
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -453,10 +453,22 @@
 
     def test_tailmatch(self, space, api):
         w_str = space.wrap(u"abcdef")
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 10, 1) == 1
-        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5, -1) == 1
+        # prefix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 9, -1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 2, 4, -1) == 0 # ends at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6, -1) == 0 # starts at 'b'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cdf"), 2, 6, -1) == 0
+        # suffix match
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 5,  1) == 1
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 3, 5,  1) == 0 # starts at 'd'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("cde"), 1, 6,  1) == 0 # ends at 'f'
+        assert api.PyUnicode_Tailmatch(w_str, space.wrap("bde"), 1, 5,  1) == 0
+        # type checks
         self.raises(space, api, TypeError,
                     api.PyUnicode_Tailmatch, w_str, space.wrap(3), 2, 10, 1)
+        self.raises(space, api, TypeError,
+                    api.PyUnicode_Tailmatch, space.wrap(3), space.wrap("abc"),
+                    2, 10, 1)
 
     def test_count(self, space, api):
         w_str = space.wrap(u"abcabdab")
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -593,7 +593,7 @@
     suffix match), 0 otherwise. Return -1 if an error occurred."""
     str = space.unicode_w(w_str)
     substr = space.unicode_w(w_substr)
-    if rffi.cast(lltype.Signed, direction) >= 0:
+    if rffi.cast(lltype.Signed, direction) <= 0:
         return stringtype.stringstartswith(str, substr, start, end)
     else:
         return stringtype.stringendswith(str, substr, start, end)
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
@@ -30,6 +31,7 @@
         'isna': 'interp_numarray.isna',
         'concatenate': 'interp_numarray.concatenate',
         'repeat': 'interp_numarray.repeat',
+        'where': 'interp_arrayops.where',
 
         'set_string_function': 'appbridge.set_string_function',
 
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -0,0 +1,90 @@
+
+from pypy.module.micronumpy.interp_numarray import convert_to_array,\
+     VirtualArray
+from pypy.module.micronumpy import signature
+
+class WhereArray(VirtualArray):
+    def __init__(self, space, arr, x, y):
+        self.arr = arr
+        self.x = x
+        self.y = y
+        VirtualArray.__init__(self, 'where', arr.shape[:],
+                              x.find_dtype())
+
+    def create_sig(self):
+        if self.forced_result is not None:
+            return self.forced_result.create_sig()
+        return signature.WhereSignature(self.res_dtype, self.arr.find_dtype(),
+                                        self.arr.create_sig(),
+                                        self.x.create_sig(),
+                                        self.y.create_sig())
+
+    def _del_sources(self):
+        self.arr = None
+        self.x = None
+        self.y = None
+
+def where(space, w_arr, w_x, w_y):
+    """where(condition, [x, y])
+
+    Return elements, either from `x` or `y`, depending on `condition`.
+
+    If only `condition` is given, return ``condition.nonzero()``.
+
+    Parameters
+    ----------
+    condition : array_like, bool
+        When True, yield `x`, otherwise yield `y`.
+    x, y : array_like, optional
+        Values from which to choose. `x` and `y` need to have the same
+        shape as `condition`.
+
+    Returns
+    -------
+    out : ndarray or tuple of ndarrays
+        If both `x` and `y` are specified, the output array contains
+        elements of `x` where `condition` is True, and elements from
+        `y` elsewhere.
+
+        If only `condition` is given, return the tuple
+        ``condition.nonzero()``, the indices where `condition` is True.
+
+    See Also
+    --------
+    nonzero, choose
+
+    Notes
+    -----
+    If `x` and `y` are given and input arrays are 1-D, `where` is
+    equivalent to::
+
+        [xv if c else yv for (c,xv,yv) in zip(condition,x,y)]
+
+    Examples
+    --------
+    >>> np.where([[True, False], [True, True]],
+    ...          [[1, 2], [3, 4]],
+    ...          [[9, 8], [7, 6]])
+    array([[1, 8],
+           [3, 4]])
+
+    >>> np.where([[0, 1], [1, 0]])
+    (array([0, 1]), array([1, 0]))
+
+    >>> x = np.arange(9.).reshape(3, 3)
+    >>> np.where( x > 5 )
+    (array([2, 2, 2]), array([0, 1, 2]))
+    >>> x[np.where( x > 3.0 )]               # Note: result is 1D.
+    array([ 4.,  5.,  6.,  7.,  8.])
+    >>> np.where(x < 5, x, -1)               # Note: broadcasting.
+    array([[ 0.,  1.,  2.],
+           [ 3.,  4., -1.],
+           [-1., -1., -1.]])
+
+    
+    NOTE: support for not passing x and y is unsupported
+    """
+    arr = convert_to_array(space, w_arr)
+    x = convert_to_array(space, w_x)
+    y = convert_to_array(space, w_y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -513,7 +514,30 @@
             arr = concrete.copy(space)
             arr.setshape(space, new_shape)
         return arr
-
+       
+    @unwrap_spec(axis1=int, axis2=int)
+    def descr_swapaxes(self, space, axis1, axis2):
+        """a.swapaxes(axis1, axis2)
+    
+        Return a view of the array with `axis1` and `axis2` interchanged.
+    
+        Refer to `numpy.swapaxes` for full documentation.
+    
+        See Also
+        --------
+        numpy.swapaxes : equivalent function
+        """
+        concrete = self.get_concrete()
+        shape = concrete.shape[:]
+        strides = concrete.strides[:]
+        backstrides = concrete.backstrides[:]
+        shape[axis1], shape[axis2] = shape[axis2], shape[axis1]   
+        strides[axis1], strides[axis2] = strides[axis2], strides[axis1]
+        backstrides[axis1], backstrides[axis2] = backstrides[axis2], backstrides[axis1] 
+        arr = W_NDimSlice(concrete.start, strides, 
+                           backstrides, shape, concrete)
+        return space.wrap(arr)   
+                                      
     def descr_tolist(self, space):
         if len(self.shape) == 0:
             assert isinstance(self, Scalar)
@@ -1412,6 +1436,7 @@
     copy = interp2app(BaseArray.descr_copy),
     flatten = interp2app(BaseArray.descr_flatten),
     reshape = interp2app(BaseArray.descr_reshape),
+    swapaxes = interp2app(BaseArray.descr_swapaxes),
     tolist = interp2app(BaseArray.descr_tolist),
     take = interp2app(BaseArray.descr_take),
     compress = interp2app(BaseArray.descr_compress),
@@ -1559,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/signature.py b/pypy/module/micronumpy/signature.py
--- a/pypy/module/micronumpy/signature.py
+++ b/pypy/module/micronumpy/signature.py
@@ -498,3 +498,63 @@
         arr.left.setitem(iterator.offset, value)
     def debug_repr(self):
         return 'AxisReduceSig(%s, %s)' % (self.name, self.right.debug_repr())
+
+class WhereSignature(Signature):
+    _immutable_fields_ = ['dtype', 'arrdtype', 'arrsig', 'xsig', 'ysig']
+    
+    def __init__(self, dtype, arrdtype, arrsig, xsig, ysig):
+        self.dtype = dtype
+        self.arrdtype = arrdtype
+        self.arrsig = arrsig
+        self.xsig = xsig
+        self.ysig = ysig
+
+    def hash(self):
+        return (intmask(self.arrsig.hash() << 1) ^
+                intmask(self.xsig.hash() << 2) ^
+                intmask(self.ysig.hash() << 3))
+
+    def eq(self, other, compare_array_no=True):
+        if type(self) is not type(other):
+            return False
+        assert isinstance(other, WhereSignature)
+        return (self.arrsig.eq(other.arrsig, compare_array_no) and
+                self.xsig.eq(other.xsig, compare_array_no) and
+                self.ysig.eq(other.ysig, compare_array_no))
+
+    def _invent_array_numbering(self, arr, cache):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        self.arrsig._invent_array_numbering(arr.arr, cache)
+        self.xsig._invent_array_numbering(arr.x, cache)
+        self.ysig._invent_array_numbering(arr.y, cache)
+
+    def _invent_numbering(self, cache, allnumbers):
+        self.arrsig._invent_numbering(cache, allnumbers)
+        self.xsig._invent_numbering(cache, allnumbers)
+        self.ysig._invent_numbering(cache, allnumbers)
+
+    def _create_iter(self, iterlist, arraylist, arr, transforms):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+
+        assert isinstance(arr, WhereArray)
+        # XXX this does not support broadcasting correctly
+        self.arrsig._create_iter(iterlist, arraylist, arr.arr, transforms)
+        self.xsig._create_iter(iterlist, arraylist, arr.x, transforms)
+        self.ysig._create_iter(iterlist, arraylist, arr.y, transforms)
+ 
+    def eval(self, frame, arr):
+        from pypy.module.micronumpy.interp_arrayops import WhereArray
+        assert isinstance(arr, WhereArray)
+        lhs = self.xsig.eval(frame, arr.x).convert_to(self.dtype)
+        rhs = self.ysig.eval(frame, arr.y).convert_to(self.dtype)
+        w_val = self.arrsig.eval(frame, arr.arr)
+        if self.arrdtype.itemtype.bool(w_val):
+            return lhs
+        else:
+            return rhs
+
+    def debug_repr(self):
+        return 'Where(%s, %s, %s)' % (self.arrsig.debug_repr(),
+                                      self.xsig.debug_repr(),
+                                      self.ysig.debug_repr())
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -0,0 +1,16 @@
+
+from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumSupport(BaseNumpyAppTest):
+    def test_where(self):
+        from _numpypy import where, ones, zeros, array
+        a = [1, 2, 3, 0, -3]
+        a = where(array(a) > 0, ones(5), zeros(5))
+        assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1,9 +1,8 @@
 
 import py
 
-from pypy.conftest import gettestobjspace, option
+from pypy.conftest import option
 from pypy.interpreter.error import OperationError
-from pypy.module.micronumpy import signature
 from pypy.module.micronumpy.appbridge import get_appbridge_cache
 from pypy.module.micronumpy.interp_iter import Chunk, Chunks
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
@@ -1410,6 +1409,35 @@
         assert (array([1, 2]).repeat(2) == array([1, 1, 2, 2])).all()
 
 
+    def test_swapaxes(self):
+        from _numpypy import array
+        # testcases from numpy docstring
+        x = array([[1, 2, 3]])
+        assert (x.swapaxes(0, 1) == array([[1], [2], [3]])).all() 
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]]) # shape = (2, 2, 2)
+        assert (x.swapaxes(0, 2) == array([[[0, 4], [2, 6]], 
+                                           [[1, 5], [3, 7]]])).all() 
+        assert (x.swapaxes(0, 1) == array([[[0, 1], [4, 5]], 
+                                           [[2, 3], [6, 7]]])).all()
+        assert (x.swapaxes(1, 2) == array([[[0, 2], [1, 3]], 
+                                           [[4, 6],[5, 7]]])).all()
+
+        # more complex shape i.e. (2, 2, 3)
+        x = array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]) 
+        assert (x.swapaxes(0, 1) == array([[[1, 2, 3], [7, 8, 9]], 
+                                           [[4, 5, 6], [10, 11, 12]]])).all() 
+        assert (x.swapaxes(0, 2) == array([[[1, 7], [4, 10]], [[2, 8], [5, 11]], 
+                                           [[3, 9], [6, 12]]])).all() 
+        assert (x.swapaxes(1, 2) == array([[[1, 4], [2, 5], [3, 6]], 
+                                           [[7, 10], [8, 11],[9, 12]]])).all() 
+        
+        # test slice
+        assert (x[0:1,0:2].swapaxes(0,2) == array([[[1], [4]], [[2], [5]], 
+                                                   [[3], [6]]])).all()
+        # test virtual
+        assert ((x + x).swapaxes(0,1) == array([[[ 2,  4,  6], [14, 16, 18]], 
+                                         [[ 8, 10, 12], [20, 22, 24]]])).all()
+                        
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import _numpypy
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -762,6 +762,8 @@
 
     def test_logaddexp(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp
 
         # From the numpy documentation
@@ -772,7 +774,8 @@
 
         assert logaddexp(0, 0) == math.log(2)
         assert logaddexp(float('-inf'), 0) == 0
-        assert logaddexp(12345678, 12345678) == float('inf')
+        assert logaddexp(float_max, float_max) == float_max
+        assert logaddexp(float_min, float_min) == math.log(2)
 
         assert math.isnan(logaddexp(float('nan'), 1))
         assert math.isnan(logaddexp(1, float('nan')))
@@ -785,6 +788,8 @@
 
     def test_logaddexp2(self):
         import math
+        import sys
+        float_max, float_min = sys.float_info.max, sys.float_info.min
         from _numpypy import logaddexp2
         log2 = math.log(2)
 
@@ -796,7 +801,8 @@
 
         assert logaddexp2(0, 0) == 1
         assert logaddexp2(float('-inf'), 0) == 0
-        assert logaddexp2(12345678, 12345678) == float('inf')
+        assert logaddexp2(float_max, float_max) == float_max
+        assert logaddexp2(float_min, float_min) == 1.0
 
         assert math.isnan(logaddexp2(float('nan'), 1))
         assert math.isnan(logaddexp2(1, float('nan')))
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -17,6 +17,7 @@
                                                 'render_as_void': True})
 degToRad = math.pi / 180.0
 log2 = math.log(2)
+log2e = 1./log2
 
 def simple_unary_op(func):
     specialize.argtype(1)(func)
@@ -841,45 +842,26 @@
 
     @simple_binary_op
     def logaddexp(self, v1, v2):
-        try:
-            v1e = math.exp(v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.exp(v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + rfloat.log1p(math.exp(-tmp))
+        elif tmp <= 0:
+            return v2 + rfloat.log1p(math.exp(tmp))
+        else:
+            return v1 + v2
 
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e)
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+    def npy_log2_1p(self, v):
+        return log2e * rfloat.log1p(v)
 
     @simple_binary_op
     def logaddexp2(self, v1, v2):
-        try:
-            v1e = math.pow(2, v1)
-        except OverflowError:
-            v1e = rfloat.INFINITY
-        try:
-            v2e = math.pow(2, v2)
-        except OverflowError:
-            v2e = rfloat.INFINITY
-
-        v12e = v1e + v2e
-        try:
-            return math.log(v12e) / log2
-        except ValueError:
-            if v12e == 0.0:
-                # CPython raises ValueError here, so we have to check
-                # the value to find the correct numpy return value
-                return -rfloat.INFINITY
-            return rfloat.NAN
+        tmp = v1 - v2
+        if tmp > 0:
+            return v1 + self.npy_log2_1p(math.pow(2, -tmp))
+        if tmp <= 0:
+            return v2 + self.npy_log2_1p(math.pow(2, tmp))
+        else:
+            return v1 + v2
 
 class NonNativeFloat(NonNativePrimitive, Float):
     _mixin_ = True
diff --git a/pypy/module/pypyjit/test_pypy_c/test_exception.py b/pypy/module/pypyjit/test_pypy_c/test_exception.py
--- a/pypy/module/pypyjit/test_pypy_c/test_exception.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_exception.py
@@ -91,3 +91,29 @@
             --TICK--
             jump(..., descr=...)
         """)
+
+    def test_continue_in_finally(self):
+        # check that 'continue' inside a try:finally: block is correctly
+        # detected as closing a loop
+        py.test.skip("is this case important?")
+        def f(n):
+            i = 0
+            while 1:
+                try:
+                    if i < n:
+                        continue
+                finally:
+                    i += 1
+                return i
+
+        log = self.run(f, [2000])
+        assert log.result == 2001
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i40 = int_add_ovf(i31, 1)
+            guard_no_overflow(descr=...)
+            i41 = int_lt(i40, i33)
+            guard_true(i41, descr=...)
+            --TICK--
+            jump(..., descr=...)
+        """)
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,10 +24,9 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
-            #include <windows.h>
-
             static HANDLE interrupt_event;
 
             static BOOL WINAPI CtrlHandlerRoutine(
diff --git a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
--- a/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
+++ b/pypy/module/test_lib_pypy/numpypy/core/test_fromnumeric.py
@@ -136,4 +136,11 @@
         raises(NotImplementedError, "transpose(x, axes=(1, 0, 2))")
         # x = ones((1, 2, 3))
         # assert transpose(x, (1, 0, 2)).shape == (2, 1, 3)
-        
+    
+    def test_fromnumeric(self):
+        from numpypy import array, swapaxes
+        x = array([[1,2,3]])
+        assert (swapaxes(x,0,1) == array([[1], [2], [3]])).all()
+        x = array([[[0,1],[2,3]],[[4,5],[6,7]]])
+        assert (swapaxes(x,0,2) == array([[[0, 4], [2, 6]], 
+                                          [[1, 5], [3, 7]]])).all()
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -229,7 +229,11 @@
         startpos = fullname.rfind('.') + 1 # 0 when not found
         assert startpos >= 0
         subname = fullname[startpos:]
-        return self.prefix + subname.replace('.', '/')
+        if ZIPSEP == os.path.sep:
+            return self.prefix + subname.replace('.', '/')
+        else:
+            return self.prefix.replace(os.path.sep, ZIPSEP) + \
+                    subname.replace('.', '/')
 
     def make_co_filename(self, filename):
         """
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -313,13 +313,11 @@
         assert z.get_filename("package") == mod.__file__
 
     def test_subdirectory_twice(self):
-        import os, zipimport
+        #import os, zipimport
  
         self.writefile("package/__init__.py", "")
         self.writefile("package/subpackage/__init__.py", "")
         self.writefile("package/subpackage/foo.py", "")
-        import sys
-        print sys.path
         mod = __import__('package.subpackage.foo', None, None, [])
         assert mod
 
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -414,7 +414,8 @@
     def contains(space, w_container, w_item):
         w_descr = space.lookup(w_container, '__contains__')
         if w_descr is not None:
-            return space.get_and_call_function(w_descr, w_container, w_item)
+            w_result = space.get_and_call_function(w_descr, w_container, w_item)
+            return space.nonzero(w_result)
         return space._contains(w_container, w_item)
 
     def _contains(space, w_container, w_item):
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -378,7 +378,10 @@
     __slots__ = ()
 
     def __repr__(self):
-        s = '%s(%s)' % (self.__class__.__name__, getattr(self, 'name', ''))
+        name = getattr(self, 'name', '')
+        if not isinstance(name, str):
+            name = ''
+        s = '%s(%s)' % (self.__class__.__name__, name)
         w_cls = getattr(self, 'w__class__', None)
         if w_cls is not None and w_cls is not self:
             s += ' instance of %s' % self.w__class__
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -14,6 +14,7 @@
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
+from pypy.rlib import jit
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
@@ -398,6 +399,8 @@
 
     return _str_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _str_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     reslen = len(self) * (size - 1)
diff --git a/pypy/objspace/std/test/test_methodcache.py b/pypy/objspace/std/test/test_methodcache.py
--- a/pypy/objspace/std/test/test_methodcache.py
+++ b/pypy/objspace/std/test/test_methodcache.py
@@ -63,8 +63,31 @@
             assert a.f() == 42 + i
             A.f = eval("lambda self: %s" % (42 + i + 1, ))
         cache_counter = __pypy__.method_cache_counter("f")
-        # the cache hits come from A.f = ..., which first does a lookup on A as
-        # well
+        #
+        # a bit of explanation about what's going on.  (1) is the line "a.f()"
+        # and (2) is "A.f = ...".
+        #
+        # at line (1) we do the lookup on type(a).f
+        #
+        # at line (2) we do a setattr on A. However, descr_setattr does also a
+        # lookup of type(A).f i.e. type.f, to check if by chance 'f' is a data
+        # descriptor.
+        #
+        # At the first iteration:
+        # (1) is a miss because it's the first lookup of A.f. The result is cached
+        #
+        # (2) is a miss because it is the first lookup of type.f. The
+        # (non-existant) result is cached. The version of A changes, and 'f'
+        # is changed to be a cell object, so that subsequest assignments won't
+        # change the version of A
+        #
+        # At the second iteration:
+        # (1) is a miss because the version of A changed just before
+        # (2) is a hit, because type.f is cached. The version of A no longer changes
+        #
+        # At the third and subsequent iterations:
+        # (1) is a hit, because the version of A did not change
+        # (2) is a hit, see above
         assert cache_counter == (17, 3)
 
     def test_subclasses(self):
diff --git a/pypy/objspace/std/test/test_stringformat.py b/pypy/objspace/std/test/test_stringformat.py
--- a/pypy/objspace/std/test/test_stringformat.py
+++ b/pypy/objspace/std/test/test_stringformat.py
@@ -295,12 +295,11 @@
         assert u'%x' % ten == 'a'
 
     def test_long_no_overflow(self):
-        big = 100000000000L
-        assert "%x" % big == "174876e800"
+        big = 0x1234567890987654321
+        assert "%x" % big == "1234567890987654321"
 
     def test_missing_cases(self):
-        big = -123456789012345678901234567890L
-        print '%032d' % big
+        big = -123456789012345678901234567890
         assert '%032d' % big == '-0123456789012345678901234567890'
 
     def test_invalid_char(self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -16,6 +16,7 @@
 from pypy.rlib.runicode import unicode_encode_unicode_escape
 from pypy.module.unicodedata import unicodedb
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rlib import jit
 
 from pypy.objspace.std.formatting import mod_format
 from pypy.objspace.std.stringtype import stringstartswith, stringendswith
@@ -214,6 +215,8 @@
 
     return _unicode_join_many_items(space, w_self, list_w, size)
 
+ at jit.look_inside_iff(lambda space, w_self, list_w, size:
+                     jit.loop_unrolling_heuristic(list_w, size))
 def _unicode_join_many_items(space, w_self, list_w, size):
     self = w_self._value
     prealloc_size = len(self) * (size - 1)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -694,5 +694,31 @@
         l = len(X(X(2)))
         assert l == 2 and type(l) is int
 
+    def test_bool___contains__(self):
+        class X(object):
+            def __contains__(self, item):
+                if item == 'foo':
+                    return 42
+                else:
+                    return 'hello world'
+        x = X()
+        res = 'foo' in x
+        assert res is True
+        res = 'bar' in x
+        assert res is True
+        #
+        class MyError(Exception):
+            pass
+        class CannotConvertToBool(object):
+            def __nonzero__(self):
+                raise MyError
+        class X(object):
+            def __contains__(self, item):
+                return CannotConvertToBool()
+        x = X()
+        raises(MyError, "'foo' in x")
+        
+            
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -204,6 +204,14 @@
     return NonConstant(False)
 isvirtual._annspecialcase_ = "specialize:call_location"
 
+LIST_CUTOFF = 2
+
+ at specialize.call_location()
+def loop_unrolling_heuristic(lst, size):
+    """ In which cases iterating over items of lst can be unrolled
+    """
+    return isvirtual(lst) or (isconstant(size) and size <= LIST_CUTOFF)
+
 class Entry(ExtRegistryEntry):
     _about_ = hint
 
diff --git a/pypy/rlib/parsing/test/test_ebnfparse.py b/pypy/rlib/parsing/test/test_ebnfparse.py
--- a/pypy/rlib/parsing/test/test_ebnfparse.py
+++ b/pypy/rlib/parsing/test/test_ebnfparse.py
@@ -103,6 +103,7 @@
 """)
     parse = make_parse_function(regexs, rules)
     tree = parse("prefix(\n\tlonger(and_nested(term(X))), Xya, _, X0, _).")
+    assert tree.children[0].children[0].children[2].children[0].getsourcepos().lineno == 1
     assert tree is not None
     tree = parse("""
 foo(X, Y) :- bar(Y, X), bar(Y, X) ; foobar(X, Y, 1234, atom).""")
diff --git a/pypy/rlib/parsing/tree.py b/pypy/rlib/parsing/tree.py
--- a/pypy/rlib/parsing/tree.py
+++ b/pypy/rlib/parsing/tree.py
@@ -23,6 +23,9 @@
         self.symbol = symbol
         self.additional_info = additional_info
         self.token = token
+
+    def getsourcepos(self):
+        return self.token.source_pos
     
     def __repr__(self):
         return "Symbol(%r, %r)" % (self.symbol, self.additional_info)
@@ -49,6 +52,9 @@
         self.children = children
         self.symbol = symbol
 
+    def getsourcepos(self):
+        return self.children[0].getsourcepos()
+
     def __str__(self):
         return "%s(%s)" % (self.symbol, ", ".join([str(c) for c in self.children]))
 
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -3,8 +3,10 @@
 from pypy.translator.platform import platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 
-import sys
+import sys, os
 
+link_files = []
+testonly_libraries = []
 if sys.platform == 'win32' and platform.name != 'mingw32':
     libraries = ['libeay32', 'ssleay32',
                  'user32', 'advapi32', 'gdi32', 'msvcrt', 'ws2_32']
@@ -17,8 +19,18 @@
         # so that openssl/ssl.h can repair this nonsense.
         'wincrypt.h']
 else:
-    libraries = ['ssl', 'crypto']
+    libraries = ['z']
     includes = []
+    if (sys.platform.startswith('linux') and
+        os.path.exists('/usr/lib/libssl.a') and
+        os.path.exists('/usr/lib/libcrypto.a')):
+        # use static linking to avoid the infinite
+        # amount of troubles due to symbol versions
+        # and 0.9.8/1.0.0
+        link_files += ['/usr/lib/libssl.a', '/usr/lib/libcrypto.a']
+        testonly_libraries += ['ssl', 'crypto']
+    else:
+        libraries += ['ssl', 'crypto']
 
 includes += [
     'openssl/ssl.h', 
@@ -30,6 +42,8 @@
 
 eci = ExternalCompilationInfo(
     libraries = libraries,
+    link_files = link_files,
+    testonly_libraries = testonly_libraries,
     includes = includes,
     export_symbols = [],
     post_include_bits = [
diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py
--- a/pypy/rlib/rweakref.py
+++ b/pypy/rlib/rweakref.py
@@ -9,6 +9,9 @@
 
 ref = weakref.ref    # basic regular weakrefs are supported in RPython
 
+def has_weakref_support():
+    return True      # returns False if --no-translation-rweakref
+
 
 class RWeakValueDictionary(object):
     """A dictionary containing weak values."""
@@ -68,6 +71,20 @@
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.tool.pairtype import pairtype
 
+class Entry(extregistry.ExtRegistryEntry):
+    _about_ = has_weakref_support
+
+    def compute_result_annotation(self):
+        translator = self.bookkeeper.annotator.translator
+        res = translator.config.translation.rweakref
+        return self.bookkeeper.immutablevalue(res)
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
+
 class SomeWeakValueDict(annmodel.SomeObject):
     knowntype = RWeakValueDictionary
 
diff --git a/pypy/rlib/test/test_rweakref.py b/pypy/rlib/test/test_rweakref.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/test/test_rweakref.py
@@ -0,0 +1,14 @@
+from pypy.rlib.rweakref import has_weakref_support
+from pypy.rpython.test.test_llinterp import interpret
+
+
+def test_has_weakref_support():
+    assert has_weakref_support()
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': True})
+    assert res == True
+
+    res = interpret(lambda: has_weakref_support(), [],
+                    **{'translation.rweakref': False})
+    assert res == False
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1072,7 +1072,7 @@
     try:
         eci = _eci_cache[old_eci]
     except KeyError:
-        eci = old_eci.compile_shared_lib()
+        eci = old_eci.compile_shared_lib(ignore_a_files=True)
         _eci_cache[old_eci] = eci
 
     libraries = eci.testonly_libraries + eci.libraries + eci.frameworks
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -709,7 +709,8 @@
         return count
 
     @enforceargs(int, None)
-    @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
+    @jit.look_inside_iff(lambda length, items: jit.loop_unrolling_heuristic(
+        items, length))
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -1168,8 +1168,11 @@
             DIRENTP = lltype.Ptr(DIRENT)
             os_opendir = self.llexternal('opendir', [rffi.CCHARP], DIRP,
                                          compilation_info=compilation_info)
+            # XXX macro=True is hack to make sure we get the correct kind of
+            # dirent struct (which depends on defines)
             os_readdir = self.llexternal('readdir', [DIRP], DIRENTP,
-                                         compilation_info=compilation_info)
+                                         compilation_info=compilation_info,
+                                         macro=True)
             os_closedir = self.llexternal('closedir', [DIRP], rffi.INT,
                                           compilation_info=compilation_info)
 
diff --git a/pypy/rpython/module/test/test_ll_os.py b/pypy/rpython/module/test/test_ll_os.py
--- a/pypy/rpython/module/test/test_ll_os.py
+++ b/pypy/rpython/module/test/test_ll_os.py
@@ -4,6 +4,7 @@
 import pypy
 from pypy.tool.udir import udir
 from pypy.translator.c.test.test_genc import compile
+from pypy.rpython.module import ll_os #has side effect of registering functions
 
 from pypy.rpython import extregistry
 import errno
diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py
--- a/pypy/tool/clean_old_branches.py
+++ b/pypy/tool/clean_old_branches.py
@@ -4,30 +4,28 @@
 called 'closed-branch'.  It reduces the number of heads.
 """
 
-import os, sys
+import os
+import sys
+import commands
 
-if not os.listdir('.hg'):
+if not os.path.isdir('.hg'):
     print 'Must run this script from the top-level directory.'
     sys.exit(1)
 
-def heads(args):
-    g = os.popen(r"hg heads --topo %s --template '{node|short}:{branches}\n'"
-                 % args, 'r')
-    result = g.read()
-    g.close()
+def heads():
+    result = commands.getoutput(
+        "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'")
     result = result.splitlines(False)
+    result = [s.split(':', 2) for s in result]
     for line in result:
-        if len(line.split(':', 1)) != 2:
+        if len(line) != 3:
             raise ValueError("'result' contains: %r" % line)
-    result = [s.split(':', 1) for s in result]
-    result = [(head, branch) for (head, branch) in result
-                if branch not in ['', 'closed-branches']]
+    result = [(head, branch) for (head, branch, extra) in result
+                if branch not in ['', 'closed-branches'] and 'close' in extra]
     return result
 
-all_heads = heads("--closed")
-opened_heads = heads("")
 
-closed_heads = [s for s in all_heads if s not in opened_heads]
+closed_heads = heads()
 
 if not closed_heads:
     print >> sys.stderr, 'no dangling closed heads.'
@@ -56,16 +54,14 @@
         print '*** error %r' % (err,)
         sys.exit(1)
 
+print '*** switching to closed branches *** '
+do("hg up --clean closed-branches")
+do("hg --config extensions.purge= purge --all")
+
 for head, branch in closed_heads:
     print
     print '***** %s ***** %s *****' % (branch, head)
-    do("hg up --clean closed-branches")
-    do("hg --config extensions.purge= purge --all")
-    do("hg merge -y %s" % head)
-    for fn in os.listdir('.'):
-        if fn.lower() != '.hg':
-            do("rm -fr -- '%s'" % fn)
-            do("hg rm --after -- '%s' || true" % fn)
+    do("hg debugsetparents closed-branches %s" % head)
     do("hg ci -m'Merge closed head %s on branch %s'" % (head, branch))
 
 print
diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py
--- a/pypy/tool/test/test_udir.py
+++ b/pypy/tool/test/test_udir.py
@@ -13,6 +13,8 @@
 def test_make_udir_with_basename():
     root = str(udir.udir.ensure('make_udir2', dir=1))
     p1 = udir.make_udir(dir=root, basename='foobar')
+    def assert_relto(path, root, expected):
+        assert path.relto(root) == expected, path.relto(root)
     assert p1.relto(root) == 'usession-foobar-0'
     p1 = udir.make_udir(dir=root, basename='-foobar')
     assert p1.relto(root) == 'usession-foobar-1'
@@ -24,3 +26,5 @@
     assert p1.relto(root) == 'usession-0'
     p1 = udir.make_udir(dir=root, basename='-')
     assert p1.relto(root) == 'usession-1'
+    p1 = udir.make_udir(dir=root, basename='fun/bar')
+    assert p1.relto(root) == 'usession-fun--bar-0'
diff --git a/pypy/tool/udir.py b/pypy/tool/udir.py
--- a/pypy/tool/udir.py
+++ b/pypy/tool/udir.py
@@ -41,6 +41,7 @@
                 basename = basename.encode(sys.getdefaultencoding())
         else:
             basename = ''
+    basename = basename.replace('/', '--')
     if not basename.startswith('-'):
         basename = '-' + basename
     if not basename.endswith('-'):
diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.lltypesystem.rstr import STR, mallocstr
 from pypy.rpython.lltypesystem import rstr
 from pypy.rpython.lltypesystem import rlist
-from pypy.rpython.module import ll_time, ll_os
 
 # table of functions hand-written in src/ll_*.h
 # Note about *.im_func: The annotator and the rtyper expect direct
diff --git a/pypy/translator/c/src/exception.h b/pypy/translator/c/src/exception.h
--- a/pypy/translator/c/src/exception.h
+++ b/pypy/translator/c/src/exception.h
@@ -43,6 +43,16 @@
           filename, lineno, functionname);
 }
 #endif
+#else   /* !DO_LOG_EXC: define the function anyway, so that we can shut
+           off the prints of a debug_exc by remaking only testing_1.o */
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname);
+#ifndef PYPY_NOT_MAIN_FILE
+void RPyDebugReturnShowException(const char *msg, const char *filename,
+                                 long lineno, const char *functionname)
+{
+}
+#endif
 #endif  /* DO_LOG_EXC */
 
 /* Hint: functions and macros not defined here, like RPyRaiseException,
diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py
--- a/pypy/translator/driver.py
+++ b/pypy/translator/driver.py
@@ -115,12 +115,10 @@
         backend, ts = self.get_backend_and_type_system()
         for task in self.tasks:
             explicit_task = task
-            parts = task.split('_')
-            if len(parts) == 1:
-                if task in ('annotate',):
-                    expose_task(task)
+            if task == 'annotate':
+                expose_task(task)
             else:
-                task, postfix = parts
+                task, postfix = task.split('_')
                 if task in ('rtype', 'backendopt', 'llinterpret',
                             'pyjitpl'):
                     if ts:
diff --git a/pypy/translator/tool/cbuild.py b/pypy/translator/tool/cbuild.py
--- a/pypy/translator/tool/cbuild.py
+++ b/pypy/translator/tool/cbuild.py
@@ -267,9 +267,12 @@
         d['separate_module_files'] = ()
         return files, ExternalCompilationInfo(**d)
 
-    def compile_shared_lib(self, outputfilename=None):
+    def compile_shared_lib(self, outputfilename=None, ignore_a_files=False):
         self = self.convert_sources_to_files()
-        if not self.separate_module_files:
+        if ignore_a_files:
+            if not [fn for fn in self.link_files if fn.endswith('.a')]:
+                ignore_a_files = False    # there are none
+        if not self.separate_module_files and not ignore_a_files:
             if sys.platform != 'win32':
                 return self
             if not self.export_symbols:
@@ -288,6 +291,13 @@
                 num += 1
             basepath.ensure(dir=1)
             outputfilename = str(pth.dirpath().join(pth.purebasename))
+
+        if ignore_a_files:
+            d = self._copy_attributes()
+            d['link_files'] = [fn for fn in d['link_files']
+                                  if not fn.endswith('.a')]
+            self = ExternalCompilationInfo(**d)
+
         lib = str(host.compile([], self, outputfilename=outputfilename,
                                standalone=False))
         d = self._copy_attributes()

From noreply at buildbot.pypy.org  Tue Apr 24 02:37:02 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue, 24 Apr 2012 02:37:02 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: better auto-casting support
	(Reflex only)
Message-ID: <20120424003702.014CE8203C@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54704:ad4fa188692f
Date: 2012-04-23 17:36 -0700
http://bitbucket.org/pypy/pypy/changeset/ad4fa188692f/

Log:	better auto-casting support (Reflex only)

diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py
--- a/pypy/module/cppyy/capi/__init__.py
+++ b/pypy/module/cppyy/capi/__init__.py
@@ -243,15 +243,15 @@
 
 _c_base_offset = rffi.llexternal(
     "cppyy_base_offset",
-    [C_TYPE, C_TYPE, C_OBJECT], rffi.SIZE_T,
+    [C_TYPE, C_TYPE, C_OBJECT, rffi.INT], rffi.LONG,
     threadsafe=threadsafe,
     compilation_info=backend.eci,
     elidable_function=True)
 @jit.elidable_promote()
-def c_base_offset(derived, base, address):
+def c_base_offset(derived, base, address, direction):
     if derived == base:
         return 0
-    return _c_base_offset(derived.handle, base.handle, address)
+    return _c_base_offset(derived.handle, base.handle, address, direction)
 
 # method/function reflection information -------------------------------------
 _c_num_methods = rffi.llexternal(
diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -584,7 +584,7 @@
         if isinstance(obj, W_CPPInstance):
             if capi.c_is_subtype(obj.cppclass, self.cppclass):
                 rawobject = obj.get_rawobject()
-                offset = capi.c_base_offset(obj.cppclass, self.cppclass, rawobject)
+                offset = capi.c_base_offset(obj.cppclass, self.cppclass, rawobject, 1)
                 obj_address = capi.direct_ptradd(rawobject, offset)
                 return rffi.cast(capi.C_OBJECT, obj_address)
         raise OperationError(space.w_TypeError,
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -59,7 +59,9 @@
     int cppyy_num_bases(cppyy_type_t type);
     char* cppyy_base_name(cppyy_type_t type, int base_index);
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
-    size_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address);
+
+    /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
+    long cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
     int cppyy_num_methods(cppyy_scope_t scope);
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -354,7 +354,7 @@
         if cppinstance:
             assert lltype.typeOf(cppinstance.cppclass.handle) == lltype.typeOf(self.scope.handle)
             offset = self.offset + capi.c_base_offset(
-                cppinstance.cppclass, self.scope, cppinstance.get_rawobject())
+                cppinstance.cppclass, self.scope, cppinstance.get_rawobject(), 1)
         else:
             offset = self.offset
         return offset
@@ -594,7 +594,7 @@
 
     def get_cppthis(self, cppinstance, calling_scope):
         assert self == cppinstance.cppclass
-        offset = capi.c_base_offset(self, calling_scope, cppinstance.get_rawobject())
+        offset = capi.c_base_offset(self, calling_scope, cppinstance.get_rawobject(), 1)
         return capi.direct_ptradd(cppinstance.get_rawobject(), offset)
 
 W_ComplexCPPClass.typedef = TypeDef(
@@ -749,7 +749,7 @@
     if rawobject:
         actual = capi.c_actual_class(cppclass, rawobject)
         if actual != cppclass.handle:
-            offset = capi._c_base_offset(actual, cppclass.handle, rawobject)
+            offset = capi._c_base_offset(actual, cppclass.handle, rawobject, -1)
             rawobject = capi.direct_ptradd(rawobject, offset)
             w_pycppclass = get_pythonized_cppclass(space, actual)
             w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy"))
diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx
--- a/pypy/module/cppyy/src/cintcwrapper.cxx
+++ b/pypy/module/cppyy/src/cintcwrapper.cxx
@@ -467,11 +467,13 @@
     return derived_type->GetBaseClass(base_type) != 0;
 }
 
-size_t cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle, cppyy_object_t address) {
+long cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle,
+                       cppyy_object_t address, int /* direction */) {
+    // WARNING: CINT can not handle actual dynamic casts!
     TClassRef derived_type = type_from_handle(derived_handle);
     TClassRef base_type = type_from_handle(base_handle);
 
-    size_t offset = 0;
+    long offset = 0;
 
     if (derived_type && base_type) {
         G__ClassInfo* base_ci    = (G__ClassInfo*)base_type->GetClassInfo();
@@ -483,18 +485,15 @@
             // with CINT's (or Reflex's) interfaces.
             long baseprop = derived_ci->IsBase(*base_ci);
             if (!baseprop || (baseprop & G__BIT_ISVIRTUALBASE))
-                offset = (size_t)derived_type->GetBaseClassOffset(base_type);
+                offset = derived_type->GetBaseClassOffset(base_type);
             else
 #endif
                 offset = G__isanybase(base_ci->Tagnum(), derived_ci->Tagnum(), (long)address);
          } else {
-             offset = (size_t)derived_type->GetBaseClassOffset(base_type);
+             offset = derived_type->GetBaseClassOffset(base_type);
          }
     }
 
-    if (offset < 0)      // error return of G__isanybase()
-        return 0;
-
     return offset;
 }
 
diff --git a/pypy/module/cppyy/src/reflexcwrapper.cxx b/pypy/module/cppyy/src/reflexcwrapper.cxx
--- a/pypy/module/cppyy/src/reflexcwrapper.cxx
+++ b/pypy/module/cppyy/src/reflexcwrapper.cxx
@@ -277,7 +277,8 @@
     return (int)derived_type.HasBase(base_type);
 }
 
-size_t cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle, cppyy_object_t address) {
+long cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle,
+                       cppyy_object_t address, int direction) {
     Reflex::Type derived_type = type_from_handle(derived_handle);
     Reflex::Type base_type = type_from_handle(base_handle);
 
@@ -291,9 +292,20 @@
         Reflex::Object bases_holder(Reflex::Type::ByTypeInfo(typeid(Bases_t)), &bases);
         getbases.Invoke(&bases_holder);
 
+        // if direction is down-cast, perform the cast in C++ first in order to ensure
+        // we have a derived object for accessing internal offset pointers
+        if (direction < 0) {
+           Reflex::Object o(base_type, (void*)address);
+           address = (cppyy_object_t)o.CastObject(derived_type).Address();
+        }
+
         for (Bases_t::iterator ibase = bases->begin(); ibase != bases->end(); ++ibase) {
-            if (ibase->first.ToType() == base_type)
-                return (size_t)ibase->first.Offset((void*)address);
+            if (ibase->first.ToType() == base_type) {
+                long offset = (long)ibase->first.Offset((void*)address);
+                if (direction < 0)
+                    return -offset;
+                return offset;
+            }
         }
 
         // contrary to typical invoke()s, the result of the internal getbases function
diff --git a/pypy/module/cppyy/test/advancedcpp.cxx b/pypy/module/cppyy/test/advancedcpp.cxx
--- a/pypy/module/cppyy/test/advancedcpp.cxx
+++ b/pypy/module/cppyy/test/advancedcpp.cxx
@@ -10,6 +10,9 @@
 
 
 // for esoteric inheritance testing
+a_class* create_c1() { return new c_class_1; }
+a_class* create_c2() { return new c_class_2; }
+
 int get_a( a_class& a ) { return a.m_a; }
 int get_b( b_class& b ) { return b.m_b; }
 int get_c( c_class& c ) { return c.m_c; }
diff --git a/pypy/module/cppyy/test/advancedcpp.h b/pypy/module/cppyy/test/advancedcpp.h
--- a/pypy/module/cppyy/test/advancedcpp.h
+++ b/pypy/module/cppyy/test/advancedcpp.h
@@ -91,6 +91,9 @@
    int m_d;
 };
 
+a_class* create_c1();
+a_class* create_c2();
+
 int get_a(a_class& a);
 int get_b(b_class& b);
 int get_c(c_class& c);
diff --git a/pypy/module/cppyy/test/advancedcpp.xml b/pypy/module/cppyy/test/advancedcpp.xml
--- a/pypy/module/cppyy/test/advancedcpp.xml
+++ b/pypy/module/cppyy/test/advancedcpp.xml
@@ -12,6 +12,7 @@
   <class name="c_class_2" />
   <class name="d_class" />
 
+  <function pattern="create_*" />
   <function pattern="get_*" />
 
   <class pattern="T1<*>" />
diff --git a/pypy/module/cppyy/test/advancedcpp_LinkDef.h b/pypy/module/cppyy/test/advancedcpp_LinkDef.h
--- a/pypy/module/cppyy/test/advancedcpp_LinkDef.h
+++ b/pypy/module/cppyy/test/advancedcpp_LinkDef.h
@@ -16,6 +16,9 @@
 #pragma link C++ class c_class_2;
 #pragma link C++ class d_class;
 
+#pragma link C++ function create_c1();
+#pragma link C++ function create_c2();
+
 #pragma link C++ function get_a(a_class&);
 #pragma link C++ function get_b(b_class&);
 #pragma link C++ function get_c(c_class&);
diff --git a/pypy/module/cppyy/test/test_advancedcpp.py b/pypy/module/cppyy/test/test_advancedcpp.py
--- a/pypy/module/cppyy/test/test_advancedcpp.py
+++ b/pypy/module/cppyy/test/test_advancedcpp.py
@@ -1,6 +1,8 @@
 import py, os, sys
 from pypy.conftest import gettestobjspace
 
+from pypy.module.cppyy import capi
+
 
 currpath = py.path.local(__file__).dirpath()
 test_dct = str(currpath.join("advancedcppDict.so"))
@@ -20,6 +22,7 @@
         cls.space = space
         env = os.environ
         cls.w_test_dct = space.wrap(test_dct)
+        cls.w_capi_identity = space.wrap(capi.identify())
         cls.w_advanced = cls.space.appexec([], """():
             import cppyy
             return cppyy.load_reflection_info(%r)""" % (test_dct, ))
@@ -466,3 +469,22 @@
 
         assert isinstance(b.clone(), base_class)      # TODO: clone() leaks
         assert isinstance(d.clone(), derived_class)   # TODO: clone() leaks
+
+    def test13_actual_type_virtual_multi(self):
+        """Test auto-downcast in adverse inheritance situation"""
+
+        import cppyy
+
+        c1 = cppyy.gbl.create_c1()
+        assert type(c1) == cppyy.gbl.c_class_1
+        assert c1.m_c == 3
+        c1.destruct()
+
+        if self.capi_identity == 'CINT':     # CINT does not support dynamic casts
+            return
+
+        c2 = cppyy.gbl.create_c2()
+        assert type(c2) == cppyy.gbl.c_class_2
+        print c2.m_c
+        assert c2.m_c == 3
+        c2.destruct()

From noreply at buildbot.pypy.org  Tue Apr 24 03:12:29 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 03:12:29 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: consts with more types
Message-ID: <20120424011229.081A58203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54705:5716a96a3c7f
Date: 2012-04-23 21:12 -0400
http://bitbucket.org/pypy/pypy/changeset/5716a96a3c7f/

Log:	consts with more types

diff --git a/pypy/rlib/rerased_raw.py b/pypy/rlib/rerased_raw.py
--- a/pypy/rlib/rerased_raw.py
+++ b/pypy/rlib/rerased_raw.py
@@ -7,7 +7,7 @@
 from pypy.annotation import model as annmodel
 from pypy.annotation.bookkeeper import getbookkeeper
 from pypy.rlib import longlong2float
-from pypy.rpython.annlowlevel import (hlstr, llstr, llhelper,
+from pypy.rpython.annlowlevel import (hlstr, llstr, llunicode, llhelper,
     cast_instance_to_base_ptr)
 from pypy.rpython.rclass import getinstancerepr
 from pypy.rpython.extregistry import ExtRegistryEntry
@@ -214,6 +214,14 @@
         for idx, (char, obj) in enumerate(zip(value.shape, value.storage)):
             if char == INT:
                 storage.data[idx] = rffi.cast(llmemory.Address, obj)
+            elif char == BOOL:
+                storage.data[idx] = rffi.cast(llmemory.Address, obj)
+            elif char == FLOAT:
+                storage.data[idx] = rffi.cast(llmemory.Address, longlong2float.float2longlong(obj))
+            elif char == STRING:
+                storage.data[idx] = llmemory.cast_ptr_to_adr(llstr(obj))
+            elif char == UNICODE:
+                storage.data[idx] = llmemory.cast_ptr_to_adr(llunicode(obj))
             elif char == INSTANCE:
                 bk = self.rtyper.annotator.bookkeeper
                 classdef = bk.getuniqueclassdef(type(obj))
@@ -308,7 +316,7 @@
         elif elem == INT:
             yield i, rffi.cast(lltype.Signed, storage.data.items[i])
         elif elem == FLOAT:
-            yield i, longlong2float.longlong2float(rffi.cast(lltype.Signed, storage.data.items[i]))
+            yield i, rffi.cast(lltype.Signed, storage.data.items[i])
         elif elem == BOOL:
             yield i, rffi.cast(lltype.Bool, storage.data.items[i])
         else:
diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -1,6 +1,6 @@
 import py
 
-from pypy.rlib import rerased_raw
+from pypy.rlib import rerased_raw, longlong2float
 from pypy.rpython.annlowlevel import hlstr
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin
 
@@ -184,6 +184,24 @@
         res = self.interpret(f, [0])
         assert res == 21
 
+    def test_const_types(self):
+        storage = rerased_raw.UntypedStorage("bfsu")
+        storage.setbool(0, True)
+        storage.setfloat(1, 2.5)
+        storage.setstr(2, "hello")
+        storage.setunicode(3, u"world!")
+
+        def f(i):
+            if i:
+                local_storage = rerased_raw.UntypedStorage("o")
+            else:
+                local_storage = storage
+            return (local_storage.getbool(0) + local_storage.getfloat(1) +
+                    len(local_storage.getstr(2)) + len(local_storage.getunicode(3)))
+
+        res = self.interpret(f, [0])
+        assert res == 14.5
+
     def test_enumerate_elements(self):
         def f():
             storage = rerased_raw.UntypedStorage("sibf")
@@ -197,7 +215,9 @@
         lst = list(rerased_raw.ll_enumerate_elements(llres))
         assert hlstr(lst[0][1]) == "abc"
         assert lst[0][0] == 0
-        assert lst[1:] == [(1, 13), (2, True), (3, 3.5)]
+        assert lst[1:3] == [(1, 13), (2, True)]
+        assert lst[3][0] == 3
+        assert longlong2float.longlong2float(lst[3][1]) == 3.5
 
 class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
     pass

From noreply at buildbot.pypy.org  Tue Apr 24 03:14:31 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 03:14:31 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Add get_latest_value_float.
Message-ID: <20120424011431.8AAE58203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54706:d881d05b561e
Date: 2012-04-23 21:12 -0400
http://bitbucket.org/pypy/pypy/changeset/d881d05b561e/

Log:	Add get_latest_value_float.

diff --git a/pypy/jit/backend/ppc/runner.py b/pypy/jit/backend/ppc/runner.py
--- a/pypy/jit/backend/ppc/runner.py
+++ b/pypy/jit/backend/ppc/runner.py
@@ -114,9 +114,11 @@
         return self.asm.fail_boxes_count
 
     # fetch the result of the computation and return it
+    def get_latest_value_float(self, index):
+        return self.asm.fail_boxes_float.getitem(index)
+
     def get_latest_value_int(self, index):
-        value = self.asm.fail_boxes_int.getitem(index)
-        return value
+        return self.asm.fail_boxes_int.getitem(index)
 
     def get_latest_value_ref(self, index):
         return self.asm.fail_boxes_ptr.getitem(index)

From noreply at buildbot.pypy.org  Tue Apr 24 03:14:32 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 03:14:32 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Add more float regalloc support.
Message-ID: <20120424011432.C93078203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54707:b08e53fa2e91
Date: 2012-04-23 21:13 -0400
http://bitbucket.org/pypy/pypy/changeset/b08e53fa2e91/

Log:	Add more float regalloc support.

diff --git a/pypy/jit/backend/ppc/regalloc.py b/pypy/jit/backend/ppc/regalloc.py
--- a/pypy/jit/backend/ppc/regalloc.py
+++ b/pypy/jit/backend/ppc/regalloc.py
@@ -280,7 +280,10 @@
         self.possibly_free_vars(list(inputargs))
 
     def possibly_free_var(self, var):
-        self.rm.possibly_free_var(var)
+        if var.type == FLOAT:
+            self.fprm.possibly_free_var(var)
+        else:
+            self.rm.possibly_free_var(var)
 
     def possibly_free_vars(self, vars):
         for var in vars:
@@ -293,12 +296,19 @@
                 self.possibly_free_var(var)
 
     def try_allocate_reg(self, v, selected_reg=None, need_lower_byte=False):
-        return self.rm.try_allocate_reg(v, selected_reg, need_lower_byte)
+        if v.type == FLOAT:
+            return self.fprm.try_allocate_reg(v, selected_reg, need_lower_byte)
+        else:
+            return self.rm.try_allocate_reg(v, selected_reg, need_lower_byte)
 
     def force_allocate_reg(self, var, forbidden_vars=[], selected_reg=None, 
             need_lower_byte=False):
-        return self.rm.force_allocate_reg(var, forbidden_vars, selected_reg,
-                need_lower_byte)
+        if var.type == FLOAT:
+            return self.fprm.force_allocate_reg(var, forbidden_vars,
+                                                selected_reg, need_lower_byte)
+        else:
+            return self.rm.force_allocate_reg(var, forbidden_vars,
+                                              selected_reg, need_lower_byte)
 
     def allocate_scratch_reg(self, type=INT, forbidden_vars=[], selected_reg=None):
         assert type == INT # XXX extend this once floats are supported
@@ -331,6 +341,7 @@
 
     def before_call(self, force_store=[], save_all_regs=False):
         self.rm.before_call(force_store, save_all_regs)
+        self.fprm.before_call(force_store, save_all_regs)
 
     def after_call(self, v):
         if v.type == FLOAT:

From noreply at buildbot.pypy.org  Tue Apr 24 03:14:34 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 03:14:34 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Define float_locs.
Message-ID: <20120424011434.122518203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54708:84efa81dcdbd
Date: 2012-04-23 21:13 -0400
http://bitbucket.org/pypy/pypy/changeset/84efa81dcdbd/

Log:	Define float_locs.

diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -500,6 +500,7 @@
         count = 0
         non_float_locs = []
         non_float_regs = []
+        float_locs = []
         for i in range(reg_args):
             arg = arglocs[i]
             if arg.type == FLOAT and count % 2 != 0:

From noreply at buildbot.pypy.org  Tue Apr 24 03:14:35 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 03:14:35 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Add float support to descriptors.
Message-ID: <20120424011435.518B88203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54709:cc2f39e17bd9
Date: 2012-04-23 21:14 -0400
http://bitbucket.org/pypy/pypy/changeset/cc2f39e17bd9/

Log:	Add float support to descriptors.

diff --git a/pypy/jit/backend/ppc/ppc_assembler.py b/pypy/jit/backend/ppc/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppc_assembler.py
@@ -197,6 +197,7 @@
         bytecode = rffi.cast(rffi.UCHARP, mem_loc)
         num = 0
         value = 0
+        fvalue = 0
         code_inputarg = False
         while True:
             code = rffi.cast(lltype.Signed, bytecode[0])
@@ -219,7 +220,8 @@
                     code = ~code
                     code_inputarg = False
                 if kind == self.DESCR_FLOAT:
-                    assert 0, "not implemented yet"
+                    start = spp + get_spp_offset(int(code))
+                    fvalue = rffi.cast(rffi.LONGP, start)[0]
                 else:
                     start = spp + get_spp_offset(int(code))
                     value = rffi.cast(rffi.LONGP, start)[0]
@@ -237,13 +239,14 @@
                     break
                 code >>= 2
                 if kind == self.DESCR_FLOAT:
-                    assert 0, "not implemented yet"
+                    fvalue = fp_registers[code]
                 else:
                     reg_index = r.get_managed_reg_index(code)
                     value = registers[reg_index]
             # store the loaded value into fail_boxes_<type>
             if kind == self.DESCR_FLOAT:
-                assert 0, "not implemented yet"
+                tgt = self.fail_boxes_float.get_addr_for_num(num)
+                rffi.cast(rffi.LONGP, tgt)[0] = fvalue
             else:
                 if kind == self.DESCR_INT:
                     tgt = self.fail_boxes_int.get_addr_for_num(num)
@@ -298,7 +301,7 @@
                 kind = code & 3
                 code >>= 2
                 if kind == self.DESCR_FLOAT:
-                    assert 0, "not implemented yet"
+                    loc = r.ALL_FLOAT_REGS[code]
                 else:
                     #loc = r.all_regs[code]
                     assert (r.ALL_REGS[code] is 
@@ -904,7 +907,7 @@
                 elif arg.type == INT:
                     kind = self.DESCR_INT
                 elif arg.type == FLOAT:
-                    assert 0, "not implemented"
+                    kind = self.DESCR_FLOAT
                 else:
                     raise AssertionError("bogus kind")
                 loc = locs[i]

From noreply at buildbot.pypy.org  Tue Apr 24 04:40:56 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 04:40:56 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: fix,
	and mark something as elidable
Message-ID: <20120424024056.971A48204C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54711:041945f6ec4d
Date: 2012-04-23 22:40 -0400
http://bitbucket.org/pypy/pypy/changeset/041945f6ec4d/

Log:	fix, and mark something as elidable

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -205,11 +205,11 @@
     def int_w(self, space):
         raise OperationError(space.w_TypeError,
                              typed_unwrap_error_msg(space, "integer", self))
-    
+
     def uint_w(self, space):
         raise OperationError(space.w_TypeError,
                              typed_unwrap_error_msg(space, "integer", self))
-    
+
     def bigint_w(self, space):
         raise OperationError(space.w_TypeError,
                              typed_unwrap_error_msg(space, "integer", self))
@@ -759,6 +759,7 @@
         self.interned_strings[s] = w_s
         return w_s
 
+    @jit.elidable
     def new_interned_str(self, s):
         try:
             return self.interned_strings[s]
@@ -1343,7 +1344,7 @@
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
             raise OperationError(self.w_TypeError,
                                  self.wrap('argument must be a string'))
-        return self.str_w(w_obj)            
+        return self.str_w(w_obj)
 
     def unicode_w(self, w_obj):
         return w_obj.unicode_w(self)
@@ -1655,7 +1656,7 @@
     'UnicodeEncodeError',
     'UnicodeDecodeError',
     ]
-    
+
 if sys.platform.startswith("win"):
     ObjSpace.ExceptionTable += ['WindowsError']
 
diff --git a/pypy/objspace/std/tupletype.py b/pypy/objspace/std/tupletype.py
--- a/pypy/objspace/std/tupletype.py
+++ b/pypy/objspace/std/tupletype.py
@@ -91,7 +91,7 @@
     assert False
 
 @jit.look_inside_iff(lambda space, w_tuple, list_w:
-    isvirtual(list_w) or (isconstant(len(list_w)) and len(list_w) <= 5)
+    jit.isvirtual(list_w) or (jit.isconstant(len(list_w)) and len(list_w) <= 5)
 )
 def make_tuple(space, w_tuple, list_w):
     from pypy.objspace.std.tupleobject import W_TupleObject

From noreply at buildbot.pypy.org  Tue Apr 24 04:40:55 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 04:40:55 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: unroll this so it's
	actually useful
Message-ID: <20120424024055.574A18203C@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54710:10fb8b0fb0df
Date: 2012-04-23 21:56 -0400
http://bitbucket.org/pypy/pypy/changeset/10fb8b0fb0df/

Log:	unroll this so it's actually useful

diff --git a/pypy/objspace/std/tupletype.py b/pypy/objspace/std/tupletype.py
--- a/pypy/objspace/std/tupletype.py
+++ b/pypy/objspace/std/tupletype.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
+from pypy.rlib import jit
 from pypy.rlib.rerased_raw import (UntypedStorage, INT, BOOL, FLOAT, INSTANCE,
     STRING, UNICODE)
 from pypy.rlib.unroll import unrolling_iterable
@@ -89,6 +90,9 @@
             return read(space, storage, idx)
     assert False
 
+ at jit.look_inside_iff(lambda space, w_tuple, list_w:
+    isvirtual(list_w) or (isconstant(len(list_w)) and len(list_w) <= 5)
+)
 def make_tuple(space, w_tuple, list_w):
     from pypy.objspace.std.tupleobject import W_TupleObject
 

From noreply at buildbot.pypy.org  Tue Apr 24 04:47:46 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 04:47:46 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Add float support to
 emit_guard_value and emit_guard_call_assembler.
Message-ID: <20120424024746.7D7CA8203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54712:9d75068f7a11
Date: 2012-04-23 22:45 -0400
http://bitbucket.org/pypy/pypy/changeset/9d75068f7a11/

Log:	Add float support to emit_guard_value and emit_guard_call_assembler.

diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -282,8 +282,9 @@
                 self.mc.cmp_op(0, l0.value, l1.getint(), imm=True)
             else:
                 self.mc.cmp_op(0, l0.value, l1.value)
-        else:
-            assert 0, "not implemented yet"
+        elif l0.is_fp_reg():
+            assert l1.is_fp_reg()
+            self.mc.cmp_op(0, l0,value, l1.value)
         self._emit_guard(op, failargs, c.NE)
 
     emit_guard_nonnull = emit_guard_true
@@ -1179,13 +1180,13 @@
             elif kind == REF:
                 adr = self.fail_boxes_ptr.get_addr_for_num(0)
             elif kind == FLOAT:
-                assert 0, "not implemented"
+                adr = self.fail_boxes_float.get_addr_for_num(0)
             else:
                 raise AssertionError(kind)
             with scratch_reg(self.mc):
                 self.mc.load_imm(r.SCRATCH, adr)
                 if op.result.type == FLOAT:
-                    assert 0, "not implemented yet"
+                    self.mc.lfdx(resloc.value, 0, r.SCRATCH.value)
                 else:
                     self.mc.loadx(resloc.value, 0, r.SCRATCH.value)
 
@@ -1201,13 +1202,15 @@
         # Path B: use assembler helper
         asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr)
         if self.cpu.supports_floats:
-            assert 0, "not implemented yet"
+            floats = r.VOLATILES_FLOAT
+        else:
+            floats = []
 
         with Saved_Volatiles(self.mc, save_RES=False):
             # result of previous call is in r3
             self.mov_loc_loc(arglocs[0], r.r4)
             self.mc.call(asm_helper_adr)
-            if op.result and resloc.is_vfp_reg():
+            if op.result and resloc.is_fp_reg():
                 assert 0, "not implemented yet"
 
         # merge point

From noreply at buildbot.pypy.org  Tue Apr 24 04:47:47 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 04:47:47 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Fix float address subtraction in
	regalloc_mov.
Message-ID: <20120424024747.B523F8203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54713:c320246747e0
Date: 2012-04-23 22:46 -0400
http://bitbucket.org/pypy/pypy/changeset/c320246747e0/

Log:	Fix float address subtraction in regalloc_mov.

diff --git a/pypy/jit/backend/ppc/ppc_assembler.py b/pypy/jit/backend/ppc/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppc_assembler.py
@@ -918,7 +918,7 @@
                         pos = ~pos
                     n = self.CODE_FROMSTACK // 4 + pos
                 else:
-                    assert loc.is_reg() or loc.is_vfp_reg()
+                    assert loc.is_reg() or loc.is_fp_reg()
                     n = loc.value
                 n = kind + 4 * n
                 while n > 0x7F:
@@ -1222,9 +1222,9 @@
                     offset = loc.value
                     if not _check_imm_arg(offset):
                         self.mc.load_imm(r.SCRATCH, offset)
-                        self.mc.sub(r.SCRATCH.value, r.SPP.value, offset)
+                        self.mc.sub(r.SCRATCH.value, r.SPP.value, r.SCRATCH.value)
                     else:
-                        self.mc.sub(r.SCRATCH.value, r.SPP.value, r.SCRATCH.value)
+                        self.mc.subi(r.SCRATCH.value, r.SPP.value, offset)
                     self.mc.stfdx(prev_loc.value, 0, r.SCRATCH.value)
                 return
             assert 0, "not supported location"

From noreply at buildbot.pypy.org  Tue Apr 24 04:47:48 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 04:47:48 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Add float support to prepare_jump.
Message-ID: <20120424024748.EA9CD8203C@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54714:b6d30e9ba7df
Date: 2012-04-23 22:47 -0400
http://bitbucket.org/pypy/pypy/changeset/b6d30e9ba7df/

Log:	Add float support to prepare_jump.

diff --git a/pypy/jit/backend/ppc/regalloc.py b/pypy/jit/backend/ppc/regalloc.py
--- a/pypy/jit/backend/ppc/regalloc.py
+++ b/pypy/jit/backend/ppc/regalloc.py
@@ -263,7 +263,7 @@
             i += 1
             if loc.is_reg():
                 self.rm.reg_bindings[arg] = loc
-            elif loc.is_vfp_reg():
+            elif loc.is_fp_reg():
                 assert 0, "not supported"
             else:
                 assert loc.is_stack()
@@ -663,10 +663,13 @@
 
         # get temporary locs
         tmploc = r.SCRATCH
+        fptmploc = r.f0
 
         # Part about non-floats
         src_locations1 = []
         dst_locations1 = []
+        src_locations2 = []
+        dst_locations2 = []
 
         # Build the four lists
         for i in range(op.numargs()):
@@ -677,10 +680,12 @@
                 src_locations1.append(src_loc)
                 dst_locations1.append(dst_loc)
             else:
-                assert 0, "not implemented yet"
+                src_locations2.append(src_loc)
+                dst_locations2.append(dst_loc)
 
-        remap_frame_layout(self.assembler, src_locations1,
-                dst_locations1, tmploc)
+        remap_frame_layout(self.assembler,
+                           src_locations1, dst_locations1, tmploc,
+                           src_locations2, dst_locations2, fptmploc)
         return []
 
     def prepare_setfield_gc(self, op):

From noreply at buildbot.pypy.org  Tue Apr 24 12:34:16 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Apr 2012 12:34:16 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix test.
Message-ID: <20120424103416.3C5CF82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54715:9cfe0f9a4478
Date: 2012-04-24 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/9cfe0f9a4478/

Log:	Fix test.

diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -368,16 +368,26 @@
         from pypy.rpython.memory.gc.test import test_stmtls
         self.gc.root_walker = test_stmtls.FakeRootWalker()
         #
-        tr1, tr1_adr = self.malloc(SR)            # local
-        tr2, tr2_adr = self.malloc(SR)            # local
+        tr1, tr1_adr = self.malloc(SR, globl=False)  # local
+        tr2, tr2_adr = self.malloc(SR, globl=False)  # local
+        self.checkflags(tr1_adr, False, False)    # check that it is local
+        self.checkflags(tr2_adr, False, False)    # check that it is local
         tr1.sr2 = tr2
         self.gc.root_walker.current_stack = [tr1]
         self.gc.enter_transactional_mode()
-        self.checkflags(tr1_adr, True, False)     # tr1 has become global
-        self.checkflags(tr2_adr, True, False)     # tr2 has become global
+        # tr1 and tr2 moved out of the nursery: check that
+        [sr1] = self.gc.root_walker.current_stack
+        assert sr1._obj0 != tr1._obj0
+        sr2 = sr1.sr2
+        assert sr2 and sr2 != sr1 and not sr2.sr2
+        assert sr2._obj0 != tr2._obj0
+        sr1_adr = llmemory.cast_ptr_to_adr(sr1)
+        sr2_adr = llmemory.cast_ptr_to_adr(sr2)
+        self.checkflags(sr1_adr, True, False)     # sr1 is a global
+        self.checkflags(sr2_adr, True, False)     # sr2 is a global
         self.gc.leave_transactional_mode()
-        self.checkflags(tr2_adr, True, False)     # tr2 is still global
-        self.checkflags(tr1_adr, False, False)    # tr1 is back to a local
+        self.checkflags(sr2_adr, True, False)     # sr2 is still global
+        self.checkflags(sr1_adr, False, False)    # sr1 is back to a local
 
     def test_collect_from_main_thread_was_global_objects(self):
         tr1, tr1_adr = self.malloc(SR, globl=True)  # a global prebuilt object

From noreply at buildbot.pypy.org  Tue Apr 24 12:34:17 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Apr 2012 12:34:17 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add a passing test.
Message-ID: <20120424103417.8D16882009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54716:9ac765b1a237
Date: 2012-04-24 11:05 +0200
http://bitbucket.org/pypy/pypy/changeset/9ac765b1a237/

Log:	Add a passing test.

diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -338,6 +338,97 @@
         assert obj == t_adr                       # doesn't make a copy
         self.checkflags(obj, False, False)        # but it becomes local
 
+    def test_nontransactional_mode(self):
+        import random
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        #
+        def reachable(source_objects):
+            pending = list(source_objects)
+            found = set(obj._obj for obj in pending)
+            for x in pending:
+                for name in ('sr2', 'sr3'):
+                    obj = getattr(x, name)
+                    if obj and obj._obj not in found:
+                        found.add(obj._obj)
+                        pending.append(obj)
+            return found
+        #
+        def shape_of_reachable(source_object):
+            shape = []
+            pending = [source_object]
+            found = {source_object._obj: 0}
+            for x in pending:
+                for name in ('sr2', 'sr3'):
+                    obj = getattr(x, name)
+                    if not obj:
+                        shape.append(None)
+                    else:
+                        if obj._obj not in found:
+                            found[obj._obj] = len(found)
+                            pending.append(obj)
+                        shape.append(found[obj._obj])
+            return shape
+        #
+        prebuilt = [self.malloc(SR, globl=True)[0] for i in range(15)]
+        globals = set(obj._obj for obj in prebuilt)
+        root_objects = prebuilt[:]
+        all_objects = root_objects[:]
+        NO_OBJECT = lltype.nullptr(SR)
+        #
+        for iteration in range(10):
+            # add 6 freshly malloced objects from the nursery
+            new_objects = [self.malloc(SR, globl=False)[0] for i in range(6)]
+            all_objects = all_objects + new_objects
+            set_all_objects = set(obj._obj for obj in all_objects)
+            #
+            # pick 4 random objects to be stack roots
+            fromstack = random.sample(all_objects, 4)
+            root_objects = prebuilt + fromstack
+            #
+            # randomly add or remove connections between objects, until they
+            # are all reachable from root_objects
+            while True:
+                missing_objects = set_all_objects - reachable(root_objects)
+                if not missing_objects:
+                    break
+                srcobj = random.choice(all_objects)
+                # give a higher chance to 'missing_objects', but also
+                # allows other objects
+                missing_objects = [obj._as_ptr() for obj in missing_objects]
+                missing_objects.append(NO_OBJECT)
+                missing_objects *= 2
+                missing_objects.extend(all_objects)
+                dstobj = random.choice(missing_objects)
+                name = random.choice(('sr2', 'sr3'))
+                src_adr = llmemory.cast_ptr_to_adr(srcobj)
+                adr2 = self.gc.stm_writebarrier(src_adr)
+                assert adr2 == src_adr
+                setattr(srcobj, name, dstobj)
+                if srcobj._obj in globals:
+                    globals.remove(srcobj._obj)
+            #
+            # Record the shape of the graph of reachable objects
+            shapes = [shape_of_reachable(obj) for obj in root_objects]
+            #
+            # Do a collection
+            self.gc.root_walker.current_stack = fromstack[:]
+            self.gc.collect(0)
+            #
+            # Reload 'fromstack', which may have moved, and compare the shape
+            # of the graph of reachable objects now
+            fromstack[:] = self.gc.root_walker.current_stack
+            root_objects = prebuilt + fromstack
+            shapes2 = [shape_of_reachable(obj) for obj in root_objects]
+            assert shapes == shapes2
+            #
+            # Reset the list of all objects for the next iteration
+            all_objects = [obj._as_ptr() for obj in reachable(root_objects)]
+            #
+            # Check the GLOBAL flag, and check that the objects really survived
+            for obj in all_objects:
+                self.checkflags(obj, obj._obj in globals, False)
+
     def test_relocalize_objects_after_transactional_mode(self):
         from pypy.rpython.memory.gc.test import test_stmtls
         self.gc.root_walker = test_stmtls.FakeRootWalker()

From noreply at buildbot.pypy.org  Tue Apr 24 13:54:23 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 24 Apr 2012 13:54:23 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: this test is a
	whitebox one
Message-ID: <20120424115423.669FB82009@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54717:69db8b662f4d
Date: 2012-04-24 13:54 +0200
http://bitbucket.org/pypy/pypy/changeset/69db8b662f4d/

Log:	this test is a whitebox one

diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -202,6 +202,7 @@
         res = self.interpret(f, [0])
         assert res == 14.5
 
+class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
     def test_enumerate_elements(self):
         def f():
             storage = rerased_raw.UntypedStorage("sibf")
@@ -218,6 +219,3 @@
         assert lst[1:3] == [(1, 13), (2, True)]
         assert lst[3][0] == 3
         assert longlong2float.longlong2float(lst[3][1]) == 3.5
-
-class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
-    pass

From noreply at buildbot.pypy.org  Tue Apr 24 14:52:52 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 24 Apr 2012 14:52:52 +0200 (CEST)
Subject: [pypy-commit] pypy faster-heapcache: close to-be-merged branch
Message-ID: <20120424125252.5F26C82009@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: faster-heapcache
Changeset: r54718:eda61a488279
Date: 2012-04-24 14:52 +0200
http://bitbucket.org/pypy/pypy/changeset/eda61a488279/

Log:	close to-be-merged branch


From noreply at buildbot.pypy.org  Tue Apr 24 14:52:53 2012
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 24 Apr 2012 14:52:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge faster-heapcache
Message-ID: <20120424125253.C991F82009@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r54719:1ecd25d20dab
Date: 2012-04-24 14:52 +0200
http://bitbucket.org/pypy/pypy/changeset/1ecd25d20dab/

Log:	merge faster-heapcache

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -20,6 +20,7 @@
         self.dependencies = {}
         # contains frame boxes that are not virtualizables
         self.nonstandard_virtualizables = {}
+
         # heap cache
         # maps descrs to {from_box, to_box} dicts
         self.heap_cache = {}
@@ -29,6 +30,26 @@
         # cache the length of arrays
         self.length_cache = {}
 
+        # replace_box is called surprisingly often, therefore it's not efficient
+        # to go over all the dicts and fix them.
+        # instead, these two dicts are kept, and a replace_box adds an entry to
+        # each of them.
+        # every time one of the dicts heap_cache, heap_array_cache, length_cache
+        # is accessed, suitable indirections need to be performed
+
+        # this looks all very subtle, but in practice the patterns of
+        # replacements should not be that complex. Usually a box is replaced by
+        # a const, once. Also, if something goes wrong, the effect is that less
+        # caching than possible is done, which is not a huge problem.
+        self.input_indirections = {}
+        self.output_indirections = {}
+
+    def _input_indirection(self, box):
+        return self.input_indirections.get(box, box)
+
+    def _output_indirection(self, box):
+        return self.output_indirections.get(box, box)
+
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, argboxes)
         self.clear_caches(opnum, descr, argboxes)
@@ -132,14 +153,16 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
+        box = self._input_indirection(box)
         d = self.heap_cache.get(descr, None)
         if d:
             tobox = d.get(box, None)
-            if tobox:
-                return tobox
+            return self._output_indirection(tobox)
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         self.heap_cache.setdefault(descr, {})[box] = fieldbox
 
     def setfield(self, box, descr, fieldbox):
@@ -148,6 +171,8 @@
         self.heap_cache[descr] = new_d
 
     def _do_write_with_aliasing(self, d, box, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         # slightly subtle logic here
         # a write to an arbitrary box, all other boxes can alias this one
         if not d or box not in self.new_boxes:
@@ -166,6 +191,7 @@
         return new_d
 
     def getarrayitem(self, box, descr, indexbox):
+        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -173,9 +199,11 @@
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return indexcache.get(box, None)
+                return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        box = self._input_indirection(box)
+        valuebox = self._input_indirection(valuebox)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -198,25 +226,13 @@
         cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
 
     def arraylen(self, box):
-        return self.length_cache.get(box, None)
+        box = self._input_indirection(box)
+        return self._output_indirection(self.length_cache.get(box, None))
 
     def arraylen_now_known(self, box, lengthbox):
-        self.length_cache[box] = lengthbox
-
-    def _replace_box(self, d, oldbox, newbox):
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            if frombox is oldbox:
-                frombox = newbox
-            if tobox is oldbox:
-                tobox = newbox
-            new_d[frombox] = tobox
-        return new_d
+        box = self._input_indirection(box)
+        self.length_cache[box] = self._input_indirection(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        for descr, d in self.heap_cache.iteritems():
-            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
-        for descr, d in self.heap_array_cache.iteritems():
-            for index, cache in d.iteritems():
-                d[index] = self._replace_box(cache, oldbox, newbox)
-        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
+        self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox)
+        self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox)
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -2,12 +2,14 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp.history import ConstInt
 
-box1 = object()
-box2 = object()
-box3 = object()
-box4 = object()
+box1 = "box1"
+box2 = "box2"
+box3 = "box3"
+box4 = "box4"
+box5 = "box5"
 lengthbox1 = object()
 lengthbox2 = object()
+lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -276,11 +278,43 @@
         h.setfield(box1, descr2, box3)
         h.setfield(box2, descr3, box3)
         h.replace_box(box1, box4)
-        assert h.getfield(box1, descr1) is None
-        assert h.getfield(box1, descr2) is None
         assert h.getfield(box4, descr1) is box2
         assert h.getfield(box4, descr2) is box3
         assert h.getfield(box2, descr3) is box3
+        h.setfield(box4, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box4
+        assert h.getfield(box2, descr3) is box4
+
+    def test_replace_box_twice(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box5, descr1) is box2
+        assert h.getfield(box5, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+        h.setfield(box5, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box5
+        assert h.getfield(box2, descr3) is box5
 
     def test_replace_box_array(self):
         h = HeapCache()
@@ -291,9 +325,6 @@
         h.setarrayitem(box3, descr2, index2, box1)
         h.setarrayitem(box2, descr3, index2, box3)
         h.replace_box(box1, box4)
-        assert h.getarrayitem(box1, descr1, index1) is None
-        assert h.getarrayitem(box1, descr2, index1) is None
-        assert h.arraylen(box1) is None
         assert h.arraylen(box4) is lengthbox1
         assert h.getarrayitem(box4, descr1, index1) is box2
         assert h.getarrayitem(box4, descr2, index1) is box3
@@ -304,6 +335,27 @@
         h.replace_box(lengthbox1, lengthbox2)
         assert h.arraylen(box4) is lengthbox2
 
+    def test_replace_box_array_twice(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box5, descr1, index1) is box2
+        assert h.getarrayitem(box5, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box5
+        assert h.getarrayitem(box3, descr2, index2) is box5
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        h.replace_box(lengthbox2, lengthbox3)
+        assert h.arraylen(box4) is lengthbox3
+
     def test_ll_arraycopy(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)

From noreply at buildbot.pypy.org  Tue Apr 24 15:54:47 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Apr 2012 15:54:47 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: More passing tests.
Message-ID: <20120424135447.D3AF88208A@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54721:e7ebebda0c37
Date: 2012-04-24 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/e7ebebda0c37/

Log:	More passing tests.

diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -776,6 +776,40 @@
         assert s2.a == 4242
         assert s2 == tr1.s1   # tr1 is a root, so not copied yet
 
+    def test_weakref_to_local_in_main_thread(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        #
+        sr1, sr1_adr = self.malloc(SR, globl=False)
+        wr1, wr1_adr = self.malloc(WR, globl=False, weakref=True)
+        wr1.wadr = sr1_adr
+        self.gc.root_walker.current_stack = [wr1]
+        self.gc.collect(0)
+        #
+        [wr1] = self.gc.root_walker.current_stack
+        assert not wr1.wadr        # weakref to dead object
+
+    def test_weakref_to_global_turned_local(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        #
+        sr1, sr1_adr = self.malloc(SR, globl=True)
+        sr2, sr2_adr = self.malloc(SR, globl=True)
+        self.gc.stm_writebarrier(sr1_adr)
+        self.gc.stm_writebarrier(sr2_adr)
+        self.checkflags(sr1_adr, False, False)    # turned local
+        self.checkflags(sr2_adr, False, False)    # turned local
+        wr1, wr1_adr = self.malloc(WR, globl=False, weakref=True)
+        wr2, wr2_adr = self.malloc(WR, globl=False, weakref=True)
+        wr1.wadr = sr1_adr
+        wr2.wadr = sr2_adr
+        self.gc.root_walker.current_stack = [wr1, wr2]
+        self.gc.collect(0)
+        #
+        [wr1, wr2] = self.gc.root_walker.current_stack
+        assert wr1.wadr == sr1_adr
+        assert wr2.wadr == sr2_adr
+
     def test_normalize_global_null(self):
         a = self.gc.stm_normalize_global(llmemory.NULL)
         assert a == llmemory.NULL

From noreply at buildbot.pypy.org  Tue Apr 24 15:54:46 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Apr 2012 15:54:46 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Improve testing before translation.
Message-ID: <20120424135446.9D4CB82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54720:93a836c8b6b1
Date: 2012-04-24 15:21 +0200
http://bitbucket.org/pypy/pypy/changeset/93a836c8b6b1/

Log:	Improve testing before translation.

diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1198,6 +1198,15 @@
         raise AttributeError("%r instance has no field %r" % (self._T,
                                                               field_name))
 
+    def __delattr__(self, field_name):
+        if isinstance(self._T, Struct):
+            if field_name in self._T._flds:
+                T1 = self._T._flds[field_name]
+                setattr(self._obj, field_name, _uninitialized(T1))
+                return
+        raise AttributeError("cannot delete %r on %r instance" % (field_name,
+                                                                  self._T))
+
     def __getitem__(self, i): # ! can only return basic or ptr !
         if isinstance(self._T, (Array, FixedSizeArray)):
             start, stop = self._obj.getbounds()
diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py
--- a/pypy/rpython/lltypesystem/test/test_lltype.py
+++ b/pypy/rpython/lltypesystem/test/test_lltype.py
@@ -862,6 +862,15 @@
     assert F.RESULT == Signed
     assert F.ARGS == (Signed,)
 
+def test_make_field_uninitialized_again():
+    S = lltype.GcStruct('S', ('x', lltype.Signed))
+    s = lltype.malloc(S)
+    py.test.raises(UninitializedMemoryAccess, "s.x")
+    s.x = 42
+    assert s.x == 42
+    del s.x    # not RPython
+    py.test.raises(UninitializedMemoryAccess, "s.x")
+
 
 class TestTrackAllocation:
     def test_automatic_tracking(self):
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -118,8 +118,8 @@
             obj = hdr.version
             ll_assert(hdr.tid & GCFLAG_GLOBAL == 0, "already GLOBAL [2]")
             ll_assert(hdr.tid & GCFLAG_VISITED != 0, "missing VISITED [2]")
-            hdr.version = NULL
             hdr.tid += GCFLAG_GLOBAL - GCFLAG_VISITED
+            self._clear_version_for_global_object(hdr)
         if not we_are_translated():
             del self.mt_global_turned_local   # don't use any more
 
@@ -320,8 +320,18 @@
             obj = hdr.version
             ll_assert(hdr.tid & GCFLAG_GLOBAL == 0, "already GLOBAL [1]")
             ll_assert(hdr.tid & GCFLAG_VISITED == 0, "unexpected VISITED [1]")
+            hdr.tid |= GCFLAG_GLOBAL
+            self._clear_version_for_global_object(hdr)
+
+    def _clear_version_for_global_object(self, hdr):
+        # Reset the 'version' to initialize a newly global object.
+        # When translated with C code, we set it to NULL (version 0).
+        # When non-translated, we reset it instead to '_uninitialized'
+        # to simulate the fact that the C code might change it.
+        if we_are_translated():
             hdr.version = NULL
-            hdr.tid |= GCFLAG_GLOBAL
+        else:
+            del hdr.version
 
     def _cleanup_state(self):
         #if self.rawmalloced_objects:

From noreply at buildbot.pypy.org  Tue Apr 24 15:54:49 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Apr 2012 15:54:49 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Improve the tests.
Message-ID: <20120424135449.215F882009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54722:9cb2b55a043e
Date: 2012-04-24 15:50 +0200
http://bitbucket.org/pypy/pypy/changeset/9cb2b55a043e/

Log:	Improve the tests.

diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -783,11 +783,15 @@
         sr1, sr1_adr = self.malloc(SR, globl=False)
         wr1, wr1_adr = self.malloc(WR, globl=False, weakref=True)
         wr1.wadr = sr1_adr
+        #
         self.gc.root_walker.current_stack = [wr1]
         self.gc.collect(0)
-        #
         [wr1] = self.gc.root_walker.current_stack
         assert not wr1.wadr        # weakref to dead object
+        #
+        self.gc.collect(0)
+        assert self.gc.root_walker.current_stack == [wr1]
+        assert not wr1.wadr
 
     def test_weakref_to_global_turned_local(self):
         from pypy.rpython.memory.gc.test import test_stmtls
@@ -803,10 +807,15 @@
         wr2, wr2_adr = self.malloc(WR, globl=False, weakref=True)
         wr1.wadr = sr1_adr
         wr2.wadr = sr2_adr
+        #
         self.gc.root_walker.current_stack = [wr1, wr2]
         self.gc.collect(0)
+        [wr1, wr2] = self.gc.root_walker.current_stack
+        assert wr1.wadr == sr1_adr
+        assert wr2.wadr == sr2_adr
         #
-        [wr1, wr2] = self.gc.root_walker.current_stack
+        self.gc.collect(0)
+        assert self.gc.root_walker.current_stack == [wr1, wr2]
         assert wr1.wadr == sr1_adr
         assert wr2.wadr == sr2_adr
 

From noreply at buildbot.pypy.org  Tue Apr 24 16:11:33 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 16:11:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: Failing test for an elidable function
 with arguments that become pure as a result of the vstring optimization.
Message-ID: <20120424141133.E115C82009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r54723:dd34d145d841
Date: 2012-04-24 10:11 -0400
http://bitbucket.org/pypy/pypy/changeset/dd34d145d841/

Log:	Failing test for an elidable function with arguments that become
	pure as a result of the vstring optimization.

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3022,6 +3022,27 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
 
+    def test_elidable_vstring_const(self):
+        myjitdriver = JitDriver(greens = ["n"], reds = ["i", "c"])
+        @elidable
+        def g(s):
+            return s + "hello"
+        @unroll_safe
+        def make_str(n):
+            return "".join(["a" for _ in range(n)])
+        def f(n):
+            i = 0
+            c = 0
+            while i < n:
+                myjitdriver.jit_merge_point(n=n, i=i, c=c)
+                s = make_str(n // 5)
+                c += len(g(s))
+                i += 1
+            return c
+        res = self.meta_interp(f, [10])
+        assert res == 70
+        self.check_resops({"int_lt": 2, "int_add": 4, "guard_true": 2, "jump": 1})
+
 
 class TestOOtype(BasicTests, OOJitMixin):
 

From noreply at buildbot.pypy.org  Tue Apr 24 16:33:49 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 16:33:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: convert this test to optimizeopt
Message-ID: <20120424143349.DC93B82009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r54724:7e5daf087990
Date: 2012-04-24 10:33 -0400
http://bitbucket.org/pypy/pypy/changeset/7e5daf087990/

Log:	convert this test to optimizeopt

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -7,7 +7,7 @@
 import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt
 import pypy.jit.metainterp.optimizeopt.virtualize as virtualize
 from pypy.jit.metainterp.optimize import InvalidLoop
-from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
+from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, get_const_ptr_for_string
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -5067,6 +5067,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_call_pure_vstring_const(self):
+        ops = """
+        []
+        p0 = newstr(3)
+        strsetitem(p0, 0, 97)
+        strsetitem(p0, 1, 98)
+        strsetitem(p0, 2, 99)
+        i0 = call_pure(123, p0, descr=nonwritedescr)
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(5)
+        """
+        call_pure_results = {
+            (ConstInt(123), get_const_ptr_for_string("abc"),): ConstInt(5),
+        }
+        self.optimize_loop(ops, expected, call_pure_results)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3022,27 +3022,6 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
 
-    def test_elidable_vstring_const(self):
-        myjitdriver = JitDriver(greens = ["n"], reds = ["i", "c"])
-        @elidable
-        def g(s):
-            return s + "hello"
-        @unroll_safe
-        def make_str(n):
-            return "".join(["a" for _ in range(n)])
-        def f(n):
-            i = 0
-            c = 0
-            while i < n:
-                myjitdriver.jit_merge_point(n=n, i=i, c=c)
-                s = make_str(n // 5)
-                c += len(g(s))
-                i += 1
-            return c
-        res = self.meta_interp(f, [10])
-        assert res == 70
-        self.check_resops({"int_lt": 2, "int_add": 4, "guard_true": 2, "jump": 1})
-
 
 class TestOOtype(BasicTests, OOJitMixin):
 

From noreply at buildbot.pypy.org  Tue Apr 24 16:47:47 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 16:47:47 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Use subfic instruction for
	CONST - REG.
Message-ID: <20120424144747.3B61882009@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r54725:732d857aeb12
Date: 2012-04-24 10:47 -0400
http://bitbucket.org/pypy/pypy/changeset/732d857aeb12/

Log:	Use subfic instruction for CONST - REG.

diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -53,8 +53,7 @@
     def emit_int_sub(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         if l0.is_imm():
-            self.mc.load_imm(r.r0, l0.value)
-            self.mc.sub(res.value, r.r0.value, l1.value)
+            self.mc.subfic(res.value, l1.value, l0.value)
         elif l1.is_imm():
             self.mc.subi(res.value, l0.value, l1.value)
         else:

From noreply at buildbot.pypy.org  Tue Apr 24 18:07:37 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 18:07:37 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: for now, promote this.
Message-ID: <20120424160737.C24D182009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54726:47f74017d7e0
Date: 2012-04-24 17:45 +0200
http://bitbucket.org/pypy/pypy/changeset/47f74017d7e0/

Log:	for now, promote this.

diff --git a/pypy/objspace/std/tupletype.py b/pypy/objspace/std/tupletype.py
--- a/pypy/objspace/std/tupletype.py
+++ b/pypy/objspace/std/tupletype.py
@@ -101,6 +101,7 @@
         shape_chars[i] = get_char_from_obj(space, w_item)
 
     shape = space.str_w(space.new_interned_str("".join(shape_chars)))
+    jit.promote(shape)
     storage = UntypedStorage(shape)
     for i, w_item in enumerate(list_w):
         store_obj(space, storage, shape[i], i, w_item)

From noreply at buildbot.pypy.org  Tue Apr 24 18:07:39 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 18:07:39 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: for now, promote this
Message-ID: <20120424160739.04A7182009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54727:af7067c9c2a3
Date: 2012-04-24 18:02 +0200
http://bitbucket.org/pypy/pypy/changeset/af7067c9c2a3/

Log:	for now, promote this

diff --git a/pypy/objspace/std/tupletype.py b/pypy/objspace/std/tupletype.py
--- a/pypy/objspace/std/tupletype.py
+++ b/pypy/objspace/std/tupletype.py
@@ -100,8 +100,9 @@
     for i, w_item in enumerate(list_w):
         shape_chars[i] = get_char_from_obj(space, w_item)
 
-    shape = space.str_w(space.new_interned_str("".join(shape_chars)))
-    jit.promote(shape)
+    w_shape = space.new_interned_str("".join(shape_chars))
+    jit.promote(w_shape)
+    shape = space.str_w(w_shape)
     storage = UntypedStorage(shape)
     for i, w_item in enumerate(list_w):
         store_obj(space, storage, shape[i], i, w_item)

From noreply at buildbot.pypy.org  Tue Apr 24 18:07:40 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 18:07:40 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: merged upstream
Message-ID: <20120424160740.3A88F82009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54728:d092a8be277a
Date: 2012-04-24 18:04 +0200
http://bitbucket.org/pypy/pypy/changeset/d092a8be277a/

Log:	merged upstream

diff --git a/pypy/rlib/test/test_rerased_raw.py b/pypy/rlib/test/test_rerased_raw.py
--- a/pypy/rlib/test/test_rerased_raw.py
+++ b/pypy/rlib/test/test_rerased_raw.py
@@ -202,6 +202,7 @@
         res = self.interpret(f, [0])
         assert res == 14.5
 
+class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
     def test_enumerate_elements(self):
         def f():
             storage = rerased_raw.UntypedStorage("sibf")
@@ -218,6 +219,3 @@
         assert lst[1:3] == [(1, 13), (2, True)]
         assert lst[3][0] == 3
         assert longlong2float.longlong2float(lst[3][1]) == 3.5
-
-class TestUntypedStorageLLtype(LLRtypeMixin, BaseTestUntypedStorage):
-    pass

From noreply at buildbot.pypy.org  Tue Apr 24 19:23:20 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 24 Apr 2012 19:23:20 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fixes.
Message-ID: <20120424172320.0D2FF82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54729:a3c0cada9e2c
Date: 2012-04-24 17:44 +0200
http://bitbucket.org/pypy/pypy/changeset/a3c0cada9e2c/

Log:	Fixes.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -116,14 +116,20 @@
         NOTE: never register() the same instance multiple times.
         """
         ec = self.state.getvalue()
+        assert ec is not None       # must have been created first
         ec._transaction_pending.append(self)
 
     def run(self):
+        ec = self.space.getexecutioncontext()    # create it if needed
+        #
         if self.retry_counter > 0:
+            # Note that even in this case we have to ensure the ec is
+            # created first.  Otherwise, if the first transaction of a
+            # thread is retrying, the creation of the ec is cancelled
+            # and we end up in register() with no ec...
             self.register() # retrying: will be done later, try others first
             return
         #
-        ec = self.space.getexecutioncontext()    # create it if needed
         assert len(ec._transaction_pending) == 0
         #
         self.space.call_args(self.w_callback, self.args)
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -297,9 +297,9 @@
 
     def main_thread_writes_to_global_obj(self, obj):
         hdr = self.gc.header(obj)
-        ll_assert(hdr.tid & (GCFLAG_WAS_COPIED|GCFLAG_VISITED) == 0,
-                  "write in main thread: unexpected GCFLAG_WAS_COPIED"
-                  " or GCFLAG_VISITED")
+        # XXX should we also remove GCFLAG_WAS_COPIED here if it is set?
+        ll_assert(hdr.tid & GCFLAG_VISITED == 0,
+                  "write in main thread: unexpected GCFLAG_VISITED")
         # remove GCFLAG_GLOBAL, and add GCFLAG_VISITED
         hdr.tid += (GCFLAG_VISITED - GCFLAG_GLOBAL)
         # add the object into this linked list

From noreply at buildbot.pypy.org  Tue Apr 24 19:39:02 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 19:39:02 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: include the fields in
 forced stuff on an array
Message-ID: <20120424173902.8831882009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54730:4a8980d38df8
Date: 2012-04-24 13:38 -0400
http://bitbucket.org/pypy/pypy/changeset/4a8980d38df8/

Log:	include the fields in forced stuff on an array

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5081,6 +5081,25 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_forced_gcstruct_as_array(self):
+        ops = """
+        [i0]
+        p0 = new_array(3, descr=gcstructarraydescr)
+        setfield_gc(p0, i0, descr=gcstructarray_fielddescr)
+        escape(p0)
+        i1 = getfield_gc(p0, descr=gcstructarray_fielddescr)
+        finish(i1)
+        """
+        expected = """
+        [i0]
+        p0 = new_array(3, descr=gcstructarraydescr)
+        setfield_gc(p0, i0, descr=gcstructarray_fielddescr)
+        escape(p0)
+        i1 = getfield_gc(p0, descr=gcstructarray_fielddescr)
+        finish(i1)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -278,6 +278,14 @@
             self.source_op.name = 'FORCE ' + self.source_op.name
         optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
+        if self._fields:
+            for ofs, value in self._fields.iteritems():
+                if value.is_null():
+                    continue
+                subbox = value.force_box(optforce)
+                op = ResOperation(rop.SETFIELD_GC, [box, subbox], None, descr=ofs)
+                optforce.emit_operation(op)
+
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:

From noreply at buildbot.pypy.org  Tue Apr 24 19:42:18 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 24 Apr 2012 19:42:18 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: merged default in
Message-ID: <20120424174218.B5F0B82009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54731:4f3c2643ce7a
Date: 2012-04-24 13:42 -0400
http://bitbucket.org/pypy/pypy/changeset/4f3c2643ce7a/

Log:	merged default in

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -20,6 +20,7 @@
         self.dependencies = {}
         # contains frame boxes that are not virtualizables
         self.nonstandard_virtualizables = {}
+
         # heap cache
         # maps descrs to {from_box, to_box} dicts
         self.heap_cache = {}
@@ -29,6 +30,26 @@
         # cache the length of arrays
         self.length_cache = {}
 
+        # replace_box is called surprisingly often, therefore it's not efficient
+        # to go over all the dicts and fix them.
+        # instead, these two dicts are kept, and a replace_box adds an entry to
+        # each of them.
+        # every time one of the dicts heap_cache, heap_array_cache, length_cache
+        # is accessed, suitable indirections need to be performed
+
+        # this looks all very subtle, but in practice the patterns of
+        # replacements should not be that complex. Usually a box is replaced by
+        # a const, once. Also, if something goes wrong, the effect is that less
+        # caching than possible is done, which is not a huge problem.
+        self.input_indirections = {}
+        self.output_indirections = {}
+
+    def _input_indirection(self, box):
+        return self.input_indirections.get(box, box)
+
+    def _output_indirection(self, box):
+        return self.output_indirections.get(box, box)
+
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, argboxes)
         self.clear_caches(opnum, descr, argboxes)
@@ -132,14 +153,16 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
+        box = self._input_indirection(box)
         d = self.heap_cache.get(descr, None)
         if d:
             tobox = d.get(box, None)
-            if tobox:
-                return tobox
+            return self._output_indirection(tobox)
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         self.heap_cache.setdefault(descr, {})[box] = fieldbox
 
     def setfield(self, box, descr, fieldbox):
@@ -148,6 +171,8 @@
         self.heap_cache[descr] = new_d
 
     def _do_write_with_aliasing(self, d, box, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         # slightly subtle logic here
         # a write to an arbitrary box, all other boxes can alias this one
         if not d or box not in self.new_boxes:
@@ -166,6 +191,7 @@
         return new_d
 
     def getarrayitem(self, box, descr, indexbox):
+        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -173,9 +199,11 @@
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return indexcache.get(box, None)
+                return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        box = self._input_indirection(box)
+        valuebox = self._input_indirection(valuebox)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -198,25 +226,13 @@
         cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
 
     def arraylen(self, box):
-        return self.length_cache.get(box, None)
+        box = self._input_indirection(box)
+        return self._output_indirection(self.length_cache.get(box, None))
 
     def arraylen_now_known(self, box, lengthbox):
-        self.length_cache[box] = lengthbox
-
-    def _replace_box(self, d, oldbox, newbox):
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            if frombox is oldbox:
-                frombox = newbox
-            if tobox is oldbox:
-                tobox = newbox
-            new_d[frombox] = tobox
-        return new_d
+        box = self._input_indirection(box)
+        self.length_cache[box] = self._input_indirection(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        for descr, d in self.heap_cache.iteritems():
-            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
-        for descr, d in self.heap_array_cache.iteritems():
-            for index, cache in d.iteritems():
-                d[index] = self._replace_box(cache, oldbox, newbox)
-        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
+        self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox)
+        self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -7,7 +7,7 @@
 import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt
 import pypy.jit.metainterp.optimizeopt.virtualize as virtualize
 from pypy.jit.metainterp.optimize import InvalidLoop
-from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
+from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, get_const_ptr_for_string
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -5067,6 +5067,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_call_pure_vstring_const(self):
+        ops = """
+        []
+        p0 = newstr(3)
+        strsetitem(p0, 0, 97)
+        strsetitem(p0, 1, 98)
+        strsetitem(p0, 2, 99)
+        i0 = call_pure(123, p0, descr=nonwritedescr)
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(5)
+        """
+        call_pure_results = {
+            (ConstInt(123), get_const_ptr_for_string("abc"),): ConstInt(5),
+        }
+        self.optimize_loop(ops, expected, call_pure_results)
+
     def test_gcstruct_as_array(self):
         ops = """
         [i0]
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -2,12 +2,14 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp.history import ConstInt
 
-box1 = object()
-box2 = object()
-box3 = object()
-box4 = object()
+box1 = "box1"
+box2 = "box2"
+box3 = "box3"
+box4 = "box4"
+box5 = "box5"
 lengthbox1 = object()
 lengthbox2 = object()
+lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -276,11 +278,43 @@
         h.setfield(box1, descr2, box3)
         h.setfield(box2, descr3, box3)
         h.replace_box(box1, box4)
-        assert h.getfield(box1, descr1) is None
-        assert h.getfield(box1, descr2) is None
         assert h.getfield(box4, descr1) is box2
         assert h.getfield(box4, descr2) is box3
         assert h.getfield(box2, descr3) is box3
+        h.setfield(box4, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box4
+        assert h.getfield(box2, descr3) is box4
+
+    def test_replace_box_twice(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box5, descr1) is box2
+        assert h.getfield(box5, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+        h.setfield(box5, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box5
+        assert h.getfield(box2, descr3) is box5
 
     def test_replace_box_array(self):
         h = HeapCache()
@@ -291,9 +325,6 @@
         h.setarrayitem(box3, descr2, index2, box1)
         h.setarrayitem(box2, descr3, index2, box3)
         h.replace_box(box1, box4)
-        assert h.getarrayitem(box1, descr1, index1) is None
-        assert h.getarrayitem(box1, descr2, index1) is None
-        assert h.arraylen(box1) is None
         assert h.arraylen(box4) is lengthbox1
         assert h.getarrayitem(box4, descr1, index1) is box2
         assert h.getarrayitem(box4, descr2, index1) is box3
@@ -304,6 +335,27 @@
         h.replace_box(lengthbox1, lengthbox2)
         assert h.arraylen(box4) is lengthbox2
 
+    def test_replace_box_array_twice(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box5, descr1, index1) is box2
+        assert h.getarrayitem(box5, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box5
+        assert h.getarrayitem(box3, descr2, index2) is box5
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        h.replace_box(lengthbox2, lengthbox3)
+        assert h.arraylen(box4) is lengthbox3
+
     def test_ll_arraycopy(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -38,10 +38,19 @@
 	PyObject_VAR_HEAD
 } PyVarObject;
 
+#ifndef PYPY_DEBUG_REFCOUNT
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)ob))
 #define Py_DECREF(ob)   (Py_DecRef((PyObject *)ob))
 #define Py_XINCREF(ob)  (Py_IncRef((PyObject *)ob))
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)ob))
+#else
+#define Py_INCREF(ob)   (((PyObject *)ob)->ob_refcnt++)
+#define Py_DECREF(ob)  ((((PyObject *)ob)->ob_refcnt > 1) ? \
+			((PyObject *)ob)->ob_refcnt-- : (Py_DecRef((PyObject *)ob)))
+
+#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
+#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
+#endif
 
 #define Py_CLEAR(op)				\
         do {                            	\
diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -1234,7 +1234,7 @@
             pos += 1
             continue
 
-        if 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
             # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
             pos += 1
             oc2 = ord(s[pos])
@@ -1350,6 +1350,20 @@
     pos = 0
     while pos < size:
         oc = ord(s[pos])
+
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+            # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
+            pos += 1
+            oc2 = ord(s[pos])
+
+            if 0xDC00 <= oc2 <= 0xDFFF:
+                ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000
+                raw_unicode_escape_helper(result, ucs)
+                pos += 1
+                continue
+            # Fall through: isolated surrogates are copied as-is
+            pos -= 1
+
         if oc < 0x100:
             result.append(chr(oc))
         else:
diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py
--- a/pypy/rlib/test/test_runicode.py
+++ b/pypy/rlib/test/test_runicode.py
@@ -728,3 +728,18 @@
 
         res = interpret(f, [0x10140])
         assert res == 0x10140
+
+    def test_encode_surrogate_pair(self):
+        u = runicode.UNICHR(0xD800) + runicode.UNICHR(0xDC00)
+        if runicode.MAXUNICODE < 65536:
+            # Narrow unicode build, consider utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+        else:
+            # Wide unicode build, don't merge utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'

From noreply at buildbot.pypy.org  Tue Apr 24 19:52:06 2012
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 24 Apr 2012 19:52:06 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-backend-2: Add mffgpr and mftgpr
	instructions. Use them.
Message-ID: <20120424175206.8B43B82009@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-backend-2
Changeset: r54732:cc4776340572
Date: 2012-04-24 13:51 -0400
http://bitbucket.org/pypy/pypy/changeset/cc4776340572/

Log:	Add mffgpr and mftgpr instructions. Use them. Use subfic.

diff --git a/pypy/jit/backend/ppc/codebuilder.py b/pypy/jit/backend/ppc/codebuilder.py
--- a/pypy/jit/backend/ppc/codebuilder.py
+++ b/pypy/jit/backend/ppc/codebuilder.py
@@ -225,6 +225,9 @@
 
     fsqrt = XDB(63, XO1=22, Rc=0)
 
+    mffgpr = XS(31, XO1=607, Rc=0)
+    mftgpr = XS(31, XO1=735, Rc=0)
+
     icbi = X0(31, XO1=982)
 
     lbzux = XD(31, XO1=119)
diff --git a/pypy/jit/backend/ppc/opassembler.py b/pypy/jit/backend/ppc/opassembler.py
--- a/pypy/jit/backend/ppc/opassembler.py
+++ b/pypy/jit/backend/ppc/opassembler.py
@@ -53,8 +53,7 @@
     def emit_int_sub(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         if l0.is_imm():
-            self.mc.load_imm(r.r0, l0.value)
-            self.mc.sub(res.value, r.r0.value, l1.value)
+            self.mc.subfic(res.value, l1.value, l0.value)
         elif l1.is_imm():
             self.mc.subi(res.value, l0.value, l1.value)
         else:
@@ -215,11 +214,11 @@
     def emit_op_cast_float_to_int(self, op, arglocs, regalloc):
         l0, temp_loc, res = arglocs
         self.mc.fctidz(temp_loc.value, l0.value)
-        self.mc.mfprgpr(res.value, temp_loc.value)
+        self.mc.mftgpr(res.value, temp_loc.value)
 
     def emit_op_cast_int_to_float(self, op, arglocs, regalloc):
         l0, temp_loc, res = arglocs
-        self.mc.mgprfpr(temp_loc.value, l0.value)
+        self.mc.mffgpr(temp_loc.value, l0.value)
         self.mc.fcfid(res.value, temp_loc.value)
 
 class GuardOpAssembler(object):

From noreply at buildbot.pypy.org  Tue Apr 24 20:44:41 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue, 24 Apr 2012 20:44:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: more advanced cppyy exampler: shows the
 use of a selection file, and demonstrates auto-casting
Message-ID: <20120424184441.C4CEE82009@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r54733:e286c17b3167
Date: 2012-04-24 11:44 -0700
http://bitbucket.org/pypy/pypy/changeset/e286c17b3167/

Log:	more advanced cppyy exampler: shows the use of a selection file, and
	demonstrates auto-casting

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -111,6 +111,121 @@
 That's all there is to it!
 
 
+Advanced example
+================
+The following snippet of C++ is very contrived, to allow showing that such
+pathological code can be handled and to show how certain features play out in
+practice::
+
+    $ cat MyAdvanced.h
+    #include <string>
+
+    class Base1 {
+    public:
+        Base1(int i) : m_i(i) {}
+        virtual ~Base1() {}
+        int m_i;
+    };
+
+    class Base2 {
+    public:
+        Base2(double d) : m_d(d) {}
+        virtual ~Base2() {}
+        double m_d;
+    };
+
+    class C;
+
+    class Derived : public virtual Base1, public virtual Base2 {
+    public:
+        Derived(const std::string& name, int i, double d) : Base1(i), Base2(d), m_name(name) {}
+        virtual C* gimeC() { return (C*)0; }
+        std::string m_name;
+    };
+
+    Base1* BaseFactory(const std::string& name, int i, double d) {
+        return new Derived(name, i, d);
+    }
+
+This code is still only in a header file, with all functions inline, for
+convenience of the example.
+If the implementations live in a separate source file or shared library, the
+only change needed is to link those in when building the reflection library.
+
+If you were to run ``genreflex`` like above in the basic example, you will
+find that not all classes of interest will be reflected, nor will be the
+global factory function.
+In particular, ``std::string`` will be missing, since it is not defined in
+this header file, but in a header file that is included.
+In practical terms, general classes such as ``std::string`` should live in a
+core reflection set, but for the moment assume we want to have it in the
+reflection library that we are building for this example.
+
+The ``genreflex`` script can be steered using a so-called `selection file`_,
+which is a simple XML file specifying, either explicitly or by using a
+pattern, which classes, variables, namespaces, etc. to select from the given
+header file.
+With the aid of a selection file, a large project can be easily managed:
+simply ``#include`` all relevant headers into a single header file that is
+handed to ``genreflex``.
+Then, apply a selection file to pick up all the relevant classes.
+For our purposes, the following rather straightforward selection will do
+(the name ``lcgdict`` for the root is historical, but required)::
+
+    $ cat MyAdvanced.xml
+    <lcgdict>
+       <class pattern="Base?" />
+       <class name="Derived" />
+       <class name="std::string" />
+       <function name="BaseFactory" />
+    </lcgdict>
+
+.. _`selection file`: http://root.cern.ch/drupal/content/generating-reflex-dictionaries
+
+Now the reflection info can be generated and compiled::
+
+    $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so
+
+and subsequently be used from PyPy::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libAdvExDict.so")
+    <CPPLibrary object at 0x00007fdb48fc8120>
+    >>>> d = cppyy.gbl.BaseFactory("name", 42, 3.14)
+    >>>> type(d)
+    <class '__main__.Derived'>
+    >>>> d.m_i
+    42
+    >>>> d.m_d
+    3.14
+    >>>> d.m_name == "name"
+    True
+    >>>>
+
+Again, that's all there is to it!
+
+A couple of things to note, though.
+If you look back at the C++ definition of the ``BaseFactory`` function,
+you will see that it declares the return type to be a ``Base1``, yet the
+bindings return an object of the actual type ``Derived``?
+This choice is made for a couple of reasons.
+First, it makes method dispatching easier: if bound objects are always their
+most derived type, then it is easy to calculate any offsets, if necessary.
+Second, it makes memory management easier: the combination of the type and
+the memory address uniquely identifies an object.
+That way, it can be recycled and object identity can be maintained if it is
+entered as a function argument into C++ and comes back to PyPy as a return
+value.
+Last, but not least, casting is decidedly unpythonistic.
+By always providing the most derived type known, casting becomes unnecessary.
+For example, the data member of ``Base2`` is simply directly available.
+Note also that the unreflected ``gimeC`` method of ``Derived`` does not
+preclude its use.
+It is only the ``gimeC`` method that is unusable as long as class ``C`` is
+unknown to the system.
+
+
 Features
 ========
 
@@ -160,6 +275,8 @@
 * **doc strings**: The doc string of a method or function contains the C++
   arguments and return types of all overloads of that name, as applicable.
 
+* **enums**: Are translated as ints with no further checking.
+
 * **functions**: Work as expected and live in their appropriate namespace
   (which can be the global one, ``cppyy.gbl``).
 

From noreply at buildbot.pypy.org  Tue Apr 24 22:08:38 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 24 Apr 2012 22:08:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: update build instructions for windows
Message-ID: <20120424200838.CEB0D82009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54734:d270e84ff61f
Date: 2012-04-24 22:57 +0300
http://bitbucket.org/pypy/pypy/changeset/d270e84ff61f/

Log:	update build instructions for windows

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -24,7 +24,8 @@
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
 (32 bits, 64 bits) is automatically selected.  A 32 bit build can only be built
-using a 32 bit Python and vice versa.
+using a 32 bit Python and vice versa. By default pypy is built using the 
+Multi-threaded DLL (/MD) runtime environment.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
 will fail in this case.
@@ -102,10 +103,12 @@
 
 Download the source code of expat on sourceforge:
 http://sourceforge.net/projects/expat/ and extract it in the base
-directory.  Then open the project file ``expat.dsw`` with Visual
+directory.  Version 2.1.0 is known to pass tests. Then open the project 
+file ``expat.dsw`` with Visual
 Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, and build the solution (the
-``expat`` project is actually enough for pypy).
+switch to the "Release" configuration, reconfigure the runtime for 
+Multi-threaded DLL (/MD) and build the solution (the ``expat`` project 
+is actually enough for pypy).
 
 Then, copy the file ``win32\bin\release\libexpat.dll`` somewhere in
 your PATH.

From noreply at buildbot.pypy.org  Tue Apr 24 22:41:04 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 24 Apr 2012 22:41:04 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: revert nt specific test
Message-ID: <20120424204104.CD4B582009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54735:2c48d071ff78
Date: 2012-04-24 23:40 +0300
http://bitbucket.org/pypy/pypy/changeset/2c48d071ff78/

Log:	revert nt specific test

diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -678,12 +678,8 @@
 
     def set_error(self, space, code):
         err = rffi.charp2strn(XML_ErrorString(code), 200)
-        if os.name == 'nt':
-            lineno = -1
-            colno = -1
-        else:
-            lineno = XML_GetCurrentLineNumber(self.itself)
-            colno = XML_GetCurrentColumnNumber(self.itself)
+        lineno = XML_GetCurrentLineNumber(self.itself)
+        colno = XML_GetCurrentColumnNumber(self.itself)
         msg = "%s: line %d, column %d" % (err, lineno, colno)
         w_errorcls = space.fromcache(Cache).w_error
         w_error = space.call_function(w_errorcls, space.wrap(msg))

From noreply at buildbot.pypy.org  Tue Apr 24 23:08:55 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 24 Apr 2012 23:08:55 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge default into branch
Message-ID: <20120424210855.6B9DE82009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54736:6c8bf5a7e2e1
Date: 2012-04-25 00:08 +0300
http://bitbucket.org/pypy/pypy/changeset/6c8bf5a7e2e1/

Log:	merge default into branch

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -111,6 +111,121 @@
 That's all there is to it!
 
 
+Advanced example
+================
+The following snippet of C++ is very contrived, to allow showing that such
+pathological code can be handled and to show how certain features play out in
+practice::
+
+    $ cat MyAdvanced.h
+    #include <string>
+
+    class Base1 {
+    public:
+        Base1(int i) : m_i(i) {}
+        virtual ~Base1() {}
+        int m_i;
+    };
+
+    class Base2 {
+    public:
+        Base2(double d) : m_d(d) {}
+        virtual ~Base2() {}
+        double m_d;
+    };
+
+    class C;
+
+    class Derived : public virtual Base1, public virtual Base2 {
+    public:
+        Derived(const std::string& name, int i, double d) : Base1(i), Base2(d), m_name(name) {}
+        virtual C* gimeC() { return (C*)0; }
+        std::string m_name;
+    };
+
+    Base1* BaseFactory(const std::string& name, int i, double d) {
+        return new Derived(name, i, d);
+    }
+
+This code is still only in a header file, with all functions inline, for
+convenience of the example.
+If the implementations live in a separate source file or shared library, the
+only change needed is to link those in when building the reflection library.
+
+If you were to run ``genreflex`` like above in the basic example, you will
+find that not all classes of interest will be reflected, nor will be the
+global factory function.
+In particular, ``std::string`` will be missing, since it is not defined in
+this header file, but in a header file that is included.
+In practical terms, general classes such as ``std::string`` should live in a
+core reflection set, but for the moment assume we want to have it in the
+reflection library that we are building for this example.
+
+The ``genreflex`` script can be steered using a so-called `selection file`_,
+which is a simple XML file specifying, either explicitly or by using a
+pattern, which classes, variables, namespaces, etc. to select from the given
+header file.
+With the aid of a selection file, a large project can be easily managed:
+simply ``#include`` all relevant headers into a single header file that is
+handed to ``genreflex``.
+Then, apply a selection file to pick up all the relevant classes.
+For our purposes, the following rather straightforward selection will do
+(the name ``lcgdict`` for the root is historical, but required)::
+
+    $ cat MyAdvanced.xml
+    <lcgdict>
+       <class pattern="Base?" />
+       <class name="Derived" />
+       <class name="std::string" />
+       <function name="BaseFactory" />
+    </lcgdict>
+
+.. _`selection file`: http://root.cern.ch/drupal/content/generating-reflex-dictionaries
+
+Now the reflection info can be generated and compiled::
+
+    $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so
+
+and subsequently be used from PyPy::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libAdvExDict.so")
+    <CPPLibrary object at 0x00007fdb48fc8120>
+    >>>> d = cppyy.gbl.BaseFactory("name", 42, 3.14)
+    >>>> type(d)
+    <class '__main__.Derived'>
+    >>>> d.m_i
+    42
+    >>>> d.m_d
+    3.14
+    >>>> d.m_name == "name"
+    True
+    >>>>
+
+Again, that's all there is to it!
+
+A couple of things to note, though.
+If you look back at the C++ definition of the ``BaseFactory`` function,
+you will see that it declares the return type to be a ``Base1``, yet the
+bindings return an object of the actual type ``Derived``?
+This choice is made for a couple of reasons.
+First, it makes method dispatching easier: if bound objects are always their
+most derived type, then it is easy to calculate any offsets, if necessary.
+Second, it makes memory management easier: the combination of the type and
+the memory address uniquely identifies an object.
+That way, it can be recycled and object identity can be maintained if it is
+entered as a function argument into C++ and comes back to PyPy as a return
+value.
+Last, but not least, casting is decidedly unpythonistic.
+By always providing the most derived type known, casting becomes unnecessary.
+For example, the data member of ``Base2`` is simply directly available.
+Note also that the unreflected ``gimeC`` method of ``Derived`` does not
+preclude its use.
+It is only the ``gimeC`` method that is unusable as long as class ``C`` is
+unknown to the system.
+
+
 Features
 ========
 
@@ -160,6 +275,8 @@
 * **doc strings**: The doc string of a method or function contains the C++
   arguments and return types of all overloads of that name, as applicable.
 
+* **enums**: Are translated as ints with no further checking.
+
 * **functions**: Work as expected and live in their appropriate namespace
   (which can be the global one, ``cppyy.gbl``).
 
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -24,7 +24,8 @@
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
 (32 bits, 64 bits) is automatically selected.  A 32 bit build can only be built
-using a 32 bit Python and vice versa.
+using a 32 bit Python and vice versa. By default pypy is built using the 
+Multi-threaded DLL (/MD) runtime environment.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
 will fail in this case.
@@ -102,10 +103,12 @@
 
 Download the source code of expat on sourceforge:
 http://sourceforge.net/projects/expat/ and extract it in the base
-directory.  Then open the project file ``expat.dsw`` with Visual
+directory.  Version 2.1.0 is known to pass tests. Then open the project 
+file ``expat.dsw`` with Visual
 Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, and build the solution (the
-``expat`` project is actually enough for pypy).
+switch to the "Release" configuration, reconfigure the runtime for 
+Multi-threaded DLL (/MD) and build the solution (the ``expat`` project 
+is actually enough for pypy).
 
 Then, copy the file ``win32\bin\release\libexpat.dll`` somewhere in
 your PATH.
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -20,6 +20,7 @@
         self.dependencies = {}
         # contains frame boxes that are not virtualizables
         self.nonstandard_virtualizables = {}
+
         # heap cache
         # maps descrs to {from_box, to_box} dicts
         self.heap_cache = {}
@@ -29,6 +30,26 @@
         # cache the length of arrays
         self.length_cache = {}
 
+        # replace_box is called surprisingly often, therefore it's not efficient
+        # to go over all the dicts and fix them.
+        # instead, these two dicts are kept, and a replace_box adds an entry to
+        # each of them.
+        # every time one of the dicts heap_cache, heap_array_cache, length_cache
+        # is accessed, suitable indirections need to be performed
+
+        # this looks all very subtle, but in practice the patterns of
+        # replacements should not be that complex. Usually a box is replaced by
+        # a const, once. Also, if something goes wrong, the effect is that less
+        # caching than possible is done, which is not a huge problem.
+        self.input_indirections = {}
+        self.output_indirections = {}
+
+    def _input_indirection(self, box):
+        return self.input_indirections.get(box, box)
+
+    def _output_indirection(self, box):
+        return self.output_indirections.get(box, box)
+
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, argboxes)
         self.clear_caches(opnum, descr, argboxes)
@@ -132,14 +153,16 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
+        box = self._input_indirection(box)
         d = self.heap_cache.get(descr, None)
         if d:
             tobox = d.get(box, None)
-            if tobox:
-                return tobox
+            return self._output_indirection(tobox)
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         self.heap_cache.setdefault(descr, {})[box] = fieldbox
 
     def setfield(self, box, descr, fieldbox):
@@ -148,6 +171,8 @@
         self.heap_cache[descr] = new_d
 
     def _do_write_with_aliasing(self, d, box, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         # slightly subtle logic here
         # a write to an arbitrary box, all other boxes can alias this one
         if not d or box not in self.new_boxes:
@@ -166,6 +191,7 @@
         return new_d
 
     def getarrayitem(self, box, descr, indexbox):
+        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -173,9 +199,11 @@
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return indexcache.get(box, None)
+                return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        box = self._input_indirection(box)
+        valuebox = self._input_indirection(valuebox)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -198,25 +226,13 @@
         cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
 
     def arraylen(self, box):
-        return self.length_cache.get(box, None)
+        box = self._input_indirection(box)
+        return self._output_indirection(self.length_cache.get(box, None))
 
     def arraylen_now_known(self, box, lengthbox):
-        self.length_cache[box] = lengthbox
-
-    def _replace_box(self, d, oldbox, newbox):
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            if frombox is oldbox:
-                frombox = newbox
-            if tobox is oldbox:
-                tobox = newbox
-            new_d[frombox] = tobox
-        return new_d
+        box = self._input_indirection(box)
+        self.length_cache[box] = self._input_indirection(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        for descr, d in self.heap_cache.iteritems():
-            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
-        for descr, d in self.heap_array_cache.iteritems():
-            for index, cache in d.iteritems():
-                d[index] = self._replace_box(cache, oldbox, newbox)
-        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
+        self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox)
+        self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -7,7 +7,7 @@
 import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt
 import pypy.jit.metainterp.optimizeopt.virtualize as virtualize
 from pypy.jit.metainterp.optimize import InvalidLoop
-from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
+from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, get_const_ptr_for_string
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -5067,6 +5067,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_call_pure_vstring_const(self):
+        ops = """
+        []
+        p0 = newstr(3)
+        strsetitem(p0, 0, 97)
+        strsetitem(p0, 1, 98)
+        strsetitem(p0, 2, 99)
+        i0 = call_pure(123, p0, descr=nonwritedescr)
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(5)
+        """
+        call_pure_results = {
+            (ConstInt(123), get_const_ptr_for_string("abc"),): ConstInt(5),
+        }
+        self.optimize_loop(ops, expected, call_pure_results)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -2,12 +2,14 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp.history import ConstInt
 
-box1 = object()
-box2 = object()
-box3 = object()
-box4 = object()
+box1 = "box1"
+box2 = "box2"
+box3 = "box3"
+box4 = "box4"
+box5 = "box5"
 lengthbox1 = object()
 lengthbox2 = object()
+lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -276,11 +278,43 @@
         h.setfield(box1, descr2, box3)
         h.setfield(box2, descr3, box3)
         h.replace_box(box1, box4)
-        assert h.getfield(box1, descr1) is None
-        assert h.getfield(box1, descr2) is None
         assert h.getfield(box4, descr1) is box2
         assert h.getfield(box4, descr2) is box3
         assert h.getfield(box2, descr3) is box3
+        h.setfield(box4, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box4
+        assert h.getfield(box2, descr3) is box4
+
+    def test_replace_box_twice(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box5, descr1) is box2
+        assert h.getfield(box5, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+        h.setfield(box5, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box5
+        assert h.getfield(box2, descr3) is box5
 
     def test_replace_box_array(self):
         h = HeapCache()
@@ -291,9 +325,6 @@
         h.setarrayitem(box3, descr2, index2, box1)
         h.setarrayitem(box2, descr3, index2, box3)
         h.replace_box(box1, box4)
-        assert h.getarrayitem(box1, descr1, index1) is None
-        assert h.getarrayitem(box1, descr2, index1) is None
-        assert h.arraylen(box1) is None
         assert h.arraylen(box4) is lengthbox1
         assert h.getarrayitem(box4, descr1, index1) is box2
         assert h.getarrayitem(box4, descr2, index1) is box3
@@ -304,6 +335,27 @@
         h.replace_box(lengthbox1, lengthbox2)
         assert h.arraylen(box4) is lengthbox2
 
+    def test_replace_box_array_twice(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box5, descr1, index1) is box2
+        assert h.getarrayitem(box5, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box5
+        assert h.getarrayitem(box3, descr2, index2) is box5
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        h.replace_box(lengthbox2, lengthbox3)
+        assert h.arraylen(box4) is lengthbox3
+
     def test_ll_arraycopy(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -38,10 +38,19 @@
 	PyObject_VAR_HEAD
 } PyVarObject;
 
+#ifndef PYPY_DEBUG_REFCOUNT
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)ob))
 #define Py_DECREF(ob)   (Py_DecRef((PyObject *)ob))
 #define Py_XINCREF(ob)  (Py_IncRef((PyObject *)ob))
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)ob))
+#else
+#define Py_INCREF(ob)   (((PyObject *)ob)->ob_refcnt++)
+#define Py_DECREF(ob)  ((((PyObject *)ob)->ob_refcnt > 1) ? \
+			((PyObject *)ob)->ob_refcnt-- : (Py_DecRef((PyObject *)ob)))
+
+#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
+#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
+#endif
 
 #define Py_CLEAR(op)				\
         do {                            	\
diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -1234,7 +1234,7 @@
             pos += 1
             continue
 
-        if 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
             # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
             pos += 1
             oc2 = ord(s[pos])
@@ -1350,6 +1350,20 @@
     pos = 0
     while pos < size:
         oc = ord(s[pos])
+
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+            # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
+            pos += 1
+            oc2 = ord(s[pos])
+
+            if 0xDC00 <= oc2 <= 0xDFFF:
+                ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000
+                raw_unicode_escape_helper(result, ucs)
+                pos += 1
+                continue
+            # Fall through: isolated surrogates are copied as-is
+            pos -= 1
+
         if oc < 0x100:
             result.append(chr(oc))
         else:
diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py
--- a/pypy/rlib/test/test_runicode.py
+++ b/pypy/rlib/test/test_runicode.py
@@ -728,3 +728,18 @@
 
         res = interpret(f, [0x10140])
         assert res == 0x10140
+
+    def test_encode_surrogate_pair(self):
+        u = runicode.UNICHR(0xD800) + runicode.UNICHR(0xDC00)
+        if runicode.MAXUNICODE < 65536:
+            # Narrow unicode build, consider utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+        else:
+            # Wide unicode build, don't merge utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'

From noreply at buildbot.pypy.org  Tue Apr 24 23:19:01 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Tue, 24 Apr 2012 23:19:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: cppyy documentation on how to handle
	templates
Message-ID: <20120424211901.4F92A82009@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r54737:1e469996fdab
Date: 2012-04-24 14:18 -0700
http://bitbucket.org/pypy/pypy/changeset/1e469996fdab/

Log:	cppyy documentation on how to handle templates

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -80,7 +80,7 @@
         void SetMyInt(int i) { m_myint = i; }
 
     public:
-       int m_myint;
+        int m_myint;
     };
 
 Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
@@ -174,10 +174,10 @@
 
     $ cat MyAdvanced.xml
     <lcgdict>
-       <class pattern="Base?" />
-       <class name="Derived" />
-       <class name="std::string" />
-       <function name="BaseFactory" />
+        <class pattern="Base?" />
+        <class name="Derived" />
+        <class name="std::string" />
+        <function name="BaseFactory" />
     </lcgdict>
 
 .. _`selection file`: http://root.cern.ch/drupal/content/generating-reflex-dictionaries
@@ -353,6 +353,9 @@
   using classes that themselves are templates (etc.) in the arguments.
   All classes must already exist in the loaded reflection info.
 
+* **typedefs**: Are simple python references to the actual classes to which
+  they refer.
+
 * **unary operators**: Are supported if a python equivalent exists, and if the
   operator is defined in the C++ class.
 
@@ -370,6 +373,107 @@
 Only that one specific method can not be used.
 
 
+Templates
+=========
+
+A bit of special care needs to be taken for the use of templates.
+For a templated class to be completely available, it must be guaranteed that
+said class is fully instantiated, and hence all executable C++ code is
+generated and compiled in.
+The easiest way to fulfill that guarantee is by explicit instantiation in the
+header file that is handed to ``genreflex``.
+The following example should make that clear::
+
+    $ cat MyTemplate.h
+    #include <vector>
+
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_i(i) {}
+        MyClass(const MyClass& s) : m_i(s.m_i) {}
+        MyClass& operator=(const MyClass& s) { m_i = s.m_i; return *this; }
+        ~MyClass() {}
+        int m_i;
+    };
+
+    template class std::vector<MyClass>;
+
+If you know for certain that all symbols will be linked in from other sources,
+you can also declare the explicit template instantiation ``extern``.
+
+Unfortunately, this is not enough for gcc.
+The iterators, if they are going to be used, need to be instantiated as well,
+as do the comparison operators on those iterators, as these live in an
+internal namespace, rather than in the iterator classes.
+One way to handle this, is to deal with this once in a macro, then reuse that
+macro for all ``vector`` classes.
+Thus, the header above needs this, instead of just the explicit instantiation
+of the ``vector<MyClass>``::
+
+    #define STLTYPES_EXPLICIT_INSTANTIATION_DECL(STLTYPE, TTYPE)                      \
+    template class std::STLTYPE< TTYPE >;                                             \
+    template class __gnu_cxx::__normal_iterator<TTYPE*, std::STLTYPE< TTYPE > >;      \
+    template class __gnu_cxx::__normal_iterator<const TTYPE*, std::STLTYPE< TTYPE > >;\
+    namespace __gnu_cxx {                                                             \
+    template bool operator==(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    template bool operator!=(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    }
+
+    STLTYPES_EXPLICIT_INSTANTIATION_DECL(vector, MyClass)
+
+Then, still for gcc, the selection file needs to contain the full hierarchy as
+well as the global overloads for comparisons for the iterators::
+
+    $ cat MyTemplate.xml
+    <lcgdict>
+        <class pattern="std::vector<*>" />
+        <class pattern="__gnu_cxx::__normal_iterator<*>" />
+        <class pattern="__gnu_cxx::new_allocator<*>" />
+        <class pattern="std::_Vector_base<*>" />
+        <class pattern="std::_Vector_base<*>::_Vector_impl" />
+        <class pattern="std::allocator<*>" />
+        <function name="__gnu_cxx::operator=="/>
+        <function name="__gnu_cxx::operator!="/>
+
+        <class name="MyClass" />
+    </lcgdict>
+
+Run the normal ``genreflex`` and compilation steps::
+
+    $ genreflex MyTemplate.h --selection=MyTemplate.xm
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so
+
+Note: this is a dirty corner that clearly could do with some automation,
+even if the macro already helps.
+Such automation is planned.
+In fact, in the cling world, the backend can perform the template
+instantations and generate the reflection info on the fly, and none of the
+above will any longer be necessary.
+
+Subsequent use should be as expected.
+Note the meta-class style of "instantiating" the template::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libTemplateDict.so")
+    >>>> std = cppyy.gbl.std
+    >>>> MyClass = cppyy.gbl.MyClass
+    >>>> v = std.vector(MyClass)()
+    >>>> v += [MyClass(1), MyClass(2), MyClass(3)]
+    >>>> for m in v:
+    ....     print m.m_i,
+    ....
+    1 2 3
+    >>>>
+
+Other templates work similarly.
+The arguments to the template instantiation can either be a string with the
+full list of arguments, or the explicit classes.
+The latter makes for easier code writing if the classes passed to the
+instantiation are themselves templates.
+
+
 The fast lane
 =============
 

From noreply at buildbot.pypy.org  Wed Apr 25 00:44:28 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Wed, 25 Apr 2012 00:44:28 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: add a __dispatch__ method to
 classes to allow the selection of a specific overload based on its
 signature
Message-ID: <20120424224428.34AEC82009@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54738:1ad80dbd5299
Date: 2012-04-24 10:37 -0700
http://bitbucket.org/pypy/pypy/changeset/1ad80dbd5299/

Log:	add a __dispatch__ method to classes to allow the selection of a
	specific overload based on its signature

diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -442,6 +442,15 @@
         self.datamembers[name] = new_dm
         return new_dm
 
+    @jit.elidable_promote('0')
+    def dispatch(self, name, signature):
+        overload = self.get_overload(name)
+        sig = '(%s)' % signature
+        for f in overload.functions:
+            if 0 < f.signature().find(sig):
+                return W_CPPOverload(self.space, self, [f])
+        raise OperationError(space.w_TypeError, space.wrap("no overload matches signature"))
+
     def missing_attribute_error(self, name):
         return OperationError(
             self.space.w_AttributeError,
@@ -585,6 +594,7 @@
     get_datamember_names = interp2app(W_CPPClass.get_datamember_names, unwrap_spec=['self']),
     get_datamember = interp2app(W_CPPClass.get_datamember, unwrap_spec=['self', str]),
     is_namespace = interp2app(W_CPPClass.is_namespace, unwrap_spec=['self']),
+    dispatch = interp2app(W_CPPClass.dispatch, unwrap_spec=['self', str, str]),
 )
 W_CPPClass.typedef.acceptable_as_base_class = False
 
diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -1,5 +1,6 @@
 # NOT_RPYTHON
 import cppyy
+import types
 
 
 # For now, keep namespaces and classes separate as namespaces are extensible
@@ -164,8 +165,12 @@
     metacpp = type(CppyyClass)(class_name+'_meta', _drop_cycles(metabases), {})
 
     # create the python-side C++ class representation
-    d = {"_cpp_proxy" : cppclass,
-         "__new__"    : make_new(class_name, cppclass),
+    def dispatch(self, name, signature):
+        cppol = cppclass.dispatch(name, signature)
+        return types.MethodType(make_method(name, cppol), self, type(self))
+    d = {"_cpp_proxy"   : cppclass,
+         "__dispatch__" : dispatch,
+         "__new__"      : make_new(class_name, cppclass),
          }
     pycppclass = metacpp(class_name, _drop_cycles(bases), d)
  
@@ -275,8 +280,6 @@
 
 
 # pythonization by decoration (move to their own file?)
-import types
-
 def python_style_getitem(self, idx):
     # python-style indexing: check for size and allow indexing from the back
     sz = len(self)
diff --git a/pypy/module/cppyy/test/test_advancedcpp.py b/pypy/module/cppyy/test/test_advancedcpp.py
--- a/pypy/module/cppyy/test/test_advancedcpp.py
+++ b/pypy/module/cppyy/test/test_advancedcpp.py
@@ -485,6 +485,5 @@
 
         c2 = cppyy.gbl.create_c2()
         assert type(c2) == cppyy.gbl.c_class_2
-        print c2.m_c
         assert c2.m_c == 3
         c2.destruct()
diff --git a/pypy/module/cppyy/test/test_overloads.py b/pypy/module/cppyy/test/test_overloads.py
--- a/pypy/module/cppyy/test/test_overloads.py
+++ b/pypy/module/cppyy/test/test_overloads.py
@@ -49,8 +49,6 @@
     def test02_class_based_overloads_explicit_resolution(self):
         """Test explicitly resolved function overloads"""
 
-        # TODO: write disp() or equivalent on methods for ol selection
-
         import cppyy
         a_overload = cppyy.gbl.a_overload
         b_overload = cppyy.gbl.b_overload
@@ -60,16 +58,16 @@
         ns_a_overload = cppyy.gbl.ns_a_overload
 
         c = c_overload()
-#        raises(TypeError, c.get_int.disp, 12)
-#        assert c.get_int.disp('a_overload* a')(a_overload()) == 42
-#        assert c.get_int.disp('b_overload* b')(b_overload()) == 13
+        raises(TypeError, c.__dispatch__, 'get_int', 12)
+        assert c.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
+        assert c.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
 
-#        assert c_overload().get_int.disp('a_overload* a')(a_overload())  == 42
-#        assert c_overload.get_int.disp('b_overload* b')(c, b_overload()) == 13
+        assert c_overload().__dispatch__('get_int', 'a_overload*')(a_overload())  == 42
+#        assert c_overload.__dispatch__('get_int', 'b_overload*')(c, b_overload()) == 13
 
         d = d_overload()
-#        assert d.get_int.disp('a_overload* a')(a_overload()) == 42
-#        assert d.get_int.disp('b_overload* b')(b_overload()) == 13
+        assert d.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
+        assert d.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
 
         nb = ns_a_overload.b_overload()
         raises(TypeError, nb.f, c_overload())

From noreply at buildbot.pypy.org  Wed Apr 25 00:44:29 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Wed, 25 Apr 2012 00:44:29 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: translation fixes and extra test
 to catch them earlier next time
Message-ID: <20120424224429.73E588208A@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54739:c606b30252d7
Date: 2012-04-24 11:48 -0700
http://bitbucket.org/pypy/pypy/changeset/c606b30252d7/

Log:	translation fixes and extra test to catch them earlier next time

diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py
--- a/pypy/module/cppyy/capi/__init__.py
+++ b/pypy/module/cppyy/capi/__init__.py
@@ -243,7 +243,7 @@
 
 _c_base_offset = rffi.llexternal(
     "cppyy_base_offset",
-    [C_TYPE, C_TYPE, C_OBJECT, rffi.INT], rffi.LONG,
+    [C_TYPE, C_TYPE, C_OBJECT, rffi.INT], rffi.SIZE_T,
     threadsafe=threadsafe,
     compilation_info=backend.eci,
     elidable_function=True)
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -61,7 +61,7 @@
     int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base);
 
     /* calculate offsets between declared and actual type, up-cast: direction > 0; down-cast: direction < 0 */
-    long cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
+    size_t cppyy_base_offset(cppyy_type_t derived, cppyy_type_t base, cppyy_object_t address, int direction);
 
     /* method/function reflection information --------------------------------- */
     int cppyy_num_methods(cppyy_scope_t scope);
diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py
--- a/pypy/module/cppyy/interp_cppyy.py
+++ b/pypy/module/cppyy/interp_cppyy.py
@@ -449,7 +449,7 @@
         for f in overload.functions:
             if 0 < f.signature().find(sig):
                 return W_CPPOverload(self.space, self, [f])
-        raise OperationError(space.w_TypeError, space.wrap("no overload matches signature"))
+        raise OperationError(self.space.w_TypeError, self.space.wrap("no overload matches signature"))
 
     def missing_attribute_error(self, name):
         return OperationError(
@@ -616,6 +616,7 @@
     get_datamember_names = interp2app(W_ComplexCPPClass.get_datamember_names, unwrap_spec=['self']),
     get_datamember = interp2app(W_ComplexCPPClass.get_datamember, unwrap_spec=['self', str]),
     is_namespace = interp2app(W_ComplexCPPClass.is_namespace, unwrap_spec=['self']),
+    dispatch = interp2app(W_CPPClass.dispatch, unwrap_spec=['self', str, str]),
 )
 W_ComplexCPPClass.typedef.acceptable_as_base_class = False
 
diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx
--- a/pypy/module/cppyy/src/cintcwrapper.cxx
+++ b/pypy/module/cppyy/src/cintcwrapper.cxx
@@ -467,7 +467,7 @@
     return derived_type->GetBaseClass(base_type) != 0;
 }
 
-long cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle,
+size_t cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle,
                        cppyy_object_t address, int /* direction */) {
     // WARNING: CINT can not handle actual dynamic casts!
     TClassRef derived_type = type_from_handle(derived_handle);
@@ -494,7 +494,7 @@
          }
     }
 
-    return offset;
+    return (size_t) offset;   // may be negative (will roll over)
 }
 
 
diff --git a/pypy/module/cppyy/src/reflexcwrapper.cxx b/pypy/module/cppyy/src/reflexcwrapper.cxx
--- a/pypy/module/cppyy/src/reflexcwrapper.cxx
+++ b/pypy/module/cppyy/src/reflexcwrapper.cxx
@@ -277,7 +277,7 @@
     return (int)derived_type.HasBase(base_type);
 }
 
-long cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle,
+size_t cppyy_base_offset(cppyy_type_t derived_handle, cppyy_type_t base_handle,
                        cppyy_object_t address, int direction) {
     Reflex::Type derived_type = type_from_handle(derived_handle);
     Reflex::Type base_type = type_from_handle(base_handle);
@@ -303,8 +303,8 @@
             if (ibase->first.ToType() == base_type) {
                 long offset = (long)ibase->first.Offset((void*)address);
                 if (direction < 0)
-                    return -offset;
-                return offset;
+                   return (size_t) -offset;  // note negative; rolls over
+                return (size_t)offset;
             }
         }
 
diff --git a/pypy/module/cppyy/test/test_overloads.py b/pypy/module/cppyy/test/test_overloads.py
--- a/pypy/module/cppyy/test/test_overloads.py
+++ b/pypy/module/cppyy/test/test_overloads.py
@@ -59,6 +59,7 @@
 
         c = c_overload()
         raises(TypeError, c.__dispatch__, 'get_int', 12)
+        raises(TypeError, c.__dispatch__, 'get_int', 'does_not_exist')
         assert c.__dispatch__('get_int', 'a_overload*')(a_overload()) == 42
         assert c.__dispatch__('get_int', 'b_overload*')(b_overload()) == 13
 

From noreply at buildbot.pypy.org  Wed Apr 25 00:44:33 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Wed, 25 Apr 2012 00:44:33 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: merge default into branch
Message-ID: <20120424224434.0569B820C1@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54742:c0d3d317564f
Date: 2012-04-24 15:44 -0700
http://bitbucket.org/pypy/pypy/changeset/c0d3d317564f/

Log:	merge default into branch

diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -21,7 +21,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC']
     res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
@@ -34,6 +34,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST'] # needed for VC10
     else:
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -16,7 +16,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
     res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
@@ -29,6 +29,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
                         '/EXPORT:init_testcapi']
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -80,7 +80,7 @@
         void SetMyInt(int i) { m_myint = i; }
 
     public:
-       int m_myint;
+        int m_myint;
     };
 
 Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
@@ -111,6 +111,121 @@
 That's all there is to it!
 
 
+Advanced example
+================
+The following snippet of C++ is very contrived, to allow showing that such
+pathological code can be handled and to show how certain features play out in
+practice::
+
+    $ cat MyAdvanced.h
+    #include <string>
+
+    class Base1 {
+    public:
+        Base1(int i) : m_i(i) {}
+        virtual ~Base1() {}
+        int m_i;
+    };
+
+    class Base2 {
+    public:
+        Base2(double d) : m_d(d) {}
+        virtual ~Base2() {}
+        double m_d;
+    };
+
+    class C;
+
+    class Derived : public virtual Base1, public virtual Base2 {
+    public:
+        Derived(const std::string& name, int i, double d) : Base1(i), Base2(d), m_name(name) {}
+        virtual C* gimeC() { return (C*)0; }
+        std::string m_name;
+    };
+
+    Base1* BaseFactory(const std::string& name, int i, double d) {
+        return new Derived(name, i, d);
+    }
+
+This code is still only in a header file, with all functions inline, for
+convenience of the example.
+If the implementations live in a separate source file or shared library, the
+only change needed is to link those in when building the reflection library.
+
+If you were to run ``genreflex`` like above in the basic example, you will
+find that not all classes of interest will be reflected, nor will be the
+global factory function.
+In particular, ``std::string`` will be missing, since it is not defined in
+this header file, but in a header file that is included.
+In practical terms, general classes such as ``std::string`` should live in a
+core reflection set, but for the moment assume we want to have it in the
+reflection library that we are building for this example.
+
+The ``genreflex`` script can be steered using a so-called `selection file`_,
+which is a simple XML file specifying, either explicitly or by using a
+pattern, which classes, variables, namespaces, etc. to select from the given
+header file.
+With the aid of a selection file, a large project can be easily managed:
+simply ``#include`` all relevant headers into a single header file that is
+handed to ``genreflex``.
+Then, apply a selection file to pick up all the relevant classes.
+For our purposes, the following rather straightforward selection will do
+(the name ``lcgdict`` for the root is historical, but required)::
+
+    $ cat MyAdvanced.xml
+    <lcgdict>
+        <class pattern="Base?" />
+        <class name="Derived" />
+        <class name="std::string" />
+        <function name="BaseFactory" />
+    </lcgdict>
+
+.. _`selection file`: http://root.cern.ch/drupal/content/generating-reflex-dictionaries
+
+Now the reflection info can be generated and compiled::
+
+    $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so
+
+and subsequently be used from PyPy::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libAdvExDict.so")
+    <CPPLibrary object at 0x00007fdb48fc8120>
+    >>>> d = cppyy.gbl.BaseFactory("name", 42, 3.14)
+    >>>> type(d)
+    <class '__main__.Derived'>
+    >>>> d.m_i
+    42
+    >>>> d.m_d
+    3.14
+    >>>> d.m_name == "name"
+    True
+    >>>>
+
+Again, that's all there is to it!
+
+A couple of things to note, though.
+If you look back at the C++ definition of the ``BaseFactory`` function,
+you will see that it declares the return type to be a ``Base1``, yet the
+bindings return an object of the actual type ``Derived``?
+This choice is made for a couple of reasons.
+First, it makes method dispatching easier: if bound objects are always their
+most derived type, then it is easy to calculate any offsets, if necessary.
+Second, it makes memory management easier: the combination of the type and
+the memory address uniquely identifies an object.
+That way, it can be recycled and object identity can be maintained if it is
+entered as a function argument into C++ and comes back to PyPy as a return
+value.
+Last, but not least, casting is decidedly unpythonistic.
+By always providing the most derived type known, casting becomes unnecessary.
+For example, the data member of ``Base2`` is simply directly available.
+Note also that the unreflected ``gimeC`` method of ``Derived`` does not
+preclude its use.
+It is only the ``gimeC`` method that is unusable as long as class ``C`` is
+unknown to the system.
+
+
 Features
 ========
 
@@ -160,6 +275,8 @@
 * **doc strings**: The doc string of a method or function contains the C++
   arguments and return types of all overloads of that name, as applicable.
 
+* **enums**: Are translated as ints with no further checking.
+
 * **functions**: Work as expected and live in their appropriate namespace
   (which can be the global one, ``cppyy.gbl``).
 
@@ -236,6 +353,9 @@
   using classes that themselves are templates (etc.) in the arguments.
   All classes must already exist in the loaded reflection info.
 
+* **typedefs**: Are simple python references to the actual classes to which
+  they refer.
+
 * **unary operators**: Are supported if a python equivalent exists, and if the
   operator is defined in the C++ class.
 
@@ -253,6 +373,107 @@
 Only that one specific method can not be used.
 
 
+Templates
+=========
+
+A bit of special care needs to be taken for the use of templates.
+For a templated class to be completely available, it must be guaranteed that
+said class is fully instantiated, and hence all executable C++ code is
+generated and compiled in.
+The easiest way to fulfill that guarantee is by explicit instantiation in the
+header file that is handed to ``genreflex``.
+The following example should make that clear::
+
+    $ cat MyTemplate.h
+    #include <vector>
+
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_i(i) {}
+        MyClass(const MyClass& s) : m_i(s.m_i) {}
+        MyClass& operator=(const MyClass& s) { m_i = s.m_i; return *this; }
+        ~MyClass() {}
+        int m_i;
+    };
+
+    template class std::vector<MyClass>;
+
+If you know for certain that all symbols will be linked in from other sources,
+you can also declare the explicit template instantiation ``extern``.
+
+Unfortunately, this is not enough for gcc.
+The iterators, if they are going to be used, need to be instantiated as well,
+as do the comparison operators on those iterators, as these live in an
+internal namespace, rather than in the iterator classes.
+One way to handle this, is to deal with this once in a macro, then reuse that
+macro for all ``vector`` classes.
+Thus, the header above needs this, instead of just the explicit instantiation
+of the ``vector<MyClass>``::
+
+    #define STLTYPES_EXPLICIT_INSTANTIATION_DECL(STLTYPE, TTYPE)                      \
+    template class std::STLTYPE< TTYPE >;                                             \
+    template class __gnu_cxx::__normal_iterator<TTYPE*, std::STLTYPE< TTYPE > >;      \
+    template class __gnu_cxx::__normal_iterator<const TTYPE*, std::STLTYPE< TTYPE > >;\
+    namespace __gnu_cxx {                                                             \
+    template bool operator==(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    template bool operator!=(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    }
+
+    STLTYPES_EXPLICIT_INSTANTIATION_DECL(vector, MyClass)
+
+Then, still for gcc, the selection file needs to contain the full hierarchy as
+well as the global overloads for comparisons for the iterators::
+
+    $ cat MyTemplate.xml
+    <lcgdict>
+        <class pattern="std::vector<*>" />
+        <class pattern="__gnu_cxx::__normal_iterator<*>" />
+        <class pattern="__gnu_cxx::new_allocator<*>" />
+        <class pattern="std::_Vector_base<*>" />
+        <class pattern="std::_Vector_base<*>::_Vector_impl" />
+        <class pattern="std::allocator<*>" />
+        <function name="__gnu_cxx::operator=="/>
+        <function name="__gnu_cxx::operator!="/>
+
+        <class name="MyClass" />
+    </lcgdict>
+
+Run the normal ``genreflex`` and compilation steps::
+
+    $ genreflex MyTemplate.h --selection=MyTemplate.xm
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so
+
+Note: this is a dirty corner that clearly could do with some automation,
+even if the macro already helps.
+Such automation is planned.
+In fact, in the cling world, the backend can perform the template
+instantations and generate the reflection info on the fly, and none of the
+above will any longer be necessary.
+
+Subsequent use should be as expected.
+Note the meta-class style of "instantiating" the template::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libTemplateDict.so")
+    >>>> std = cppyy.gbl.std
+    >>>> MyClass = cppyy.gbl.MyClass
+    >>>> v = std.vector(MyClass)()
+    >>>> v += [MyClass(1), MyClass(2), MyClass(3)]
+    >>>> for m in v:
+    ....     print m.m_i,
+    ....
+    1 2 3
+    >>>>
+
+Other templates work similarly.
+The arguments to the template instantiation can either be a string with the
+full list of arguments, or the explicit classes.
+The latter makes for easier code writing if the classes passed to the
+instantiation are themselves templates.
+
+
 The fast lane
 =============
 
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -24,7 +24,8 @@
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
 (32 bits, 64 bits) is automatically selected.  A 32 bit build can only be built
-using a 32 bit Python and vice versa.
+using a 32 bit Python and vice versa. By default pypy is built using the 
+Multi-threaded DLL (/MD) runtime environment.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
 will fail in this case.
@@ -102,10 +103,12 @@
 
 Download the source code of expat on sourceforge:
 http://sourceforge.net/projects/expat/ and extract it in the base
-directory.  Then open the project file ``expat.dsw`` with Visual
+directory.  Version 2.1.0 is known to pass tests. Then open the project 
+file ``expat.dsw`` with Visual
 Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, and build the solution (the
-``expat`` project is actually enough for pypy).
+switch to the "Release" configuration, reconfigure the runtime for 
+Multi-threaded DLL (/MD) and build the solution (the ``expat`` project 
+is actually enough for pypy).
 
 Then, copy the file ``win32\bin\release\libexpat.dll`` somewhere in
 your PATH.
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -20,6 +20,7 @@
         self.dependencies = {}
         # contains frame boxes that are not virtualizables
         self.nonstandard_virtualizables = {}
+
         # heap cache
         # maps descrs to {from_box, to_box} dicts
         self.heap_cache = {}
@@ -29,6 +30,26 @@
         # cache the length of arrays
         self.length_cache = {}
 
+        # replace_box is called surprisingly often, therefore it's not efficient
+        # to go over all the dicts and fix them.
+        # instead, these two dicts are kept, and a replace_box adds an entry to
+        # each of them.
+        # every time one of the dicts heap_cache, heap_array_cache, length_cache
+        # is accessed, suitable indirections need to be performed
+
+        # this looks all very subtle, but in practice the patterns of
+        # replacements should not be that complex. Usually a box is replaced by
+        # a const, once. Also, if something goes wrong, the effect is that less
+        # caching than possible is done, which is not a huge problem.
+        self.input_indirections = {}
+        self.output_indirections = {}
+
+    def _input_indirection(self, box):
+        return self.input_indirections.get(box, box)
+
+    def _output_indirection(self, box):
+        return self.output_indirections.get(box, box)
+
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, argboxes)
         self.clear_caches(opnum, descr, argboxes)
@@ -132,14 +153,16 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
+        box = self._input_indirection(box)
         d = self.heap_cache.get(descr, None)
         if d:
             tobox = d.get(box, None)
-            if tobox:
-                return tobox
+            return self._output_indirection(tobox)
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         self.heap_cache.setdefault(descr, {})[box] = fieldbox
 
     def setfield(self, box, descr, fieldbox):
@@ -148,6 +171,8 @@
         self.heap_cache[descr] = new_d
 
     def _do_write_with_aliasing(self, d, box, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         # slightly subtle logic here
         # a write to an arbitrary box, all other boxes can alias this one
         if not d or box not in self.new_boxes:
@@ -166,6 +191,7 @@
         return new_d
 
     def getarrayitem(self, box, descr, indexbox):
+        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -173,9 +199,11 @@
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return indexcache.get(box, None)
+                return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        box = self._input_indirection(box)
+        valuebox = self._input_indirection(valuebox)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -198,25 +226,13 @@
         cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
 
     def arraylen(self, box):
-        return self.length_cache.get(box, None)
+        box = self._input_indirection(box)
+        return self._output_indirection(self.length_cache.get(box, None))
 
     def arraylen_now_known(self, box, lengthbox):
-        self.length_cache[box] = lengthbox
-
-    def _replace_box(self, d, oldbox, newbox):
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            if frombox is oldbox:
-                frombox = newbox
-            if tobox is oldbox:
-                tobox = newbox
-            new_d[frombox] = tobox
-        return new_d
+        box = self._input_indirection(box)
+        self.length_cache[box] = self._input_indirection(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        for descr, d in self.heap_cache.iteritems():
-            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
-        for descr, d in self.heap_array_cache.iteritems():
-            for index, cache in d.iteritems():
-                d[index] = self._replace_box(cache, oldbox, newbox)
-        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
+        self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox)
+        self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -7,7 +7,7 @@
 import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt
 import pypy.jit.metainterp.optimizeopt.virtualize as virtualize
 from pypy.jit.metainterp.optimize import InvalidLoop
-from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
+from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, get_const_ptr_for_string
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -5067,6 +5067,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_call_pure_vstring_const(self):
+        ops = """
+        []
+        p0 = newstr(3)
+        strsetitem(p0, 0, 97)
+        strsetitem(p0, 1, 98)
+        strsetitem(p0, 2, 99)
+        i0 = call_pure(123, p0, descr=nonwritedescr)
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(5)
+        """
+        call_pure_results = {
+            (ConstInt(123), get_const_ptr_for_string("abc"),): ConstInt(5),
+        }
+        self.optimize_loop(ops, expected, call_pure_results)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -2,12 +2,14 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp.history import ConstInt
 
-box1 = object()
-box2 = object()
-box3 = object()
-box4 = object()
+box1 = "box1"
+box2 = "box2"
+box3 = "box3"
+box4 = "box4"
+box5 = "box5"
 lengthbox1 = object()
 lengthbox2 = object()
+lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -276,11 +278,43 @@
         h.setfield(box1, descr2, box3)
         h.setfield(box2, descr3, box3)
         h.replace_box(box1, box4)
-        assert h.getfield(box1, descr1) is None
-        assert h.getfield(box1, descr2) is None
         assert h.getfield(box4, descr1) is box2
         assert h.getfield(box4, descr2) is box3
         assert h.getfield(box2, descr3) is box3
+        h.setfield(box4, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box4
+        assert h.getfield(box2, descr3) is box4
+
+    def test_replace_box_twice(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box5, descr1) is box2
+        assert h.getfield(box5, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+        h.setfield(box5, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box5
+        assert h.getfield(box2, descr3) is box5
 
     def test_replace_box_array(self):
         h = HeapCache()
@@ -291,9 +325,6 @@
         h.setarrayitem(box3, descr2, index2, box1)
         h.setarrayitem(box2, descr3, index2, box3)
         h.replace_box(box1, box4)
-        assert h.getarrayitem(box1, descr1, index1) is None
-        assert h.getarrayitem(box1, descr2, index1) is None
-        assert h.arraylen(box1) is None
         assert h.arraylen(box4) is lengthbox1
         assert h.getarrayitem(box4, descr1, index1) is box2
         assert h.getarrayitem(box4, descr2, index1) is box3
@@ -304,6 +335,27 @@
         h.replace_box(lengthbox1, lengthbox2)
         assert h.arraylen(box4) is lengthbox2
 
+    def test_replace_box_array_twice(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box5, descr1, index1) is box2
+        assert h.getarrayitem(box5, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box5
+        assert h.getarrayitem(box3, descr2, index2) is box5
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        h.replace_box(lengthbox2, lengthbox3)
+        assert h.arraylen(box4) is lengthbox3
+
     def test_ll_arraycopy(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -38,10 +38,19 @@
 	PyObject_VAR_HEAD
 } PyVarObject;
 
+#ifndef PYPY_DEBUG_REFCOUNT
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)ob))
 #define Py_DECREF(ob)   (Py_DecRef((PyObject *)ob))
 #define Py_XINCREF(ob)  (Py_IncRef((PyObject *)ob))
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)ob))
+#else
+#define Py_INCREF(ob)   (((PyObject *)ob)->ob_refcnt++)
+#define Py_DECREF(ob)  ((((PyObject *)ob)->ob_refcnt > 1) ? \
+			((PyObject *)ob)->ob_refcnt-- : (Py_DecRef((PyObject *)ob)))
+
+#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
+#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
+#endif
 
 #define Py_CLEAR(op)				\
         do {                            	\
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
     space.call_method(w_list, "reverse")
     return 0
 
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+    """Return a list of the objects in list containing the objects between low
+    and high.  Return NULL and set an exception if unsuccessful.  Analogous
+    to list[low:high].  Negative indices, as when slicing from Python, are not
+    supported."""
+    w_start = space.wrap(low)
+    w_stop = space.wrap(high)
+    return space.getslice(w_list, w_start, w_stop)
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyList_SetSlice(space, w_list, low, high, w_sequence):
     """Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
     This is the equivalent of the Python expression hash(o)."""
     return space.int_w(space.hash(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+    """Set a TypeError indicating that type(o) is not hashable and return -1.
+    This function receives special treatment when stored in a tp_hash slot,
+    allowing a type to explicitly indicate to the interpreter that it is not
+    hashable.
+    """
+    raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Dir(space, w_o):
     """This is equivalent to the Python expression dir(o), returning a (possibly
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
-    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc)
@@ -60,6 +60,16 @@
     args_w = space.fixedview(w_args)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
 
+def wrap_inquirypred(space, w_self, w_args, func):
+    func_inquiry = rffi.cast(inquiry, func)
+    check_num_args(space, w_args, 0)
+    args_w = space.fixedview(w_args)
+    res = generic_cpy_call(space, func_inquiry, w_self)
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
+    return space.wrap(bool(res))
+
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
     check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
         w_errors = space.wrap(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+    """Decode a string object by passing it to the codec registered
+    for encoding and return the result as Python object. encoding and
+    errors have the same meaning as the parameters of the same name in
+    the string encode() method.  The codec to be used is looked up
+    using the Python codec registry. Return NULL if an exception was
+    raised by the codec.
+
+    This function is not available in 3.x and does not have a PyBytes alias."""
+    if not PyString_Check(space, w_str):
+        PyErr_BadArgument(space)
+
+    w_encoding = w_errors = space.w_None
+    if encoding:
+        w_encoding = space.wrap(rffi.charp2str(encoding))
+    if errors:
+        w_errors = space.wrap(rffi.charp2str(errors))
+    return space.call_method(w_str, "decode", w_encoding, w_errors)
+
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
-    """Return a list of the objects in list containing the objects between low
-    and high.  Return NULL and set an exception if unsuccessful.  Analogous
-    to list[low:high].  Negative indices, as when slicing from Python, are not
-    supported.
-
-    This function used an int for low and high. This might
-    require changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([Py_ssize_t], PyObject)
 def PyLong_FromSsize_t(space, v):
     """Return a new PyLongObject object from a C Py_ssize_t, or
@@ -1606,15 +1595,6 @@
     for PyObject_Str()."""
     raise NotImplementedError
 
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
-    """Set a TypeError indicating that type(o) is not hashable and return -1.
-    This function receives special treatment when stored in a tp_hash slot,
-    allowing a type to explicitly indicate to the interpreter that it is not
-    hashable.
-    """
-    raise NotImplementedError
-
 @cpython_api([], PyFrameObject)
 def PyEval_GetFrame(space):
     """Return the current thread state's frame, which is NULL if no frame is
@@ -1737,17 +1717,6 @@
     changes in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
-    """Decode a string object by passing it to the codec registered for encoding and
-    return the result as Python object. encoding and errors have the same
-    meaning as the parameters of the same name in the string encode() method.
-    The codec to be used is looked up using the Python codec registry. Return NULL
-    if an exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Encode(space, s, size, encoding, errors):
     """Encode the char buffer of the given size by passing it to the codec
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
         w_t = api.PyList_AsTuple(w_l)
         assert space.unwrap(w_t) == (3, 2, 1)
 
+    def test_list_getslice(self, space, api):
+        w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+        w_s = api.PyList_GetSlice(w_l, 1, 5)
+        assert space.unwrap(w_s) == [2, 1]
+
 class AppTestListObject(AppTestCpythonExtensionBase):
     def test_listobject(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
+    def test_AsDecodedObject(self, space, api):
+        w_str = space.wrap('caf\xe9')
+        encoding = rffi.str2charp("latin-1")
+        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        rffi.free_charp(encoding)
+        assert space.unwrap(w_res) == u"caf\xe9"
+
     def test_eq(self, space, api):
         assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
         assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
+        
+    def test_bool(self):
+        module = self.import_extension('foo', [
+            ("newInt", "METH_VARARGS",
+             """
+                IntLikeObject *intObj;
+                long intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "i", &intval))
+                    return NULL;
+
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+                intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+                if (!intObj) {
+                    return NULL;
+                }
+
+                intObj->value = intval;
+                return (PyObject *)intObj;
+             """)],
+            """
+            typedef struct
+            {
+                PyObject_HEAD
+                int value;
+            } IntLikeObject;
+
+            static int
+            intlike_nb_nonzero(IntLikeObject *v)
+            {
+                if (v->value == -42) {
+                    PyErr_SetNone(PyExc_ValueError);
+                    return -1;
+                }
+                return v->value;
+            }
+
+            PyTypeObject IntLike_Type = {
+                PyObject_HEAD_INIT(0)
+                /*ob_size*/             0,
+                /*tp_name*/             "IntLike",
+                /*tp_basicsize*/        sizeof(IntLikeObject),
+            };
+            static PyNumberMethods intlike_as_number;
+            """)
+        assert not bool(module.newInt(0))
+        assert bool(module.newInt(1))
+        assert bool(module.newInt(-1))
+        raises(ValueError, bool, module.newInt(-42))
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -10,6 +10,7 @@
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
      scalar_w, W_NDimArray, array)
+from pypy.module.micronumpy.interp_arrayops import where
 from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize, instantiate
 
@@ -35,6 +36,7 @@
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring"]
 TWO_ARG_FUNCTIONS = ["dot", 'take']
+THREE_ARG_FUNCTIONS = ['where']
 
 class FakeSpace(object):
     w_ValueError = None
@@ -445,14 +447,25 @@
             arg = self.args[1].execute(interp)
             if not isinstance(arg, BaseArray):
                 raise ArgumentNotAnArray
-            if not isinstance(arg, BaseArray):
-                raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
             else:
                 assert False # unreachable code
+        elif self.name in THREE_ARG_FUNCTIONS:
+            if len(self.args) != 3:
+                raise ArgumentMismatch
+            arg1 = self.args[1].execute(interp)
+            arg2 = self.args[2].execute(interp)
+            if not isinstance(arg1, BaseArray):
+                raise ArgumentNotAnArray
+            if not isinstance(arg2, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "where":
+                w_res = where(interp.space, arr, arg1, arg2)
+            else:
+                assert False
         else:
             raise WrongFunctionName
         if isinstance(w_res, BaseArray):
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -270,3 +270,13 @@
         b -> 2
         """)
         assert interp.results[0].value == 3
+
+    def test_where(self):
+        interp = self.run('''
+        a = [1, 0, 3, 0]
+        b = [1, 1, 1, 1]
+        c = [0, 0, 0, 0]
+        d = where(a, b, c)
+        d -> 1
+        ''')
+        assert interp.results[0].value == 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1830,6 +1830,19 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
+    def test_array_indexing_bool_setitem_multidim(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
+
+    def test_array_indexing_bool_setitem_2(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a = a[::2]
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 5], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,6 @@
 
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_ufunc_instance(self):
         from _numpypy import add, ufunc
@@ -149,7 +148,11 @@
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+        # Since comparisons with nnan and nan all return false,
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
+
 
     def test_fmin(self):
         from _numpypy import fmin
@@ -165,7 +168,9 @@
         assert math.isnan(fmin(0, nan))
         assert math.isnan(fmin(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
     def test_fmod(self):
         from _numpypy import fmod
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -596,7 +596,7 @@
         import sys
         size = 0x14FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
@@ -618,7 +618,7 @@
         import sys
         size = 0x17FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assert=plain -rf
+addopts = --assert=reinterp -rf
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -40,7 +40,7 @@
 # In that case, do 5 bits at a time.  The potential drawback is that
 # a table of 2**5 intermediate results is computed.
 
-## FIVEARY_CUTOFF = 8   disabled for now
+FIVEARY_CUTOFF = 8
 
 
 def _mask_digit(x):
@@ -456,7 +456,7 @@
 
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
-        if 1:   ## b.numdigits() <= FIVEARY_CUTOFF:
+        if b.numdigits() <= FIVEARY_CUTOFF:
             # Left-to-right binary exponentiation (HAC Algorithm 14.79)
             # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
             i = b.numdigits() - 1
@@ -469,30 +469,51 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 i -= 1
-##        else:
-##            This code is disabled for now, because it assumes that
-##            SHIFT is a multiple of 5.  It could be fixed but it looks
-##            like it's more troubles than benefits...
-##
-##            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-##            # This is only useful in the case where c != None.
-##            # z still holds 1L
-##            table = [z] * 32
-##            table[0] = z
-##            for i in range(1, 32):
-##                table[i] = _help_mult(table[i-1], a, c)
-##            i = b.numdigits() - 1
-##            while i >= 0:
-##                bi = b.digit(i)
-##                j = SHIFT - 5
-##                while j >= 0:
-##                    index = (bi >> j) & 0x1f
-##                    for k in range(5):
-##                        z = _help_mult(z, z, c)
-##                    if index:
-##                        z = _help_mult(z, table[index], c)
-##                    j -= 5
-##                i -= 1
+        else:
+            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
+            # This is only useful in the case where c != None.
+            # z still holds 1L
+            table = [z] * 32
+            table[0] = z
+            for i in range(1, 32):
+                table[i] = _help_mult(table[i-1], a, c)
+            i = b.numdigits()
+            # Note that here SHIFT is not a multiple of 5.  The difficulty
+            # is to extract 5 bits at a time from 'b', starting from the
+            # most significant digits, so that at the end of the algorithm
+            # it falls exactly to zero.
+            # m  = max number of bits = i * SHIFT
+            # m+ = m rounded up to the next multiple of 5
+            # j  = (m+) % SHIFT = (m+) - (i * SHIFT)
+            # (computed without doing "i * SHIFT", which might overflow)
+            j = i % 5
+            if j != 0:
+                j = 5 - j
+            if not we_are_translated():
+                assert j == (i*SHIFT+4)//5*5 - i*SHIFT
+            #
+            accum = r_uint(0)
+            while True:
+                j -= 5
+                if j >= 0:
+                    index = (accum >> j) & 0x1f
+                else:
+                    # 'accum' does not have enough digit.
+                    # must get the next digit from 'b' in order to complete
+                    i -= 1
+                    if i < 0:
+                        break    # done
+                    bi = b.udigit(i)
+                    index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f
+                    accum = bi
+                    j += SHIFT
+                #
+                for k in range(5):
+                    z = _help_mult(z, z, c)
+                if index:
+                    z = _help_mult(z, table[index], c)
+            #
+            assert j == -5
 
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -1234,7 +1234,7 @@
             pos += 1
             continue
 
-        if 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
             # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
             pos += 1
             oc2 = ord(s[pos])
@@ -1350,6 +1350,20 @@
     pos = 0
     while pos < size:
         oc = ord(s[pos])
+
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+            # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
+            pos += 1
+            oc2 = ord(s[pos])
+
+            if 0xDC00 <= oc2 <= 0xDFFF:
+                ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000
+                raw_unicode_escape_helper(result, ucs)
+                pos += 1
+                continue
+            # Fall through: isolated surrogates are copied as-is
+            pos -= 1
+
         if oc < 0x100:
             result.append(chr(oc))
         else:
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint.py
@@ -379,6 +379,18 @@
         for n, expected in [(37, 9), (1291, 931), (67889, 39464)]:
             v = two.pow(t, rbigint.fromint(n))
             assert v.toint() == expected
+        #
+        # more tests, comparing against CPython's answer
+        enabled = sample(range(5*32), 10)
+        for i in range(5*32):
+            t = t.mul(two)      # add one random bit
+            if random() >= 0.5:
+                t = t.add(rbigint.fromint(1))
+            if i not in enabled:
+                continue    # don't take forever
+            n = randint(1, sys.maxint)
+            v = two.pow(t, rbigint.fromint(n))
+            assert v.toint() == pow(2, t.tolong(), n)
 
     def test_pow_lln(self):
         x = 10L
diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py
--- a/pypy/rlib/test/test_runicode.py
+++ b/pypy/rlib/test/test_runicode.py
@@ -728,3 +728,18 @@
 
         res = interpret(f, [0x10140])
         assert res == 0x10140
+
+    def test_encode_surrogate_pair(self):
+        u = runicode.UNICHR(0xD800) + runicode.UNICHR(0xDC00)
+        if runicode.MAXUNICODE < 65536:
+            # Narrow unicode build, consider utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+        else:
+            # Wide unicode build, don't merge utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'
diff --git a/pypy/tool/pytest/pypy_test_failure_demo.py b/pypy/tool/pytest/pypy_test_failure_demo.py
--- a/pypy/tool/pytest/pypy_test_failure_demo.py
+++ b/pypy/tool/pytest/pypy_test_failure_demo.py
@@ -8,6 +8,10 @@
 def test_interp_func(space): 
     assert space.is_true(space.w_None) 
 
+def test_interp_reinterpret(space):
+    a = 1
+    assert a == 2
+
 class TestInterpTest: 
     def test_interp_method(self): 
         assert self.space.is_true(self.space.w_False) 

From noreply at buildbot.pypy.org  Wed Apr 25 00:44:30 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Wed, 25 Apr 2012 00:44:30 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: assignment from str to
 std::string and __str__ for std::string
Message-ID: <20120424224430.B06A48208B@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54740:ad83dba9b24b
Date: 2012-04-24 13:33 -0700
http://bitbucket.org/pypy/pypy/changeset/ad83dba9b24b/

Log:	assignment from str to std::string and __str__ for std::string

diff --git a/pypy/module/cppyy/capi/__init__.py b/pypy/module/cppyy/capi/__init__.py
--- a/pypy/module/cppyy/capi/__init__.py
+++ b/pypy/module/cppyy/capi/__init__.py
@@ -430,6 +430,11 @@
     [C_OBJECT], C_OBJECT,
     threadsafe=threadsafe,
     compilation_info=backend.eci)
+c_assign2stdstring = rffi.llexternal(
+    "cppyy_assign2stdstring",
+    [C_OBJECT, rffi.CCHARP], lltype.Void,
+    threadsafe=threadsafe,
+    compilation_info=backend.eci)
 c_free_stdstring = rffi.llexternal(
     "cppyy_free_stdstring",
     [C_OBJECT], lltype.Void,
diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py
--- a/pypy/module/cppyy/converter.py
+++ b/pypy/module/cppyy/converter.py
@@ -643,6 +643,17 @@
            arg = InstanceConverter._unwrap_object(self, space, w_obj)
            return capi.c_stdstring2stdstring(arg)
 
+    def to_memory(self, space, w_obj, w_value, offset):
+        try:
+            address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset))
+            charp = rffi.str2charp(space.str_w(w_value))
+            capi.c_assign2stdstring(address, charp)
+            rffi.free_charp(charp)
+            return
+        except Exception:
+            pass
+        return InstanceConverter.to_memory(self, space, w_obj, w_value, offset)
+
     def free_argument(self, arg):
         capi.c_free_stdstring(rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, arg)[0]))
 
diff --git a/pypy/module/cppyy/include/capi.h b/pypy/module/cppyy/include/capi.h
--- a/pypy/module/cppyy/include/capi.h
+++ b/pypy/module/cppyy/include/capi.h
@@ -100,6 +100,7 @@
 
     cppyy_object_t cppyy_charp2stdstring(const char* str);
     cppyy_object_t cppyy_stdstring2stdstring(cppyy_object_t ptr);
+    void cppyy_assign2stdstring(cppyy_object_t ptr, const char* str);
     void cppyy_free_stdstring(cppyy_object_t ptr);
 
 #ifdef __cplusplus
diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -354,6 +354,7 @@
             else:
                 return self.c_str() == other
         pyclass.__eq__ = eq
+        pyclass.__str__ = pyclass.c_str
 
     # TODO: clean this up
     # fixup lack of __getitem__ if no const return
diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx
--- a/pypy/module/cppyy/src/cintcwrapper.cxx
+++ b/pypy/module/cppyy/src/cintcwrapper.cxx
@@ -761,6 +761,10 @@
     delete (std::string*)ptr;
 }
 
+void cppyy_assign2stdstring(cppyy_object_t ptr, const char* str) {
+   *((std::string*)ptr) = str;
+}
+
 void* cppyy_load_dictionary(const char* lib_name) {
     if (0 <= gSystem->Load(lib_name))
         return (void*)1;
diff --git a/pypy/module/cppyy/src/reflexcwrapper.cxx b/pypy/module/cppyy/src/reflexcwrapper.cxx
--- a/pypy/module/cppyy/src/reflexcwrapper.cxx
+++ b/pypy/module/cppyy/src/reflexcwrapper.cxx
@@ -520,6 +520,10 @@
     return (cppyy_object_t)new std::string(*(std::string*)ptr);
 }
 
+void cppyy_assign2stdstring(cppyy_object_t ptr, const char* str) {
+   *((std::string*)ptr) = str;
+}
+
 void cppyy_free_stdstring(cppyy_object_t ptr) {
     delete (std::string*)ptr;
 }
diff --git a/pypy/module/cppyy/test/test_stltypes.py b/pypy/module/cppyy/test/test_stltypes.py
--- a/pypy/module/cppyy/test/test_stltypes.py
+++ b/pypy/module/cppyy/test/test_stltypes.py
@@ -244,25 +244,27 @@
 
         raises(TypeError, c.get_string2, "temp string")
 
-    def test02_string_data_ccess(self):
+    def test02_string_data_access(self):
         """Test access to std::string object data members"""
 
         import cppyy
         std = cppyy.gbl.std
         stringy_class = cppyy.gbl.stringy_class
 
-        return
-
         c, s = stringy_class(""), std.string("test string")
 
-        c.m_string = s
-        assert c.m_string == s
-        assert c.get_string1() == s
-
         c.m_string = "another test"
         assert c.m_string == "another test"
+        assert str(c.m_string) == c.m_string
         assert c.get_string1() == "another test"
 
+        return
+        # TODO: assignment from object
+        c.m_string = s
+        assert str(c.m_string) == s
+        assert c.m_string == s
+        assert c.get_string1() == s
+
     def test03_string_with_null_character(self):
         """Test that strings with NULL do not get truncated"""
 

From noreply at buildbot.pypy.org  Wed Apr 25 00:44:31 2012
From: noreply at buildbot.pypy.org (wlav)
Date: Wed, 25 Apr 2012 00:44:31 +0200 (CEST)
Subject: [pypy-commit] pypy reflex-support: doc-strings for global functions
Message-ID: <20120424224431.EA86A820B1@wyvern.cs.uni-duesseldorf.de>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: reflex-support
Changeset: r54741:2162bc565f20
Date: 2012-04-24 13:39 -0700
http://bitbucket.org/pypy/pypy/changeset/2162bc565f20/

Log:	doc-strings for global functions

diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py
--- a/pypy/module/cppyy/pythonify.py
+++ b/pypy/module/cppyy/pythonify.py
@@ -52,6 +52,7 @@
     def function(*args):
         return cppol.call(None, *args)
     function.__name__ = func_name
+    function.__doc__ = cppol.signature()
     return staticmethod(function)
 
 def make_method(meth_name, cppol):
diff --git a/pypy/module/cppyy/test/fragile.cxx b/pypy/module/cppyy/test/fragile.cxx
--- a/pypy/module/cppyy/test/fragile.cxx
+++ b/pypy/module/cppyy/test/fragile.cxx
@@ -5,3 +5,7 @@
 }
 
 fragile::I fragile::gI;
+
+void fragile::fglobal(int, double, char) {
+    /* empty; only used for doc-string testing */
+}
diff --git a/pypy/module/cppyy/test/fragile.h b/pypy/module/cppyy/test/fragile.h
--- a/pypy/module/cppyy/test/fragile.h
+++ b/pypy/module/cppyy/test/fragile.h
@@ -75,4 +75,6 @@
     int method1(int, double) { return 0; }
 };
 
+void fglobal(int, double, char);
+
 } // namespace fragile
diff --git a/pypy/module/cppyy/test/fragile.xml b/pypy/module/cppyy/test/fragile.xml
--- a/pypy/module/cppyy/test/fragile.xml
+++ b/pypy/module/cppyy/test/fragile.xml
@@ -6,4 +6,6 @@
 
   <variable name="fragile::gI" />
 
+  <function pattern="fragile::f*" />
+
 </lcgdict>
diff --git a/pypy/module/cppyy/test/fragile_LinkDef.h b/pypy/module/cppyy/test/fragile_LinkDef.h
--- a/pypy/module/cppyy/test/fragile_LinkDef.h
+++ b/pypy/module/cppyy/test/fragile_LinkDef.h
@@ -19,4 +19,6 @@
 
 #pragma link C++ variable fragile::gI;
 
+#pragma link C++ function fragile::fglobal;
+
 #endif
diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py
--- a/pypy/module/cppyy/test/test_fragile.py
+++ b/pypy/module/cppyy/test/test_fragile.py
@@ -191,3 +191,6 @@
         j = fragile.J()
         assert fragile.J.method1.__doc__ == j.method1.__doc__
         assert j.method1.__doc__ == "int fragile::J::method1(int, double)"
+
+        f = fragile.fglobal
+        assert f.__doc__ == "void fragile::fglobal(int, double, char)"

From noreply at buildbot.pypy.org  Wed Apr 25 11:56:41 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 25 Apr 2012 11:56:41 +0200 (CEST)
Subject: [pypy-commit] jitviewer default: bold new
Message-ID: <20120425095641.74EA882009@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r200:13e1f8c97ca7
Date: 2012-04-25 11:56 +0200
http://bitbucket.org/pypy/jitviewer/changeset/13e1f8c97ca7/

Log:	bold new

diff --git a/_jitviewer/parser.py b/_jitviewer/parser.py
--- a/_jitviewer/parser.py
+++ b/_jitviewer/parser.py
@@ -31,6 +31,8 @@
             return "single-operation guard"
         elif 'call' in self.name:
             return "single-operation call"
+        elif self.name.startswith('new'):
+            return 'single-operation new'
         else:
             return "single-operation"
 
diff --git a/_jitviewer/static/style.css b/_jitviewer/static/style.css
--- a/_jitviewer/static/style.css
+++ b/_jitviewer/static/style.css
@@ -178,6 +178,10 @@
     font-size: 12px;
     font-weight: bold;
 }
+.new {
+    font-size: 12px;
+    font-weight: bold;
+}
 span.dmp,
 span.guard,
 span.call,

From noreply at buildbot.pypy.org  Wed Apr 25 12:22:00 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Apr 2012 12:22:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Shut off a C compiler warning.
Message-ID: <20120425102200.2858C82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54743:a67352890558
Date: 2012-04-25 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/a67352890558/

Log:	Shut off a C compiler warning.

diff --git a/pypy/translator/c/src/cjkcodecs/cjkcodecs.h b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
--- a/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
@@ -210,15 +210,15 @@
 
 #define BEGIN_CODECS_LIST /* empty */
 #define _CODEC(name)                                                    \
-  static const MultibyteCodec _pypy_cjkcodec_##name;                    \
-  const MultibyteCodec *pypy_cjkcodec_##name(void) {                    \
+  static MultibyteCodec _pypy_cjkcodec_##name;                          \
+  MultibyteCodec *pypy_cjkcodec_##name(void) {                          \
     if (_pypy_cjkcodec_##name.codecinit != NULL) {                      \
       int r = _pypy_cjkcodec_##name.codecinit(_pypy_cjkcodec_##name.config); \
       assert(r == 0);                                                   \
     }                                                                   \
     return &_pypy_cjkcodec_##name;                                      \
   }                                                                     \
-  static const MultibyteCodec _pypy_cjkcodec_##name
+  static MultibyteCodec _pypy_cjkcodec_##name
 #define _STATEFUL_METHODS(enc)          \
     enc##_encode,                       \
     enc##_encode_init,                  \
diff --git a/pypy/translator/c/src/cjkcodecs/multibytecodec.h b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.h
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
@@ -131,7 +131,7 @@
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    const MultibyteCodec *pypy_cjkcodec_##name(void);
+    MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)

From noreply at buildbot.pypy.org  Wed Apr 25 12:22:01 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Apr 2012 12:22:01 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Ah bah.
Message-ID: <20120425102201.6800582009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54744:4920a633a5b5
Date: 2012-04-25 12:02 +0200
http://bitbucket.org/pypy/pypy/changeset/4920a633a5b5/

Log:	Ah bah.

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -120,16 +120,11 @@
         ec._transaction_pending.append(self)
 
     def run(self):
+        #if self.retry_counter > 0:
+        #    # retrying: will be done later, try others first
+        #    return [self]     # XXX does not work, will be retried immediately
+        #
         ec = self.space.getexecutioncontext()    # create it if needed
-        #
-        if self.retry_counter > 0:
-            # Note that even in this case we have to ensure the ec is
-            # created first.  Otherwise, if the first transaction of a
-            # thread is retrying, the creation of the ec is cancelled
-            # and we end up in register() with no ec...
-            self.register() # retrying: will be done later, try others first
-            return
-        #
         assert len(ec._transaction_pending) == 0
         #
         self.space.call_args(self.w_callback, self.args)

From noreply at buildbot.pypy.org  Wed Apr 25 12:22:02 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Apr 2012 12:22:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: An additional test (that fails on
 CPython too right now).
Message-ID: <20120425102202.A072F82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54745:27558fbd3171
Date: 2012-04-25 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/27558fbd3171/

Log:	An additional test (that fails on CPython too right now). Fix by
	disabling the optimization of u'foo'[constant].

diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -304,14 +304,19 @@
                     # produce compatible pycs.
                     if (self.space.isinstance_w(w_obj, self.space.w_unicode) and
                         self.space.isinstance_w(w_const, self.space.w_unicode)):
-                        unistr = self.space.unicode_w(w_const)
-                        if len(unistr) == 1:
-                            ch = ord(unistr[0])
-                        else:
-                            ch = 0
-                        if (ch > 0xFFFF or
-                            (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
-                            return subs
+                        #unistr = self.space.unicode_w(w_const)
+                        #if len(unistr) == 1:
+                        #    ch = ord(unistr[0])
+                        #else:
+                        #    ch = 0
+                        #if (ch > 0xFFFF or
+                        #    (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
+                        # --XXX-- for now we always disable optimization of
+                        # u'...'[constant] because the tests above are not
+                        # enough to fix issue5057 (CPython has the same
+                        # problem as of April 24, 2012).
+                        # See test_const_fold_unicode_subscr
+                        return subs
 
                     return ast.Const(w_const, subs.lineno, subs.col_offset)
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -844,7 +844,8 @@
         return u"abc"[0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        if 0:   # xxx later?
+            assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
 
         # getitem outside of the BMP should not be optimized
         source = """def f():
@@ -854,12 +855,20 @@
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
                           ops.RETURN_VALUE: 1}
 
+        source = """def f():
+        return u"\U00012345abcdef"[3]
+        """
+        counts = self.count_instructions(source)
+        assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
+                          ops.RETURN_VALUE: 1}
+
         monkeypatch.setattr(optimize, "MAXUNICODE", 0xFFFF)
         source = """def f():
         return u"\uE01F"[0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        if 0:   # xxx later?
+            assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
         monkeypatch.undo()
 
         # getslice is not yet optimized.

From noreply at buildbot.pypy.org  Wed Apr 25 12:22:03 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 25 Apr 2012 12:22:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120425102203.EE2EA82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54746:0585f40cf68b
Date: 2012-04-25 12:15 +0200
http://bitbucket.org/pypy/pypy/changeset/0585f40cf68b/

Log:	merge heads

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -80,7 +80,7 @@
         void SetMyInt(int i) { m_myint = i; }
 
     public:
-       int m_myint;
+        int m_myint;
     };
 
 Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
@@ -111,6 +111,121 @@
 That's all there is to it!
 
 
+Advanced example
+================
+The following snippet of C++ is very contrived, to allow showing that such
+pathological code can be handled and to show how certain features play out in
+practice::
+
+    $ cat MyAdvanced.h
+    #include <string>
+
+    class Base1 {
+    public:
+        Base1(int i) : m_i(i) {}
+        virtual ~Base1() {}
+        int m_i;
+    };
+
+    class Base2 {
+    public:
+        Base2(double d) : m_d(d) {}
+        virtual ~Base2() {}
+        double m_d;
+    };
+
+    class C;
+
+    class Derived : public virtual Base1, public virtual Base2 {
+    public:
+        Derived(const std::string& name, int i, double d) : Base1(i), Base2(d), m_name(name) {}
+        virtual C* gimeC() { return (C*)0; }
+        std::string m_name;
+    };
+
+    Base1* BaseFactory(const std::string& name, int i, double d) {
+        return new Derived(name, i, d);
+    }
+
+This code is still only in a header file, with all functions inline, for
+convenience of the example.
+If the implementations live in a separate source file or shared library, the
+only change needed is to link those in when building the reflection library.
+
+If you were to run ``genreflex`` like above in the basic example, you will
+find that not all classes of interest will be reflected, nor will be the
+global factory function.
+In particular, ``std::string`` will be missing, since it is not defined in
+this header file, but in a header file that is included.
+In practical terms, general classes such as ``std::string`` should live in a
+core reflection set, but for the moment assume we want to have it in the
+reflection library that we are building for this example.
+
+The ``genreflex`` script can be steered using a so-called `selection file`_,
+which is a simple XML file specifying, either explicitly or by using a
+pattern, which classes, variables, namespaces, etc. to select from the given
+header file.
+With the aid of a selection file, a large project can be easily managed:
+simply ``#include`` all relevant headers into a single header file that is
+handed to ``genreflex``.
+Then, apply a selection file to pick up all the relevant classes.
+For our purposes, the following rather straightforward selection will do
+(the name ``lcgdict`` for the root is historical, but required)::
+
+    $ cat MyAdvanced.xml
+    <lcgdict>
+        <class pattern="Base?" />
+        <class name="Derived" />
+        <class name="std::string" />
+        <function name="BaseFactory" />
+    </lcgdict>
+
+.. _`selection file`: http://root.cern.ch/drupal/content/generating-reflex-dictionaries
+
+Now the reflection info can be generated and compiled::
+
+    $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so
+
+and subsequently be used from PyPy::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libAdvExDict.so")
+    <CPPLibrary object at 0x00007fdb48fc8120>
+    >>>> d = cppyy.gbl.BaseFactory("name", 42, 3.14)
+    >>>> type(d)
+    <class '__main__.Derived'>
+    >>>> d.m_i
+    42
+    >>>> d.m_d
+    3.14
+    >>>> d.m_name == "name"
+    True
+    >>>>
+
+Again, that's all there is to it!
+
+A couple of things to note, though.
+If you look back at the C++ definition of the ``BaseFactory`` function,
+you will see that it declares the return type to be a ``Base1``, yet the
+bindings return an object of the actual type ``Derived``?
+This choice is made for a couple of reasons.
+First, it makes method dispatching easier: if bound objects are always their
+most derived type, then it is easy to calculate any offsets, if necessary.
+Second, it makes memory management easier: the combination of the type and
+the memory address uniquely identifies an object.
+That way, it can be recycled and object identity can be maintained if it is
+entered as a function argument into C++ and comes back to PyPy as a return
+value.
+Last, but not least, casting is decidedly unpythonistic.
+By always providing the most derived type known, casting becomes unnecessary.
+For example, the data member of ``Base2`` is simply directly available.
+Note also that the unreflected ``gimeC`` method of ``Derived`` does not
+preclude its use.
+It is only the ``gimeC`` method that is unusable as long as class ``C`` is
+unknown to the system.
+
+
 Features
 ========
 
@@ -160,6 +275,8 @@
 * **doc strings**: The doc string of a method or function contains the C++
   arguments and return types of all overloads of that name, as applicable.
 
+* **enums**: Are translated as ints with no further checking.
+
 * **functions**: Work as expected and live in their appropriate namespace
   (which can be the global one, ``cppyy.gbl``).
 
@@ -236,6 +353,9 @@
   using classes that themselves are templates (etc.) in the arguments.
   All classes must already exist in the loaded reflection info.
 
+* **typedefs**: Are simple python references to the actual classes to which
+  they refer.
+
 * **unary operators**: Are supported if a python equivalent exists, and if the
   operator is defined in the C++ class.
 
@@ -253,6 +373,107 @@
 Only that one specific method can not be used.
 
 
+Templates
+=========
+
+A bit of special care needs to be taken for the use of templates.
+For a templated class to be completely available, it must be guaranteed that
+said class is fully instantiated, and hence all executable C++ code is
+generated and compiled in.
+The easiest way to fulfill that guarantee is by explicit instantiation in the
+header file that is handed to ``genreflex``.
+The following example should make that clear::
+
+    $ cat MyTemplate.h
+    #include <vector>
+
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_i(i) {}
+        MyClass(const MyClass& s) : m_i(s.m_i) {}
+        MyClass& operator=(const MyClass& s) { m_i = s.m_i; return *this; }
+        ~MyClass() {}
+        int m_i;
+    };
+
+    template class std::vector<MyClass>;
+
+If you know for certain that all symbols will be linked in from other sources,
+you can also declare the explicit template instantiation ``extern``.
+
+Unfortunately, this is not enough for gcc.
+The iterators, if they are going to be used, need to be instantiated as well,
+as do the comparison operators on those iterators, as these live in an
+internal namespace, rather than in the iterator classes.
+One way to handle this, is to deal with this once in a macro, then reuse that
+macro for all ``vector`` classes.
+Thus, the header above needs this, instead of just the explicit instantiation
+of the ``vector<MyClass>``::
+
+    #define STLTYPES_EXPLICIT_INSTANTIATION_DECL(STLTYPE, TTYPE)                      \
+    template class std::STLTYPE< TTYPE >;                                             \
+    template class __gnu_cxx::__normal_iterator<TTYPE*, std::STLTYPE< TTYPE > >;      \
+    template class __gnu_cxx::__normal_iterator<const TTYPE*, std::STLTYPE< TTYPE > >;\
+    namespace __gnu_cxx {                                                             \
+    template bool operator==(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    template bool operator!=(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    }
+
+    STLTYPES_EXPLICIT_INSTANTIATION_DECL(vector, MyClass)
+
+Then, still for gcc, the selection file needs to contain the full hierarchy as
+well as the global overloads for comparisons for the iterators::
+
+    $ cat MyTemplate.xml
+    <lcgdict>
+        <class pattern="std::vector<*>" />
+        <class pattern="__gnu_cxx::__normal_iterator<*>" />
+        <class pattern="__gnu_cxx::new_allocator<*>" />
+        <class pattern="std::_Vector_base<*>" />
+        <class pattern="std::_Vector_base<*>::_Vector_impl" />
+        <class pattern="std::allocator<*>" />
+        <function name="__gnu_cxx::operator=="/>
+        <function name="__gnu_cxx::operator!="/>
+
+        <class name="MyClass" />
+    </lcgdict>
+
+Run the normal ``genreflex`` and compilation steps::
+
+    $ genreflex MyTemplate.h --selection=MyTemplate.xm
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so
+
+Note: this is a dirty corner that clearly could do with some automation,
+even if the macro already helps.
+Such automation is planned.
+In fact, in the cling world, the backend can perform the template
+instantations and generate the reflection info on the fly, and none of the
+above will any longer be necessary.
+
+Subsequent use should be as expected.
+Note the meta-class style of "instantiating" the template::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libTemplateDict.so")
+    >>>> std = cppyy.gbl.std
+    >>>> MyClass = cppyy.gbl.MyClass
+    >>>> v = std.vector(MyClass)()
+    >>>> v += [MyClass(1), MyClass(2), MyClass(3)]
+    >>>> for m in v:
+    ....     print m.m_i,
+    ....
+    1 2 3
+    >>>>
+
+Other templates work similarly.
+The arguments to the template instantiation can either be a string with the
+full list of arguments, or the explicit classes.
+The latter makes for easier code writing if the classes passed to the
+instantiation are themselves templates.
+
+
 The fast lane
 =============
 
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -24,7 +24,8 @@
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
 (32 bits, 64 bits) is automatically selected.  A 32 bit build can only be built
-using a 32 bit Python and vice versa.
+using a 32 bit Python and vice versa. By default pypy is built using the 
+Multi-threaded DLL (/MD) runtime environment.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
 will fail in this case.
@@ -102,10 +103,12 @@
 
 Download the source code of expat on sourceforge:
 http://sourceforge.net/projects/expat/ and extract it in the base
-directory.  Then open the project file ``expat.dsw`` with Visual
+directory.  Version 2.1.0 is known to pass tests. Then open the project 
+file ``expat.dsw`` with Visual
 Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, and build the solution (the
-``expat`` project is actually enough for pypy).
+switch to the "Release" configuration, reconfigure the runtime for 
+Multi-threaded DLL (/MD) and build the solution (the ``expat`` project 
+is actually enough for pypy).
 
 Then, copy the file ``win32\bin\release\libexpat.dll`` somewhere in
 your PATH.

From noreply at buildbot.pypy.org  Wed Apr 25 15:19:39 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 25 Apr 2012 15:19:39 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: promote the shape of
 a tuple on a read when the index is constant
Message-ID: <20120425131939.2A5E282009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54747:e0f2cfe66872
Date: 2012-04-25 09:19 -0400
http://bitbucket.org/pypy/pypy/changeset/e0f2cfe66872/

Log:	promote the shape of a tuple on a read when the index is constant

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -496,7 +496,6 @@
     def rewrite_op_hint(self, op):
         hints = op.args[1].value
         if hints.get('promote') and op.args[0].concretetype is not lltype.Void:
-            assert op.args[0].concretetype != lltype.Ptr(rstr.STR)
             kind = getkind(op.args[0].concretetype)
             op0 = SpaceOperation('-live-', [], None)
             op1 = SpaceOperation('%s_guard_value' % kind, [op.args[0]], None)
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -49,6 +49,8 @@
     def getitem(self, space, i):
         from pypy.objspace.std.tupletype import read_obj
 
+        if jit.isconstant(i):
+            jit.promote(self.tuplestorage.getshape())
         return read_obj(space, self.tuplestorage, i)
 
 registerimplementation(W_TupleObject)

From noreply at buildbot.pypy.org  Wed Apr 25 15:57:38 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 25 Apr 2012 15:57:38 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: unroll this sometimes
Message-ID: <20120425135738.1B94182009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54748:41a3b45ba449
Date: 2012-04-25 09:57 -0400
http://bitbucket.org/pypy/pypy/changeset/41a3b45ba449/

Log:	unroll this sometimes

diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -34,6 +34,9 @@
     def __init__(self, tuplestorage):
         self.tuplestorage = tuplestorage
 
+    @jit.look_inside_iff(lambda self, space:
+        jit.isvirtual(self) or (jit.isconstant(self.length()) and self.length < UNROLL_TUPLE_LIMIT)
+    )
     def tolist(self, space):
         items_w = [None] * self.length()
         for i in xrange(self.length()):

From noreply at buildbot.pypy.org  Wed Apr 25 16:22:37 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 25 Apr 2012 16:22:37 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: promote the shape for
 constant index lookups before we do anything with the shape,
 so it can be constant folded
Message-ID: <20120425142237.1D1DC82009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54749:7dadce969c5e
Date: 2012-04-25 10:22 -0400
http://bitbucket.org/pypy/pypy/changeset/7dadce969c5e/

Log:	promote the shape for constant index lookups before we do anything
	with the shape, so it can be constant folded

diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -46,14 +46,17 @@
     def getitems_copy(self, space):
         return self.tolist(space)[:]
 
+    def _promote_for_read(self, idx):
+        if jit.isconstant(idx):
+            jit.promote(self.tuplestorage.getshape())
+
     def length(self):
         return self.tuplestorage.getlength()
 
     def getitem(self, space, i):
         from pypy.objspace.std.tupletype import read_obj
 
-        if jit.isconstant(i):
-            jit.promote(self.tuplestorage.getshape())
+        self._promote_for_read(i)
         return read_obj(space, self.tuplestorage, i)
 
 registerimplementation(W_TupleObject)
@@ -64,6 +67,7 @@
 
 def getitem__Tuple_ANY(space, w_tuple, w_index):
     index = space.getindex_w(w_index, space.w_IndexError, "tuple index")
+    w_tuple._promote_for_read(index)
     if index < 0:
         index += w_tuple.length()
     if not (0 <= index < w_tuple.length()):

From noreply at buildbot.pypy.org  Wed Apr 25 17:27:07 2012
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 25 Apr 2012 17:27:07 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: typo
Message-ID: <20120425152707.EE7A482009@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: dynamic-specialized-tuple
Changeset: r54750:2a5a921a1519
Date: 2012-04-25 11:26 -0400
http://bitbucket.org/pypy/pypy/changeset/2a5a921a1519/

Log:	typo

diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -35,7 +35,7 @@
         self.tuplestorage = tuplestorage
 
     @jit.look_inside_iff(lambda self, space:
-        jit.isvirtual(self) or (jit.isconstant(self.length()) and self.length < UNROLL_TUPLE_LIMIT)
+        jit.isvirtual(self) or (jit.isconstant(self.length()) and self.length() < UNROLL_TUPLE_LIMIT)
     )
     def tolist(self, space):
         items_w = [None] * self.length()

From noreply at buildbot.pypy.org  Wed Apr 25 17:57:31 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 25 Apr 2012 17:57:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: avoid obscure crashes with "has no
	attribute name"
Message-ID: <20120425155731.EA40882009@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54751:290bebb4a80b
Date: 2012-04-25 17:56 +0200
http://bitbucket.org/pypy/pypy/changeset/290bebb4a80b/

Log:	avoid obscure crashes with "has no attribute name"

diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1167,7 +1167,7 @@
         try:
             return self._lookup_adtmeth(field_name)
         except AttributeError:
-            raise AttributeError("%r instance has no field %r" % (self._T._name,
+            raise AttributeError("%r instance has no field %r" % (self._T,
                                                                   field_name))
 
     def __setattr__(self, field_name, val):

From noreply at buildbot.pypy.org  Wed Apr 25 17:57:33 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 25 Apr 2012 17:57:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120425155733.9EAAE82009@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54752:6dffe8f51e7b
Date: 2012-04-25 17:56 +0200
http://bitbucket.org/pypy/pypy/changeset/6dffe8f51e7b/

Log:	merge

diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -80,7 +80,7 @@
         void SetMyInt(int i) { m_myint = i; }
 
     public:
-       int m_myint;
+        int m_myint;
     };
 
 Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
@@ -111,6 +111,121 @@
 That's all there is to it!
 
 
+Advanced example
+================
+The following snippet of C++ is very contrived, to allow showing that such
+pathological code can be handled and to show how certain features play out in
+practice::
+
+    $ cat MyAdvanced.h
+    #include <string>
+
+    class Base1 {
+    public:
+        Base1(int i) : m_i(i) {}
+        virtual ~Base1() {}
+        int m_i;
+    };
+
+    class Base2 {
+    public:
+        Base2(double d) : m_d(d) {}
+        virtual ~Base2() {}
+        double m_d;
+    };
+
+    class C;
+
+    class Derived : public virtual Base1, public virtual Base2 {
+    public:
+        Derived(const std::string& name, int i, double d) : Base1(i), Base2(d), m_name(name) {}
+        virtual C* gimeC() { return (C*)0; }
+        std::string m_name;
+    };
+
+    Base1* BaseFactory(const std::string& name, int i, double d) {
+        return new Derived(name, i, d);
+    }
+
+This code is still only in a header file, with all functions inline, for
+convenience of the example.
+If the implementations live in a separate source file or shared library, the
+only change needed is to link those in when building the reflection library.
+
+If you were to run ``genreflex`` like above in the basic example, you will
+find that not all classes of interest will be reflected, nor will be the
+global factory function.
+In particular, ``std::string`` will be missing, since it is not defined in
+this header file, but in a header file that is included.
+In practical terms, general classes such as ``std::string`` should live in a
+core reflection set, but for the moment assume we want to have it in the
+reflection library that we are building for this example.
+
+The ``genreflex`` script can be steered using a so-called `selection file`_,
+which is a simple XML file specifying, either explicitly or by using a
+pattern, which classes, variables, namespaces, etc. to select from the given
+header file.
+With the aid of a selection file, a large project can be easily managed:
+simply ``#include`` all relevant headers into a single header file that is
+handed to ``genreflex``.
+Then, apply a selection file to pick up all the relevant classes.
+For our purposes, the following rather straightforward selection will do
+(the name ``lcgdict`` for the root is historical, but required)::
+
+    $ cat MyAdvanced.xml
+    <lcgdict>
+        <class pattern="Base?" />
+        <class name="Derived" />
+        <class name="std::string" />
+        <function name="BaseFactory" />
+    </lcgdict>
+
+.. _`selection file`: http://root.cern.ch/drupal/content/generating-reflex-dictionaries
+
+Now the reflection info can be generated and compiled::
+
+    $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so
+
+and subsequently be used from PyPy::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libAdvExDict.so")
+    <CPPLibrary object at 0x00007fdb48fc8120>
+    >>>> d = cppyy.gbl.BaseFactory("name", 42, 3.14)
+    >>>> type(d)
+    <class '__main__.Derived'>
+    >>>> d.m_i
+    42
+    >>>> d.m_d
+    3.14
+    >>>> d.m_name == "name"
+    True
+    >>>>
+
+Again, that's all there is to it!
+
+A couple of things to note, though.
+If you look back at the C++ definition of the ``BaseFactory`` function,
+you will see that it declares the return type to be a ``Base1``, yet the
+bindings return an object of the actual type ``Derived``?
+This choice is made for a couple of reasons.
+First, it makes method dispatching easier: if bound objects are always their
+most derived type, then it is easy to calculate any offsets, if necessary.
+Second, it makes memory management easier: the combination of the type and
+the memory address uniquely identifies an object.
+That way, it can be recycled and object identity can be maintained if it is
+entered as a function argument into C++ and comes back to PyPy as a return
+value.
+Last, but not least, casting is decidedly unpythonistic.
+By always providing the most derived type known, casting becomes unnecessary.
+For example, the data member of ``Base2`` is simply directly available.
+Note also that the unreflected ``gimeC`` method of ``Derived`` does not
+preclude its use.
+It is only the ``gimeC`` method that is unusable as long as class ``C`` is
+unknown to the system.
+
+
 Features
 ========
 
@@ -160,6 +275,8 @@
 * **doc strings**: The doc string of a method or function contains the C++
   arguments and return types of all overloads of that name, as applicable.
 
+* **enums**: Are translated as ints with no further checking.
+
 * **functions**: Work as expected and live in their appropriate namespace
   (which can be the global one, ``cppyy.gbl``).
 
@@ -236,6 +353,9 @@
   using classes that themselves are templates (etc.) in the arguments.
   All classes must already exist in the loaded reflection info.
 
+* **typedefs**: Are simple python references to the actual classes to which
+  they refer.
+
 * **unary operators**: Are supported if a python equivalent exists, and if the
   operator is defined in the C++ class.
 
@@ -253,6 +373,107 @@
 Only that one specific method can not be used.
 
 
+Templates
+=========
+
+A bit of special care needs to be taken for the use of templates.
+For a templated class to be completely available, it must be guaranteed that
+said class is fully instantiated, and hence all executable C++ code is
+generated and compiled in.
+The easiest way to fulfill that guarantee is by explicit instantiation in the
+header file that is handed to ``genreflex``.
+The following example should make that clear::
+
+    $ cat MyTemplate.h
+    #include <vector>
+
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_i(i) {}
+        MyClass(const MyClass& s) : m_i(s.m_i) {}
+        MyClass& operator=(const MyClass& s) { m_i = s.m_i; return *this; }
+        ~MyClass() {}
+        int m_i;
+    };
+
+    template class std::vector<MyClass>;
+
+If you know for certain that all symbols will be linked in from other sources,
+you can also declare the explicit template instantiation ``extern``.
+
+Unfortunately, this is not enough for gcc.
+The iterators, if they are going to be used, need to be instantiated as well,
+as do the comparison operators on those iterators, as these live in an
+internal namespace, rather than in the iterator classes.
+One way to handle this, is to deal with this once in a macro, then reuse that
+macro for all ``vector`` classes.
+Thus, the header above needs this, instead of just the explicit instantiation
+of the ``vector<MyClass>``::
+
+    #define STLTYPES_EXPLICIT_INSTANTIATION_DECL(STLTYPE, TTYPE)                      \
+    template class std::STLTYPE< TTYPE >;                                             \
+    template class __gnu_cxx::__normal_iterator<TTYPE*, std::STLTYPE< TTYPE > >;      \
+    template class __gnu_cxx::__normal_iterator<const TTYPE*, std::STLTYPE< TTYPE > >;\
+    namespace __gnu_cxx {                                                             \
+    template bool operator==(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    template bool operator!=(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    }
+
+    STLTYPES_EXPLICIT_INSTANTIATION_DECL(vector, MyClass)
+
+Then, still for gcc, the selection file needs to contain the full hierarchy as
+well as the global overloads for comparisons for the iterators::
+
+    $ cat MyTemplate.xml
+    <lcgdict>
+        <class pattern="std::vector<*>" />
+        <class pattern="__gnu_cxx::__normal_iterator<*>" />
+        <class pattern="__gnu_cxx::new_allocator<*>" />
+        <class pattern="std::_Vector_base<*>" />
+        <class pattern="std::_Vector_base<*>::_Vector_impl" />
+        <class pattern="std::allocator<*>" />
+        <function name="__gnu_cxx::operator=="/>
+        <function name="__gnu_cxx::operator!="/>
+
+        <class name="MyClass" />
+    </lcgdict>
+
+Run the normal ``genreflex`` and compilation steps::
+
+    $ genreflex MyTemplate.h --selection=MyTemplate.xm
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so
+
+Note: this is a dirty corner that clearly could do with some automation,
+even if the macro already helps.
+Such automation is planned.
+In fact, in the cling world, the backend can perform the template
+instantations and generate the reflection info on the fly, and none of the
+above will any longer be necessary.
+
+Subsequent use should be as expected.
+Note the meta-class style of "instantiating" the template::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libTemplateDict.so")
+    >>>> std = cppyy.gbl.std
+    >>>> MyClass = cppyy.gbl.MyClass
+    >>>> v = std.vector(MyClass)()
+    >>>> v += [MyClass(1), MyClass(2), MyClass(3)]
+    >>>> for m in v:
+    ....     print m.m_i,
+    ....
+    1 2 3
+    >>>>
+
+Other templates work similarly.
+The arguments to the template instantiation can either be a string with the
+full list of arguments, or the explicit classes.
+The latter makes for easier code writing if the classes passed to the
+instantiation are themselves templates.
+
+
 The fast lane
 =============
 
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -24,7 +24,8 @@
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
 (32 bits, 64 bits) is automatically selected.  A 32 bit build can only be built
-using a 32 bit Python and vice versa.
+using a 32 bit Python and vice versa. By default pypy is built using the 
+Multi-threaded DLL (/MD) runtime environment.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
 will fail in this case.
@@ -102,10 +103,12 @@
 
 Download the source code of expat on sourceforge:
 http://sourceforge.net/projects/expat/ and extract it in the base
-directory.  Then open the project file ``expat.dsw`` with Visual
+directory.  Version 2.1.0 is known to pass tests. Then open the project 
+file ``expat.dsw`` with Visual
 Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, and build the solution (the
-``expat`` project is actually enough for pypy).
+switch to the "Release" configuration, reconfigure the runtime for 
+Multi-threaded DLL (/MD) and build the solution (the ``expat`` project 
+is actually enough for pypy).
 
 Then, copy the file ``win32\bin\release\libexpat.dll`` somewhere in
 your PATH.
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -304,14 +304,19 @@
                     # produce compatible pycs.
                     if (self.space.isinstance_w(w_obj, self.space.w_unicode) and
                         self.space.isinstance_w(w_const, self.space.w_unicode)):
-                        unistr = self.space.unicode_w(w_const)
-                        if len(unistr) == 1:
-                            ch = ord(unistr[0])
-                        else:
-                            ch = 0
-                        if (ch > 0xFFFF or
-                            (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
-                            return subs
+                        #unistr = self.space.unicode_w(w_const)
+                        #if len(unistr) == 1:
+                        #    ch = ord(unistr[0])
+                        #else:
+                        #    ch = 0
+                        #if (ch > 0xFFFF or
+                        #    (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
+                        # --XXX-- for now we always disable optimization of
+                        # u'...'[constant] because the tests above are not
+                        # enough to fix issue5057 (CPython has the same
+                        # problem as of April 24, 2012).
+                        # See test_const_fold_unicode_subscr
+                        return subs
 
                     return ast.Const(w_const, subs.lineno, subs.col_offset)
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -844,7 +844,8 @@
         return u"abc"[0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        if 0:   # xxx later?
+            assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
 
         # getitem outside of the BMP should not be optimized
         source = """def f():
@@ -854,12 +855,20 @@
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
                           ops.RETURN_VALUE: 1}
 
+        source = """def f():
+        return u"\U00012345abcdef"[3]
+        """
+        counts = self.count_instructions(source)
+        assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
+                          ops.RETURN_VALUE: 1}
+
         monkeypatch.setattr(optimize, "MAXUNICODE", 0xFFFF)
         source = """def f():
         return u"\uE01F"[0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        if 0:   # xxx later?
+            assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
         monkeypatch.undo()
 
         # getslice is not yet optimized.
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -20,6 +20,7 @@
         self.dependencies = {}
         # contains frame boxes that are not virtualizables
         self.nonstandard_virtualizables = {}
+
         # heap cache
         # maps descrs to {from_box, to_box} dicts
         self.heap_cache = {}
@@ -29,6 +30,26 @@
         # cache the length of arrays
         self.length_cache = {}
 
+        # replace_box is called surprisingly often, therefore it's not efficient
+        # to go over all the dicts and fix them.
+        # instead, these two dicts are kept, and a replace_box adds an entry to
+        # each of them.
+        # every time one of the dicts heap_cache, heap_array_cache, length_cache
+        # is accessed, suitable indirections need to be performed
+
+        # this looks all very subtle, but in practice the patterns of
+        # replacements should not be that complex. Usually a box is replaced by
+        # a const, once. Also, if something goes wrong, the effect is that less
+        # caching than possible is done, which is not a huge problem.
+        self.input_indirections = {}
+        self.output_indirections = {}
+
+    def _input_indirection(self, box):
+        return self.input_indirections.get(box, box)
+
+    def _output_indirection(self, box):
+        return self.output_indirections.get(box, box)
+
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, argboxes)
         self.clear_caches(opnum, descr, argboxes)
@@ -132,14 +153,16 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
+        box = self._input_indirection(box)
         d = self.heap_cache.get(descr, None)
         if d:
             tobox = d.get(box, None)
-            if tobox:
-                return tobox
+            return self._output_indirection(tobox)
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         self.heap_cache.setdefault(descr, {})[box] = fieldbox
 
     def setfield(self, box, descr, fieldbox):
@@ -148,6 +171,8 @@
         self.heap_cache[descr] = new_d
 
     def _do_write_with_aliasing(self, d, box, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         # slightly subtle logic here
         # a write to an arbitrary box, all other boxes can alias this one
         if not d or box not in self.new_boxes:
@@ -166,6 +191,7 @@
         return new_d
 
     def getarrayitem(self, box, descr, indexbox):
+        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -173,9 +199,11 @@
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return indexcache.get(box, None)
+                return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        box = self._input_indirection(box)
+        valuebox = self._input_indirection(valuebox)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -198,25 +226,13 @@
         cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
 
     def arraylen(self, box):
-        return self.length_cache.get(box, None)
+        box = self._input_indirection(box)
+        return self._output_indirection(self.length_cache.get(box, None))
 
     def arraylen_now_known(self, box, lengthbox):
-        self.length_cache[box] = lengthbox
-
-    def _replace_box(self, d, oldbox, newbox):
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            if frombox is oldbox:
-                frombox = newbox
-            if tobox is oldbox:
-                tobox = newbox
-            new_d[frombox] = tobox
-        return new_d
+        box = self._input_indirection(box)
+        self.length_cache[box] = self._input_indirection(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        for descr, d in self.heap_cache.iteritems():
-            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
-        for descr, d in self.heap_array_cache.iteritems():
-            for index, cache in d.iteritems():
-                d[index] = self._replace_box(cache, oldbox, newbox)
-        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
+        self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox)
+        self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -7,7 +7,7 @@
 import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt
 import pypy.jit.metainterp.optimizeopt.virtualize as virtualize
 from pypy.jit.metainterp.optimize import InvalidLoop
-from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
+from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, get_const_ptr_for_string
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -5067,6 +5067,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_call_pure_vstring_const(self):
+        ops = """
+        []
+        p0 = newstr(3)
+        strsetitem(p0, 0, 97)
+        strsetitem(p0, 1, 98)
+        strsetitem(p0, 2, 99)
+        i0 = call_pure(123, p0, descr=nonwritedescr)
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(5)
+        """
+        call_pure_results = {
+            (ConstInt(123), get_const_ptr_for_string("abc"),): ConstInt(5),
+        }
+        self.optimize_loop(ops, expected, call_pure_results)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -2,12 +2,14 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp.history import ConstInt
 
-box1 = object()
-box2 = object()
-box3 = object()
-box4 = object()
+box1 = "box1"
+box2 = "box2"
+box3 = "box3"
+box4 = "box4"
+box5 = "box5"
 lengthbox1 = object()
 lengthbox2 = object()
+lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -276,11 +278,43 @@
         h.setfield(box1, descr2, box3)
         h.setfield(box2, descr3, box3)
         h.replace_box(box1, box4)
-        assert h.getfield(box1, descr1) is None
-        assert h.getfield(box1, descr2) is None
         assert h.getfield(box4, descr1) is box2
         assert h.getfield(box4, descr2) is box3
         assert h.getfield(box2, descr3) is box3
+        h.setfield(box4, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box4
+        assert h.getfield(box2, descr3) is box4
+
+    def test_replace_box_twice(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box5, descr1) is box2
+        assert h.getfield(box5, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+        h.setfield(box5, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box5
+        assert h.getfield(box2, descr3) is box5
 
     def test_replace_box_array(self):
         h = HeapCache()
@@ -291,9 +325,6 @@
         h.setarrayitem(box3, descr2, index2, box1)
         h.setarrayitem(box2, descr3, index2, box3)
         h.replace_box(box1, box4)
-        assert h.getarrayitem(box1, descr1, index1) is None
-        assert h.getarrayitem(box1, descr2, index1) is None
-        assert h.arraylen(box1) is None
         assert h.arraylen(box4) is lengthbox1
         assert h.getarrayitem(box4, descr1, index1) is box2
         assert h.getarrayitem(box4, descr2, index1) is box3
@@ -304,6 +335,27 @@
         h.replace_box(lengthbox1, lengthbox2)
         assert h.arraylen(box4) is lengthbox2
 
+    def test_replace_box_array_twice(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box5, descr1, index1) is box2
+        assert h.getarrayitem(box5, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box5
+        assert h.getarrayitem(box3, descr2, index2) is box5
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        h.replace_box(lengthbox2, lengthbox3)
+        assert h.arraylen(box4) is lengthbox3
+
     def test_ll_arraycopy(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
diff --git a/pypy/translator/c/src/cjkcodecs/cjkcodecs.h b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
--- a/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
@@ -210,15 +210,15 @@
 
 #define BEGIN_CODECS_LIST /* empty */
 #define _CODEC(name)                                                    \
-  static const MultibyteCodec _pypy_cjkcodec_##name;                    \
-  const MultibyteCodec *pypy_cjkcodec_##name(void) {                    \
+  static MultibyteCodec _pypy_cjkcodec_##name;                          \
+  MultibyteCodec *pypy_cjkcodec_##name(void) {                          \
     if (_pypy_cjkcodec_##name.codecinit != NULL) {                      \
       int r = _pypy_cjkcodec_##name.codecinit(_pypy_cjkcodec_##name.config); \
       assert(r == 0);                                                   \
     }                                                                   \
     return &_pypy_cjkcodec_##name;                                      \
   }                                                                     \
-  static const MultibyteCodec _pypy_cjkcodec_##name
+  static MultibyteCodec _pypy_cjkcodec_##name
 #define _STATEFUL_METHODS(enc)          \
     enc##_encode,                       \
     enc##_encode_init,                  \
diff --git a/pypy/translator/c/src/cjkcodecs/multibytecodec.h b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.h
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
@@ -131,7 +131,7 @@
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    const MultibyteCodec *pypy_cjkcodec_##name(void);
+    MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)

From noreply at buildbot.pypy.org  Thu Apr 26 10:07:55 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 26 Apr 2012 10:07:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: add a hackish tool to get build log
 diffs from the buildbot
Message-ID: <20120426080755.B8B0282009@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r54753:12cfb60a15c8
Date: 2012-04-26 10:07 +0200
http://bitbucket.org/pypy/pypy/changeset/12cfb60a15c8/

Log:	add a hackish tool to get build log diffs from the buildbot

diff --git a/pypy/tool/compare_last_builds.py b/pypy/tool/compare_last_builds.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/compare_last_builds.py
@@ -0,0 +1,122 @@
+import os
+import urllib2
+import json
+import sys
+import md5
+
+wanted = sys.argv[1:]
+if not wanted:
+    wanted = ['default']
+base = "http://buildbot.pypy.org/json/builders/"
+
+cachedir = os.environ.get('PYPY_BUILDS_CACHE')
+if cachedir and not os.path.exists(cachedir):
+    os.makedirs(cachedir)
+
+
+
+def get_json(url, cache=cachedir):
+    return json.loads(get_data(url, cache))
+
+
+def get_data(url, cache=cachedir):
+    url = str(url)
+    if cache:
+        digest = md5.md5()
+        digest.update(url)
+        digest = digest.hexdigest()
+        cachepath = os.path.join(cachedir, digest)
+        if os.path.exists(cachepath):
+            with open(cachepath) as fp:
+                return fp.read()
+
+    print 'GET', url
+    fp = urllib2.urlopen(url)
+    try:
+        data = fp.read()
+        if cache:
+            with open(cachepath, 'wb') as cp:
+                cp.write(data)
+        return data
+    finally:
+        fp.close()
+
+def parse_log(log):
+    items = []
+    for v in log.splitlines(1):
+        if not v[0].isspace() and v[1].isspace():
+            items.append(v)
+    return sorted(items) #sort cause testrunner order is non-deterministic
+
+def gather_logdata(build):
+    logdata = get_data(str(build['log']) + '?as_text=1')
+    logdata = logdata.replace('</span><span class="stdout">', '')
+    logdata = logdata.replace('</span></pre>', '')
+    del build['log']
+    build['log'] = parse_log(logdata)
+
+
+def branch_mapping(l):
+    keep = 3 - len(wanted)
+    d = {}
+    for x in reversed(l):
+        gather_logdata(x)
+        if not x['log']:
+            continue
+        b = x['branch']
+        if b not in d:
+            d[b] = []
+        d[b].insert(0, x)
+        if len(d[b]) > keep:
+            d[b].pop()
+    return d
+
+def cleanup_build(d):
+    for a in 'times eta steps slave reason sourceStamp blame currentStep text'.split():
+        del d[a]
+
+    props = d.pop(u'logs')
+    for name, val in props:
+        if name == u'pytestLog':
+            d['log'] = val
+    props = d.pop(u'properties')
+    for name, val, _ in props:
+        if name == u'branch':
+            d['branch'] = val or 'default'
+    return d
+
+def collect_builds(d):
+    name = str(d['basedir'])
+    builds = d['cachedBuilds']
+    l = []
+    for build in builds:
+        d = get_json(base + '%s/builds/%s' % (name, build))
+        cleanup_build(d)
+        l.append(d)
+
+    l = [x for x in l if x['branch'] in wanted and 'log' in x]
+    d = branch_mapping(l)
+    return [x for lst in d.values() for x in lst]
+
+
+def only_linux32(d):
+    return d['own-linux-x86-32']
+
+
+own_builds = get_json(base, cache=False)['own-linux-x86-32']
+
+builds = collect_builds(own_builds)
+
+
+builds.sort(key=lambda x: (wanted.index(x['branch']), x['number']))
+logs = [x.pop('log') for x in builds]
+for b, s in zip(builds, logs):
+    b['resultset'] = len(s)
+import pprint
+pprint.pprint(builds)
+
+from difflib import Differ
+
+for x in Differ().compare(*logs):
+    if x[0]!=' ':
+        sys.stdout.write(x)

From noreply at buildbot.pypy.org  Thu Apr 26 10:44:31 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:44:31 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Debugging. Should be safe in
 module/transaction/, but better safe than
Message-ID: <20120426084431.709D782009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54754:ddc99cc0bf55
Date: 2012-04-25 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/ddc99cc0bf55/

Log:	Debugging. Should be safe in module/transaction/, but better safe
	than horribly confusingly sorry.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -19,6 +19,7 @@
 
 class Transaction(object):
     _next_transaction = None
+    _scheduled = False         # debugging
     retry_counter = 0
 
     def run(self):
@@ -187,6 +188,7 @@
         transaction._next_transaction = transaction
     #
     transaction.retry_counter = retry_counter
+    transaction._scheduled = False
     new_transactions = transaction.run()
     return _link_new_transactions(new_transactions)
 _run_really._dont_inline_ = True
@@ -203,6 +205,11 @@
     while n >= 0:
         new_transactions[n]._next_transaction = next
         next = new_transactions[n]
+        #
+        ll_assert(not next._scheduled,
+                  "the same Transaction instance is scheduled more than once")
+        next._scheduled = True
+        #
         n -= 1
     return next
 

From noreply at buildbot.pypy.org  Thu Apr 26 10:44:32 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:44:32 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: grumble
Message-ID: <20120426084432.B15938208A@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54755:58bf2ab27d7b
Date: 2012-04-25 19:12 +0200
http://bitbucket.org/pypy/pypy/changeset/58bf2ab27d7b/

Log:	grumble

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -145,6 +145,7 @@
 GCFLAG_VISITED    = first_gcflag << 2
 GCFLAG_HAS_SHADOW = first_gcflag << 3
 GCFLAG_FIXED_HASH = first_gcflag << 4
+GCFLAG_PREBUILT   = first_gcflag << 5
 
 
 def always_inline(fn):
@@ -321,7 +322,7 @@
         hdr.tid = self.combine(typeid16, flags)
 
     def init_gc_object_immortal(self, addr, typeid16, flags=0):
-        flags |= GCFLAG_GLOBAL
+        flags |= GCFLAG_GLOBAL | GCFLAG_PREBUILT
         self.init_gc_object(addr, typeid16, flags)
 
     # ----------
diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -11,6 +11,7 @@
 from pypy.rpython.memory.gc.stmgc import always_inline, dont_inline
 from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED
 from pypy.rpython.memory.gc.stmgc import GCFLAG_WAS_COPIED, GCFLAG_HAS_SHADOW
+from pypy.rpython.memory.gc.stmgc import GCFLAG_PREBUILT
 
 
 class StmGCTLS(object):
@@ -113,8 +114,11 @@
         # We must also mark the following objects as GLOBAL again
         obj = self.mt_global_turned_local
         self.mt_global_turned_local = NULL
+        self.mt_save_prebuilt_turned_local = self.AddressStack()
         while obj:
             hdr = self.gc.header(obj)
+            if hdr.tid & GCFLAG_PREBUILT:
+                self.mt_save_prebuilt_turned_local.append(obj)
             obj = hdr.version
             ll_assert(hdr.tid & GCFLAG_GLOBAL == 0, "already GLOBAL [2]")
             ll_assert(hdr.tid & GCFLAG_VISITED != 0, "missing VISITED [2]")
@@ -144,6 +148,19 @@
         self.mt_global_turned_local = NULL
         self.gc.root_walker.walk_current_stack_roots(
             StmGCTLS._remark_object_as_local, self)
+        # Messy, because prebuilt objects may be Constants in the flow
+        # graphs and so don't appear in the stack, so need a special case.
+        # We save and restore which *prebuilt* objects were originally
+        # in mt_global_turned_local.  (Note that we can't simply save
+        # and restore mt_global_turned_local for *all* objects, because
+        # that would not be enough: the stack typically contains also many
+        # fresh objects that used to be local in enter_transactional_mode().)
+        while self.mt_save_prebuilt_turned_local.non_empty():
+            obj = self.mt_save_prebuilt_turned_local.pop()
+            hdr = self.gc.header(obj)
+            if hdr.tid & GCFLAG_GLOBAL:
+                self.main_thread_writes_to_global_obj(obj)
+        self.mt_save_prebuilt_turned_local.delete()
 
     def start_transaction(self):
         """Start a transaction: performs any pending cleanups, and set

From noreply at buildbot.pypy.org  Thu Apr 26 10:44:33 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:44:33 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: More tests until failure found.
Message-ID: <20120426084433.EC2DE82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54756:d9148faf5f90
Date: 2012-04-25 21:41 +0200
http://bitbucket.org/pypy/pypy/changeset/d9148faf5f90/

Log:	More tests until failure found.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -126,6 +126,9 @@
             self._clear_version_for_global_object(hdr)
         if not we_are_translated():
             del self.mt_global_turned_local   # don't use any more
+        #
+        if self.gc.DEBUG:
+            self.check_all_global_objects()
 
     def leave_transactional_mode(self):
         """Restart using the main thread for mallocs."""
@@ -135,7 +138,10 @@
                     del StmGCTLS.nontranslated_dict[key]
         self.start_transaction()
         #
-        # Do something special here after we restarted the "transaction"
+        if self.gc.DEBUG:
+            self.check_all_global_objects()
+        #
+        # Do something special here after we restarted the execution
         # in the main thread.  At this point, *all* objects are GLOBAL.
         # The write_barrier will ensure that any write makes the written-to
         # objects LOCAL again.  However, it is possible that the write
@@ -602,3 +608,41 @@
                 self.sharedarea_tls.free_object(obj)
             #
             obj = next
+
+    # ------------------------------------------------------------
+
+    def _debug_check_all_global1(self, root):
+        self._debug_check_all_global(root, None)
+
+    def _debug_check_all_global(self, root, ignored):
+        obj = root.address[0]
+        if self.debug_seen.contains(obj):
+            return
+        hdr = self.gc.header(obj)
+        ll_assert(hdr.tid & GCFLAG_GLOBAL != 0,
+                  "debug_check: missing GLOBAL")
+        ll_assert(hdr.tid & GCFLAG_WAS_COPIED == 0,
+                  "debug_check: unexpected WAS_COPIED")
+        ll_assert(hdr.tid & GCFLAG_VISITED == 0,
+                  "debug_check: unexpected VISITED")
+        ll_assert(hdr.tid & GCFLAG_HAS_SHADOW == 0,
+                  "debug_check: unexpected HAS_SHADOW")
+        self.gc.get_size(obj)      # extra checks
+        self.pending.append(obj)
+        self.debug_seen.setitem(obj, obj)
+
+    def check_all_global_objects(self):
+        self.pending = self.AddressStack()
+        self.debug_seen = self.AddressDict()
+        self.gc.root_walker.walk_current_stack_roots(
+            StmGCTLS._debug_check_all_global1, self)
+        while self.pending.non_empty():
+            obj = self.pending.pop()
+            offset = self.gc.weakpointer_offset(self.gc.get_type_id(obj))
+            if offset < 0:    # common case: not a weakref
+                self.gc.trace(obj, self._debug_check_all_global, None)
+            else:
+                if (obj + offset).address[0]:
+                    self._debug_check_all_global(obj + offset, None)
+        self.pending.delete()
+        self.debug_seen.delete()
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -125,7 +125,10 @@
             callback(addr, arg)
 
 def fake_weakpointer_offset(tid):
-    return llmemory.offsetof(WR, 'wadr')
+    if tid == 124:
+        return llmemory.offsetof(WR, 'wadr')
+    else:
+        return -1
 
 class FakeRootWalker:
     def walk_current_stack_roots(self, *args):
@@ -161,7 +164,7 @@
     # test helpers
     def malloc(self, STRUCT, weakref=False, globl='auto'):
         size = llarena.round_up_for_allocation(llmemory.sizeof(STRUCT))
-        tid = lltype.cast_primitive(llgroup.HALFWORD, 123)
+        tid = lltype.cast_primitive(llgroup.HALFWORD, 123 + weakref)
         if globl == 'auto':
             globl = (self.gc.stm_operations.threadnum == 0)
         if globl:
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -105,8 +105,13 @@
     #
     rstm.run_all_transactions(InitialTransaction(),
                               num_threads=glob.NUM_THREADS)
+    check_chained_list(glob.anchor.next)
     #
+    glob.anchor.next = None
+    rstm.run_all_transactions(InitialTransaction(),
+                              num_threads=glob.NUM_THREADS)
     check_chained_list(glob.anchor.next)
+    #
     return 0
 
 # _____ Define and setup target ___

From noreply at buildbot.pypy.org  Thu Apr 26 10:44:35 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:44:35 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Argh, argh.  Finally found the next bug.
Message-ID: <20120426084435.3813D82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54757:22fccf3c9b5e
Date: 2012-04-25 21:57 +0200
http://bitbucket.org/pypy/pypy/changeset/22fccf3c9b5e/

Log:	Argh, argh. Finally found the next bug.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -323,8 +323,10 @@
         # XXX should we also remove GCFLAG_WAS_COPIED here if it is set?
         ll_assert(hdr.tid & GCFLAG_VISITED == 0,
                   "write in main thread: unexpected GCFLAG_VISITED")
-        # remove GCFLAG_GLOBAL, and add GCFLAG_VISITED
-        hdr.tid += (GCFLAG_VISITED - GCFLAG_GLOBAL)
+        # remove GCFLAG_GLOBAL and (if it was there) GCFLAG_WAS_COPIED,
+        # and add GCFLAG_VISITED
+        hdr.tid &= ~(GCFLAG_GLOBAL | GCFLAG_WAS_COPIED)
+        hdr.tid |= GCFLAG_VISITED
         # add the object into this linked list
         hdr.version = self.mt_global_turned_local
         self.mt_global_turned_local = obj
@@ -550,7 +552,7 @@
             ll_assert(hdr.tid & GCFLAG_GLOBAL == 0,
                       "unexpected GCFLAG_GLOBAL in mt_global_turned_local")
             ll_assert(hdr.tid & GCFLAG_VISITED != 0,
-                      "missing GCFLAG_VISIBLE in mt_global_turned_local")
+                      "missing GCFLAG_VISITED in mt_global_turned_local")
             self.trace_and_drag_out_of_nursery(obj)
             obj = hdr.version
 
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -336,10 +336,13 @@
     def test_write_barrier_global(self):
         # check that the main thread never makes a local copy of a global obj
         t, t_adr = self.malloc(S, globl=True)
-        self.checkflags(t_adr, True, False)
+        # the following flag can be set a bit randomly on global objects
+        self.gc.header(t_adr).tid |= GCFLAG_WAS_COPIED
+        self.checkflags(t_adr, True, True)
         obj = self.gc.stm_writebarrier(t_adr)     # main thread, global:
         assert obj == t_adr                       # doesn't make a copy
         self.checkflags(obj, False, False)        # but it becomes local
+        # ^^^ but the GCFLAG_WAS_COPIED is removed in this case
 
     def test_nontransactional_mode(self):
         import random

From noreply at buildbot.pypy.org  Thu Apr 26 10:44:37 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:44:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: No-op change: fix C compiler warnings
	in certain cases.
Message-ID: <20120426084437.58C9382009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54758:936fbee7d501
Date: 2012-04-25 22:01 +0200
http://bitbucket.org/pypy/pypy/changeset/936fbee7d501/

Log:	No-op change: fix C compiler warnings in certain cases.

diff --git a/pypy/rpython/tool/rffi_platform.py b/pypy/rpython/tool/rffi_platform.py
--- a/pypy/rpython/tool/rffi_platform.py
+++ b/pypy/rpython/tool/rffi_platform.py
@@ -379,7 +379,7 @@
         self.name = name
 
     def prepare_code(self):
-        yield 'if ((%s) < 0) {' % (self.name,)
+        yield 'if ((%s) <= 0) {' % (self.name,)
         yield '    long long x = (long long)(%s);' % (self.name,)
         yield '    printf("value: %lld\\n", x);'
         yield '} else {'
@@ -401,7 +401,7 @@
     def prepare_code(self):
         yield '#ifdef %s' % self.macro
         yield 'dump("defined", 1);'
-        yield 'if ((%s) < 0) {' % (self.macro,)
+        yield 'if ((%s) <= 0) {' % (self.macro,)
         yield '    long long x = (long long)(%s);' % (self.macro,)
         yield '    printf("value: %lld\\n", x);'
         yield '} else {'

From noreply at buildbot.pypy.org  Thu Apr 26 10:44:38 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:44:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: End even these temporary C sources with
	an end-of-line.
Message-ID: <20120426084438.A271782009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54759:9803dee0cbdb
Date: 2012-04-25 22:03 +0200
http://bitbucket.org/pypy/pypy/changeset/9803dee0cbdb/

Log:	End even these temporary C sources with an end-of-line.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -103,8 +103,8 @@
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
-udir.join('pypy_decl.h').write("/* Will be filled later */")
-udir.join('pypy_macros.h').write("/* Will be filled later */")
+udir.join('pypy_decl.h').write("/* Will be filled later */\n")
+udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 globals().update(rffi_platform.configure(CConfig_constants))
 
 def copy_header_files(dstdir):

From noreply at buildbot.pypy.org  Thu Apr 26 10:44:39 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:44:39 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add an assert.
Message-ID: <20120426084439.E72C882009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54760:fb0b57f3fed1
Date: 2012-04-25 22:57 +0200
http://bitbucket.org/pypy/pypy/changeset/fb0b57f3fed1/

Log:	Add an assert.

	Status update: seems that pypy-stm running translate.py works now.
	:-)

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -413,6 +413,8 @@
             # detect_flag_combination is GCFLAG_WAS_COPIED|GCFLAG_VISITED.
             # This case is to force pointers to the LOCAL copy to be
             # replaced with pointers to the GLOBAL copy.
+            ll_assert(not self.in_main_thread,
+                      "unexpected flag combination in the main thread")
             return 2
 
         if can_be_in_nursery and self.is_in_nursery(obj):

From noreply at buildbot.pypy.org  Thu Apr 26 10:44:41 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:44:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120426084441.4C02582009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54761:8410ab2abbb0
Date: 2012-04-26 10:43 +0200
http://bitbucket.org/pypy/pypy/changeset/8410ab2abbb0/

Log:	merge heads

diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1167,7 +1167,7 @@
         try:
             return self._lookup_adtmeth(field_name)
         except AttributeError:
-            raise AttributeError("%r instance has no field %r" % (self._T._name,
+            raise AttributeError("%r instance has no field %r" % (self._T,
                                                                   field_name))
 
     def __setattr__(self, field_name, val):
diff --git a/pypy/tool/compare_last_builds.py b/pypy/tool/compare_last_builds.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/compare_last_builds.py
@@ -0,0 +1,122 @@
+import os
+import urllib2
+import json
+import sys
+import md5
+
+wanted = sys.argv[1:]
+if not wanted:
+    wanted = ['default']
+base = "http://buildbot.pypy.org/json/builders/"
+
+cachedir = os.environ.get('PYPY_BUILDS_CACHE')
+if cachedir and not os.path.exists(cachedir):
+    os.makedirs(cachedir)
+
+
+
+def get_json(url, cache=cachedir):
+    return json.loads(get_data(url, cache))
+
+
+def get_data(url, cache=cachedir):
+    url = str(url)
+    if cache:
+        digest = md5.md5()
+        digest.update(url)
+        digest = digest.hexdigest()
+        cachepath = os.path.join(cachedir, digest)
+        if os.path.exists(cachepath):
+            with open(cachepath) as fp:
+                return fp.read()
+
+    print 'GET', url
+    fp = urllib2.urlopen(url)
+    try:
+        data = fp.read()
+        if cache:
+            with open(cachepath, 'wb') as cp:
+                cp.write(data)
+        return data
+    finally:
+        fp.close()
+
+def parse_log(log):
+    items = []
+    for v in log.splitlines(1):
+        if not v[0].isspace() and v[1].isspace():
+            items.append(v)
+    return sorted(items) #sort cause testrunner order is non-deterministic
+
+def gather_logdata(build):
+    logdata = get_data(str(build['log']) + '?as_text=1')
+    logdata = logdata.replace('</span><span class="stdout">', '')
+    logdata = logdata.replace('</span></pre>', '')
+    del build['log']
+    build['log'] = parse_log(logdata)
+
+
+def branch_mapping(l):
+    keep = 3 - len(wanted)
+    d = {}
+    for x in reversed(l):
+        gather_logdata(x)
+        if not x['log']:
+            continue
+        b = x['branch']
+        if b not in d:
+            d[b] = []
+        d[b].insert(0, x)
+        if len(d[b]) > keep:
+            d[b].pop()
+    return d
+
+def cleanup_build(d):
+    for a in 'times eta steps slave reason sourceStamp blame currentStep text'.split():
+        del d[a]
+
+    props = d.pop(u'logs')
+    for name, val in props:
+        if name == u'pytestLog':
+            d['log'] = val
+    props = d.pop(u'properties')
+    for name, val, _ in props:
+        if name == u'branch':
+            d['branch'] = val or 'default'
+    return d
+
+def collect_builds(d):
+    name = str(d['basedir'])
+    builds = d['cachedBuilds']
+    l = []
+    for build in builds:
+        d = get_json(base + '%s/builds/%s' % (name, build))
+        cleanup_build(d)
+        l.append(d)
+
+    l = [x for x in l if x['branch'] in wanted and 'log' in x]
+    d = branch_mapping(l)
+    return [x for lst in d.values() for x in lst]
+
+
+def only_linux32(d):
+    return d['own-linux-x86-32']
+
+
+own_builds = get_json(base, cache=False)['own-linux-x86-32']
+
+builds = collect_builds(own_builds)
+
+
+builds.sort(key=lambda x: (wanted.index(x['branch']), x['number']))
+logs = [x.pop('log') for x in builds]
+for b, s in zip(builds, logs):
+    b['resultset'] = len(s)
+import pprint
+pprint.pprint(builds)
+
+from difflib import Differ
+
+for x in Differ().compare(*logs):
+    if x[0]!=' ':
+        sys.stdout.write(x)

From noreply at buildbot.pypy.org  Thu Apr 26 10:46:31 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:46:31 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120426084631.B02B482009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54762:f7093e6634de
Date: 2012-04-26 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/f7093e6634de/

Log:	hg merge default

diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py
--- a/lib_pypy/_ctypes_test.py
+++ b/lib_pypy/_ctypes_test.py
@@ -21,7 +21,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC']
     res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')],
@@ -34,6 +34,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST'] # needed for VC10
     else:
diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py
--- a/lib_pypy/_testcapi.py
+++ b/lib_pypy/_testcapi.py
@@ -16,7 +16,7 @@
     # Compile .c file
     include_dir = os.path.join(thisdir, '..', 'include')
     if sys.platform == 'win32':
-        ccflags = []
+        ccflags = ['-D_CRT_SECURE_NO_WARNINGS']
     else:
         ccflags = ['-fPIC', '-Wimplicit-function-declaration']
     res = compiler.compile([os.path.join(thisdir, '_testcapimodule.c')],
@@ -29,6 +29,13 @@
     if sys.platform == 'win32':
         # XXX libpypy-c.lib is currently not installed automatically
         library = os.path.join(thisdir, '..', 'include', 'libpypy-c')
+        if not os.path.exists(library + '.lib'):
+            #For a nightly build
+            library = os.path.join(thisdir, '..', 'include', 'python27')
+        if not os.path.exists(library + '.lib'):
+            # For a local translation
+            library = os.path.join(thisdir, '..', 'pypy', 'translator',
+                    'goal', 'libpypy-c')
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
                         '/EXPORT:init_testcapi']
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/cppyy.rst
@@ -0,0 +1,554 @@
+============================
+cppyy: C++ bindings for PyPy
+============================
+
+The cppyy module provides C++ bindings for PyPy by using the reflection
+information extracted from C++ header files by means of the
+`Reflex package`_.
+For this to work, you have to both install Reflex and build PyPy from the
+reflex-support branch.
+As indicated by this being a branch, support for Reflex is still
+experimental.
+However, it is functional enough to put it in the hands of those who want
+to give it a try.
+In the medium term, cppyy will move away from Reflex and instead use
+`cling`_ as its backend, which is based on `llvm`_.
+Although that will change the logistics on the generation of reflection
+information, it will not change the python-side interface.
+
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`cling`: http://root.cern.ch/drupal/content/cling
+.. _`llvm`: http://llvm.org/
+
+
+Installation
+============
+
+For now, the easiest way of getting the latest version of Reflex, is by
+installing the ROOT package.
+Besides getting the latest version of Reflex, another advantage is that with
+the full ROOT package, you can also use your Reflex-bound code on `CPython`_.
+`Download`_ a binary or install from `source`_.
+Some Linux and Mac systems may have ROOT provided in the list of scientific
+software of their packager.
+A current, standalone version of Reflex should be provided at some point,
+once the dependencies and general packaging have been thought out.
+Also, make sure you have a version of `gccxml`_ installed, which is most
+easily provided by the packager of your system.
+If you read up on gccxml, you'll probably notice that it is no longer being
+developed and hence will not provide C++11 support.
+That's why the medium term plan is to move to `cling`_.
+
+.. _`Download`: http://root.cern.ch/drupal/content/downloading-root
+.. _`source`: http://root.cern.ch/drupal/content/installing-root-source
+.. _`gccxml`: http://www.gccxml.org
+
+Next, get the `PyPy sources`_, select the reflex-support branch, and build
+pypy-c.
+For the build to succeed, the ``$ROOTSYS`` environment variable must point to
+the location of your ROOT installation::
+
+    $ hg clone https://bitbucket.org/pypy/pypy
+    $ cd pypy
+    $ hg up reflex-support
+    $ cd pypy/translator/goal
+    $ python translate.py -O jit --gcrootfinder=shadowstack targetpypystandalone.py --withmod-cppyy
+
+This will build a ``pypy-c`` that includes the cppyy module, and through that,
+Reflex support.
+Of course, if you already have a pre-built version of the ``pypy`` interpreter,
+you can use that for the translation rather than ``python``.
+
+.. _`PyPy sources`: https://bitbucket.org/pypy/pypy/overview
+
+
+Basic example
+=============
+
+Now test with a trivial example whether all packages are properly installed
+and functional.
+First, create a C++ header file with some class in it (note that all functions
+are made inline for convenience; a real-world example would of course have a
+corresponding source file)::
+
+    $ cat MyClass.h
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_myint(i) {}
+
+        int GetMyInt() { return m_myint; }
+        void SetMyInt(int i) { m_myint = i; }
+
+    public:
+        int m_myint;
+    };
+
+Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
+code::
+
+    $ genreflex MyClass.h
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyClass_rflx.cpp -o libMyClassDict.so
+
+Now you're ready to use the bindings.
+Since the bindings are designed to look pythonistic, it should be
+straightforward::
+
+    $ pypy-c
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libMyClassDict.so")
+    <CPPLibrary object at 0xb6fd7c4c>
+    >>>> myinst = cppyy.gbl.MyClass(42)
+    >>>> print myinst.GetMyInt()
+    42
+    >>>> myinst.SetMyInt(33)
+    >>>> print myinst.m_myint
+    33
+    >>>> myinst.m_myint = 77
+    >>>> print myinst.GetMyInt()
+    77
+    >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
+
+That's all there is to it!
+
+
+Advanced example
+================
+The following snippet of C++ is very contrived, to allow showing that such
+pathological code can be handled and to show how certain features play out in
+practice::
+
+    $ cat MyAdvanced.h
+    #include <string>
+
+    class Base1 {
+    public:
+        Base1(int i) : m_i(i) {}
+        virtual ~Base1() {}
+        int m_i;
+    };
+
+    class Base2 {
+    public:
+        Base2(double d) : m_d(d) {}
+        virtual ~Base2() {}
+        double m_d;
+    };
+
+    class C;
+
+    class Derived : public virtual Base1, public virtual Base2 {
+    public:
+        Derived(const std::string& name, int i, double d) : Base1(i), Base2(d), m_name(name) {}
+        virtual C* gimeC() { return (C*)0; }
+        std::string m_name;
+    };
+
+    Base1* BaseFactory(const std::string& name, int i, double d) {
+        return new Derived(name, i, d);
+    }
+
+This code is still only in a header file, with all functions inline, for
+convenience of the example.
+If the implementations live in a separate source file or shared library, the
+only change needed is to link those in when building the reflection library.
+
+If you were to run ``genreflex`` like above in the basic example, you will
+find that not all classes of interest will be reflected, nor will be the
+global factory function.
+In particular, ``std::string`` will be missing, since it is not defined in
+this header file, but in a header file that is included.
+In practical terms, general classes such as ``std::string`` should live in a
+core reflection set, but for the moment assume we want to have it in the
+reflection library that we are building for this example.
+
+The ``genreflex`` script can be steered using a so-called `selection file`_,
+which is a simple XML file specifying, either explicitly or by using a
+pattern, which classes, variables, namespaces, etc. to select from the given
+header file.
+With the aid of a selection file, a large project can be easily managed:
+simply ``#include`` all relevant headers into a single header file that is
+handed to ``genreflex``.
+Then, apply a selection file to pick up all the relevant classes.
+For our purposes, the following rather straightforward selection will do
+(the name ``lcgdict`` for the root is historical, but required)::
+
+    $ cat MyAdvanced.xml
+    <lcgdict>
+        <class pattern="Base?" />
+        <class name="Derived" />
+        <class name="std::string" />
+        <function name="BaseFactory" />
+    </lcgdict>
+
+.. _`selection file`: http://root.cern.ch/drupal/content/generating-reflex-dictionaries
+
+Now the reflection info can be generated and compiled::
+
+    $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyAdvanced_rflx.cpp -o libAdvExDict.so
+
+and subsequently be used from PyPy::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libAdvExDict.so")
+    <CPPLibrary object at 0x00007fdb48fc8120>
+    >>>> d = cppyy.gbl.BaseFactory("name", 42, 3.14)
+    >>>> type(d)
+    <class '__main__.Derived'>
+    >>>> d.m_i
+    42
+    >>>> d.m_d
+    3.14
+    >>>> d.m_name == "name"
+    True
+    >>>>
+
+Again, that's all there is to it!
+
+A couple of things to note, though.
+If you look back at the C++ definition of the ``BaseFactory`` function,
+you will see that it declares the return type to be a ``Base1``, yet the
+bindings return an object of the actual type ``Derived``?
+This choice is made for a couple of reasons.
+First, it makes method dispatching easier: if bound objects are always their
+most derived type, then it is easy to calculate any offsets, if necessary.
+Second, it makes memory management easier: the combination of the type and
+the memory address uniquely identifies an object.
+That way, it can be recycled and object identity can be maintained if it is
+entered as a function argument into C++ and comes back to PyPy as a return
+value.
+Last, but not least, casting is decidedly unpythonistic.
+By always providing the most derived type known, casting becomes unnecessary.
+For example, the data member of ``Base2`` is simply directly available.
+Note also that the unreflected ``gimeC`` method of ``Derived`` does not
+preclude its use.
+It is only the ``gimeC`` method that is unusable as long as class ``C`` is
+unknown to the system.
+
+
+Features
+========
+
+The following is not meant to be an exhaustive list, since cppyy is still
+under active development.
+Furthermore, the intention is that every feature is as natural as possible on
+the python side, so if you find something missing in the list below, simply
+try it out.
+It is not always possible to provide exact mapping between python and C++
+(active memory management is one such case), but by and large, if the use of a
+feature does not strike you as obvious, it is more likely to simply be a bug.
+That is a strong statement to make, but also a worthy goal.
+
+* **abstract classes**: Are represented as python classes, since they are
+  needed to complete the inheritance hierarchies, but will raise an exception
+  if an attempt is made to instantiate from them.
+
+* **arrays**: Supported for builtin data types only, as used from module
+  ``array``.
+  Out-of-bounds checking is limited to those cases where the size is known at
+  compile time (and hence part of the reflection info).
+
+* **builtin data types**: Map onto the expected equivalent python types, with
+  the caveat that there may be size differences, and thus it is possible that
+  exceptions are raised if an overflow is detected.
+
+* **casting**: Is supposed to be unnecessary.
+  Object pointer returns from functions provide the most derived class known
+  in the hierarchy of the object being returned.
+  This is important to preserve object identity as well as to make casting,
+  a pure C++ feature after all, superfluous.
+
+* **classes and structs**: Get mapped onto python classes, where they can be
+  instantiated as expected.
+  If classes are inner classes or live in a namespace, their naming and
+  location will reflect that.
+
+* **data members**: Public data members are represented as python properties
+  and provide read and write access on instances as expected.
+
+* **default arguments**: C++ default arguments work as expected, but python
+  keywords are not supported.
+  It is technically possible to support keywords, but for the C++ interface,
+  the formal argument names have no meaning and are not considered part of the
+  API, hence it is not a good idea to use keywords.
+
+* **doc strings**: The doc string of a method or function contains the C++
+  arguments and return types of all overloads of that name, as applicable.
+
+* **enums**: Are translated as ints with no further checking.
+
+* **functions**: Work as expected and live in their appropriate namespace
+  (which can be the global one, ``cppyy.gbl``).
+
+* **inheritance**: All combinations of inheritance on the C++ (single,
+  multiple, virtual) are supported in the binding.
+  However, new python classes can only use single inheritance from a bound C++
+  class.
+  Multiple inheritance would introduce two "this" pointers in the binding.
+  This is a current, not a fundamental, limitation.
+  The C++ side will not see any overridden methods on the python side, as
+  cross-inheritance is planned but not yet supported.
+
+* **methods**: Are represented as python methods and work as expected.
+  They are first class objects and can be bound to an instance.
+  Virtual C++ methods work as expected.
+  To select a specific virtual method, do like with normal python classes
+  that override methods: select it from the class that you need, rather than
+  calling the method on the instance.
+
+* **namespaces**: Are represented as python classes.
+  Namespaces are more open-ended than classes, so sometimes initial access may
+  result in updates as data and functions are looked up and constructed
+  lazily.
+  Thus the result of ``dir()`` on a namespace should not be relied upon: it
+  only shows the already accessed members. (TODO: to be fixed by implementing
+  __dir__.)
+  The global namespace is ``cppyy.gbl``.
+
+* **operator conversions**: If defined in the C++ class and a python
+  equivalent exists (i.e. all builtin integer and floating point types, as well
+  as ``bool``), it will map onto that python conversion.
+  Note that ``char*`` is mapped onto ``__str__``.
+
+* **operator overloads**: If defined in the C++ class and if a python
+  equivalent is available (not always the case, think e.g. of ``operator||``),
+  then they work as expected.
+  Special care needs to be taken for global operator overloads in C++: first,
+  make sure that they are actually reflected, especially for the global
+  overloads for ``operator==`` and ``operator!=`` of STL iterators in the case
+  of gcc.
+  Second, make sure that reflection info is loaded in the proper order.
+  I.e. that these global overloads are available before use.
+
+* **pointers**: For builtin data types, see arrays.
+  For objects, a pointer to an object and an object looks the same, unless
+  the pointer is a data member.
+  In that case, assigning to the data member will cause a copy of the pointer
+  and care should be taken about the object's life time.
+  If a pointer is a global variable, the C++ side can replace the underlying
+  object and the python side will immediately reflect that.
+
+* **static data members**: Are represented as python property objects on the
+  class and the meta-class.
+  Both read and write access is as expected.
+
+* **static methods**: Are represented as python's ``staticmethod`` objects
+  and can be called both from the class as well as from instances.
+
+* **strings**: The std::string class is considered a builtin C++ type and
+  mixes quite well with python's str.
+  Python's str can be passed where a ``const char*`` is expected, and an str
+  will be returned if the return type is ``const char*``.
+
+* **templated classes**: Are represented in a meta-class style in python.
+  This looks a little bit confusing, but conceptually is rather natural.
+  For example, given the class ``std::vector<int>``, the meta-class part would
+  be ``std.vector`` in python.
+  Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
+  create an instance of that class, do ``std.vector(int)()``.
+  Note that templates can be build up by handing actual types to the class
+  instantiation (as done in this vector example), or by passing in the list of
+  template arguments as a string.
+  The former is a lot easier to work with if you have template instantiations
+  using classes that themselves are templates (etc.) in the arguments.
+  All classes must already exist in the loaded reflection info.
+
+* **typedefs**: Are simple python references to the actual classes to which
+  they refer.
+
+* **unary operators**: Are supported if a python equivalent exists, and if the
+  operator is defined in the C++ class.
+
+You can always find more detailed examples and see the full of supported
+features by looking at the tests in pypy/module/cppyy/test.
+
+If a feature or reflection info is missing, this is supposed to be handled
+gracefully.
+In fact, there are unit tests explicitly for this purpose (even as their use
+becomes less interesting over time, as the number of missing features
+decreases).
+Only when a missing feature is used, should there be an exception.
+For example, if no reflection info is available for a return type, then a
+class that has a method with that return type can still be used.
+Only that one specific method can not be used.
+
+
+Templates
+=========
+
+A bit of special care needs to be taken for the use of templates.
+For a templated class to be completely available, it must be guaranteed that
+said class is fully instantiated, and hence all executable C++ code is
+generated and compiled in.
+The easiest way to fulfill that guarantee is by explicit instantiation in the
+header file that is handed to ``genreflex``.
+The following example should make that clear::
+
+    $ cat MyTemplate.h
+    #include <vector>
+
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_i(i) {}
+        MyClass(const MyClass& s) : m_i(s.m_i) {}
+        MyClass& operator=(const MyClass& s) { m_i = s.m_i; return *this; }
+        ~MyClass() {}
+        int m_i;
+    };
+
+    template class std::vector<MyClass>;
+
+If you know for certain that all symbols will be linked in from other sources,
+you can also declare the explicit template instantiation ``extern``.
+
+Unfortunately, this is not enough for gcc.
+The iterators, if they are going to be used, need to be instantiated as well,
+as do the comparison operators on those iterators, as these live in an
+internal namespace, rather than in the iterator classes.
+One way to handle this, is to deal with this once in a macro, then reuse that
+macro for all ``vector`` classes.
+Thus, the header above needs this, instead of just the explicit instantiation
+of the ``vector<MyClass>``::
+
+    #define STLTYPES_EXPLICIT_INSTANTIATION_DECL(STLTYPE, TTYPE)                      \
+    template class std::STLTYPE< TTYPE >;                                             \
+    template class __gnu_cxx::__normal_iterator<TTYPE*, std::STLTYPE< TTYPE > >;      \
+    template class __gnu_cxx::__normal_iterator<const TTYPE*, std::STLTYPE< TTYPE > >;\
+    namespace __gnu_cxx {                                                             \
+    template bool operator==(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    template bool operator!=(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    }
+
+    STLTYPES_EXPLICIT_INSTANTIATION_DECL(vector, MyClass)
+
+Then, still for gcc, the selection file needs to contain the full hierarchy as
+well as the global overloads for comparisons for the iterators::
+
+    $ cat MyTemplate.xml
+    <lcgdict>
+        <class pattern="std::vector<*>" />
+        <class pattern="__gnu_cxx::__normal_iterator<*>" />
+        <class pattern="__gnu_cxx::new_allocator<*>" />
+        <class pattern="std::_Vector_base<*>" />
+        <class pattern="std::_Vector_base<*>::_Vector_impl" />
+        <class pattern="std::allocator<*>" />
+        <function name="__gnu_cxx::operator=="/>
+        <function name="__gnu_cxx::operator!="/>
+
+        <class name="MyClass" />
+    </lcgdict>
+
+Run the normal ``genreflex`` and compilation steps::
+
+    $ genreflex MyTemplate.h --selection=MyTemplate.xm
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so
+
+Note: this is a dirty corner that clearly could do with some automation,
+even if the macro already helps.
+Such automation is planned.
+In fact, in the cling world, the backend can perform the template
+instantations and generate the reflection info on the fly, and none of the
+above will any longer be necessary.
+
+Subsequent use should be as expected.
+Note the meta-class style of "instantiating" the template::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libTemplateDict.so")
+    >>>> std = cppyy.gbl.std
+    >>>> MyClass = cppyy.gbl.MyClass
+    >>>> v = std.vector(MyClass)()
+    >>>> v += [MyClass(1), MyClass(2), MyClass(3)]
+    >>>> for m in v:
+    ....     print m.m_i,
+    ....
+    1 2 3
+    >>>>
+
+Other templates work similarly.
+The arguments to the template instantiation can either be a string with the
+full list of arguments, or the explicit classes.
+The latter makes for easier code writing if the classes passed to the
+instantiation are themselves templates.
+
+
+The fast lane
+=============
+
+The following is an experimental feature of cppyy, and that makes it doubly
+experimental, so caveat emptor.
+With a slight modification of Reflex, it can provide function pointers for
+C++ methods, and hence allow PyPy to call those pointers directly, rather than
+calling C++ through a Reflex stub.
+This results in a rather significant speed-up.
+Mind you, the normal stub path is not exactly slow, so for now only use this
+out of curiosity or if you really need it.
+
+To install this patch of Reflex, locate the file genreflex-methptrgetter.patch
+in pypy/module/cppyy and apply it to the genreflex python scripts found in
+``$ROOTSYS/lib``::
+
+    $ cd $ROOTSYS/lib
+    $ patch -p2 < genreflex-methptrgetter.patch
+
+With this patch, ``genreflex`` will have grown the ``--with-methptrgetter``
+option.
+Use this option when running ``genreflex``, and add the
+``-Wno-pmf-conversions`` option to ``g++`` when compiling.
+The rest works the same way: the fast path will be used transparently (which
+also means that you can't actually find out whether it is in use, other than
+by running a micro-benchmark).
+
+
+CPython
+=======
+
+Most of the ideas in cppyy come originally from the `PyROOT`_ project.
+Although PyROOT does not support Reflex directly, it has an alter ego called
+"PyCintex" that, in a somewhat roundabout way, does.
+If you installed ROOT, rather than just Reflex, PyCintex should be available
+immediately if you add ``$ROOTSYS/lib`` to the ``PYTHONPATH`` environment
+variable.
+
+.. _`PyROOT`: http://root.cern.ch/drupal/content/pyroot
+
+There are a couple of minor differences between PyCintex and cppyy, most to do
+with naming.
+The one that you will run into directly, is that PyCintex uses a function
+called ``loadDictionary`` rather than ``load_reflection_info``.
+The reason for this is that Reflex calls the shared libraries that contain
+reflection info "dictionaries."
+However, in python, the name `dictionary` already has a well-defined meaning,
+so a more descriptive name was chosen for cppyy.
+In addition, PyCintex requires that the names of shared libraries so loaded
+start with "lib" in their name.
+The basic example above, rewritten for PyCintex thus goes like this::
+
+    $ python
+    >>> import PyCintex
+    >>> PyCintex.loadDictionary("libMyClassDict.so")
+    >>> myinst = PyCintex.gbl.MyClass(42)
+    >>> print myinst.GetMyInt()
+    42
+    >>> myinst.SetMyInt(33)
+    >>> print myinst.m_myint
+    33
+    >>> myinst.m_myint = 77
+    >>> print myinst.GetMyInt()
+    77
+    >>> help(PyCintex.gbl.MyClass)   # shows that normal python introspection works
+
+Other naming differences are such things as taking an address of an object.
+In PyCintex, this is done with ``AddressOf`` whereas in cppyy the choice was
+made to follow the naming as in ``ctypes`` and hence use ``addressof``
+(PyROOT/PyCintex predate ``ctypes`` by several years, and the ROOT project
+follows camel-case, hence the differences).
+
+Of course, this is python, so if any of the naming is not to your liking, all
+you have to do is provide a wrapper script that you import instead of
+importing the ``cppyy`` or ``PyCintex`` modules directly.
+In that wrapper script you can rename methods exactly the way you need it.
+
+In the cling world, all these differences will be resolved.
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -23,6 +23,8 @@
 
 * Write them in RPython as mixedmodule_, using *rffi* as bindings.
 
+* Write them in C++ and bind them through Reflex_ (EXPERIMENTAL)
+
 .. _ctypes: #CTypes
 .. _\_ffi: #LibFFI
 .. _mixedmodule: #Mixed Modules
@@ -110,3 +112,34 @@
 
 XXX we should provide detailed docs about lltype and rffi, especially if we
     want people to follow that way.
+
+Reflex
+======
+
+This method is only experimental for now, and is being exercised on a branch,
+`reflex-support`_, so you will have to build PyPy yourself.
+The method works by using the `Reflex package`_ to provide reflection
+information of the C++ code, which is then used to automatically generate
+bindings at runtime, which can then be used from python.
+Full details are `available here`_.
+
+.. _`reflex-support`: cppyy.html
+.. _`Reflex package`: http://root.cern.ch/drupal/content/reflex
+.. _`available here`: cppyy.html
+
+Pros
+----
+
+If it works, it is mostly automatic, and hence easy in use.
+The bindings can make use of direct pointers, in which case the calls are
+very fast.
+
+Cons
+----
+
+C++ is a large language, and these bindings are not yet feature-complete.
+Although missing features should do no harm if you don't use them, if you do
+need a particular feature, it may be necessary to work around it in python
+or with a C++ helper function.
+Although Reflex works on various platforms, the bindings with PyPy have only
+been tested on Linux.
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -24,7 +24,8 @@
 translation.  Failing that, they will pick the most recent Visual Studio
 compiler they can find.  In addition, the target architecture
 (32 bits, 64 bits) is automatically selected.  A 32 bit build can only be built
-using a 32 bit Python and vice versa.
+using a 32 bit Python and vice versa. By default pypy is built using the 
+Multi-threaded DLL (/MD) runtime environment.
 
 **Note:** PyPy is currently not supported for 64 bit Windows, and translation
 will fail in this case.
@@ -102,10 +103,12 @@
 
 Download the source code of expat on sourceforge:
 http://sourceforge.net/projects/expat/ and extract it in the base
-directory.  Then open the project file ``expat.dsw`` with Visual
+directory.  Version 2.1.0 is known to pass tests. Then open the project 
+file ``expat.dsw`` with Visual
 Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, and build the solution (the
-``expat`` project is actually enough for pypy).
+switch to the "Release" configuration, reconfigure the runtime for 
+Multi-threaded DLL (/MD) and build the solution (the ``expat`` project 
+is actually enough for pypy).
 
 Then, copy the file ``win32\bin\release\libexpat.dll`` somewhere in
 your PATH.
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -304,14 +304,19 @@
                     # produce compatible pycs.
                     if (self.space.isinstance_w(w_obj, self.space.w_unicode) and
                         self.space.isinstance_w(w_const, self.space.w_unicode)):
-                        unistr = self.space.unicode_w(w_const)
-                        if len(unistr) == 1:
-                            ch = ord(unistr[0])
-                        else:
-                            ch = 0
-                        if (ch > 0xFFFF or
-                            (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
-                            return subs
+                        #unistr = self.space.unicode_w(w_const)
+                        #if len(unistr) == 1:
+                        #    ch = ord(unistr[0])
+                        #else:
+                        #    ch = 0
+                        #if (ch > 0xFFFF or
+                        #    (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
+                        # --XXX-- for now we always disable optimization of
+                        # u'...'[constant] because the tests above are not
+                        # enough to fix issue5057 (CPython has the same
+                        # problem as of April 24, 2012).
+                        # See test_const_fold_unicode_subscr
+                        return subs
 
                     return ast.Const(w_const, subs.lineno, subs.col_offset)
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -844,7 +844,8 @@
         return u"abc"[0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        if 0:   # xxx later?
+            assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
 
         # getitem outside of the BMP should not be optimized
         source = """def f():
@@ -854,12 +855,20 @@
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
                           ops.RETURN_VALUE: 1}
 
+        source = """def f():
+        return u"\U00012345abcdef"[3]
+        """
+        counts = self.count_instructions(source)
+        assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
+                          ops.RETURN_VALUE: 1}
+
         monkeypatch.setattr(optimize, "MAXUNICODE", 0xFFFF)
         source = """def f():
         return u"\uE01F"[0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        if 0:   # xxx later?
+            assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
         monkeypatch.undo()
 
         # getslice is not yet optimized.
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -20,6 +20,7 @@
         self.dependencies = {}
         # contains frame boxes that are not virtualizables
         self.nonstandard_virtualizables = {}
+
         # heap cache
         # maps descrs to {from_box, to_box} dicts
         self.heap_cache = {}
@@ -29,6 +30,26 @@
         # cache the length of arrays
         self.length_cache = {}
 
+        # replace_box is called surprisingly often, therefore it's not efficient
+        # to go over all the dicts and fix them.
+        # instead, these two dicts are kept, and a replace_box adds an entry to
+        # each of them.
+        # every time one of the dicts heap_cache, heap_array_cache, length_cache
+        # is accessed, suitable indirections need to be performed
+
+        # this looks all very subtle, but in practice the patterns of
+        # replacements should not be that complex. Usually a box is replaced by
+        # a const, once. Also, if something goes wrong, the effect is that less
+        # caching than possible is done, which is not a huge problem.
+        self.input_indirections = {}
+        self.output_indirections = {}
+
+    def _input_indirection(self, box):
+        return self.input_indirections.get(box, box)
+
+    def _output_indirection(self, box):
+        return self.output_indirections.get(box, box)
+
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, argboxes)
         self.clear_caches(opnum, descr, argboxes)
@@ -132,14 +153,16 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
+        box = self._input_indirection(box)
         d = self.heap_cache.get(descr, None)
         if d:
             tobox = d.get(box, None)
-            if tobox:
-                return tobox
+            return self._output_indirection(tobox)
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         self.heap_cache.setdefault(descr, {})[box] = fieldbox
 
     def setfield(self, box, descr, fieldbox):
@@ -148,6 +171,8 @@
         self.heap_cache[descr] = new_d
 
     def _do_write_with_aliasing(self, d, box, fieldbox):
+        box = self._input_indirection(box)
+        fieldbox = self._input_indirection(fieldbox)
         # slightly subtle logic here
         # a write to an arbitrary box, all other boxes can alias this one
         if not d or box not in self.new_boxes:
@@ -166,6 +191,7 @@
         return new_d
 
     def getarrayitem(self, box, descr, indexbox):
+        box = self._input_indirection(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -173,9 +199,11 @@
         if cache:
             indexcache = cache.get(index, None)
             if indexcache is not None:
-                return indexcache.get(box, None)
+                return self._output_indirection(indexcache.get(box, None))
 
     def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        box = self._input_indirection(box)
+        valuebox = self._input_indirection(valuebox)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -198,25 +226,13 @@
         cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
 
     def arraylen(self, box):
-        return self.length_cache.get(box, None)
+        box = self._input_indirection(box)
+        return self._output_indirection(self.length_cache.get(box, None))
 
     def arraylen_now_known(self, box, lengthbox):
-        self.length_cache[box] = lengthbox
-
-    def _replace_box(self, d, oldbox, newbox):
-        new_d = {}
-        for frombox, tobox in d.iteritems():
-            if frombox is oldbox:
-                frombox = newbox
-            if tobox is oldbox:
-                tobox = newbox
-            new_d[frombox] = tobox
-        return new_d
+        box = self._input_indirection(box)
+        self.length_cache[box] = self._input_indirection(lengthbox)
 
     def replace_box(self, oldbox, newbox):
-        for descr, d in self.heap_cache.iteritems():
-            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
-        for descr, d in self.heap_array_cache.iteritems():
-            for index, cache in d.iteritems():
-                d[index] = self._replace_box(cache, oldbox, newbox)
-        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
+        self.input_indirections[self._output_indirection(newbox)] = self._input_indirection(oldbox)
+        self.output_indirections[self._input_indirection(oldbox)] = self._output_indirection(newbox)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -7,7 +7,7 @@
 import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt
 import pypy.jit.metainterp.optimizeopt.virtualize as virtualize
 from pypy.jit.metainterp.optimize import InvalidLoop
-from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
+from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt, get_const_ptr_for_string
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
 from pypy.rlib.rarithmetic import LONG_BIT
@@ -5067,6 +5067,25 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_call_pure_vstring_const(self):
+        ops = """
+        []
+        p0 = newstr(3)
+        strsetitem(p0, 0, 97)
+        strsetitem(p0, 1, 98)
+        strsetitem(p0, 2, 99)
+        i0 = call_pure(123, p0, descr=nonwritedescr)
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(5)
+        """
+        call_pure_results = {
+            (ConstInt(123), get_const_ptr_for_string("abc"),): ConstInt(5),
+        }
+        self.optimize_loop(ops, expected, call_pure_results)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -2,12 +2,14 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.metainterp.history import ConstInt
 
-box1 = object()
-box2 = object()
-box3 = object()
-box4 = object()
+box1 = "box1"
+box2 = "box2"
+box3 = "box3"
+box4 = "box4"
+box5 = "box5"
 lengthbox1 = object()
 lengthbox2 = object()
+lengthbox3 = object()
 descr1 = object()
 descr2 = object()
 descr3 = object()
@@ -276,11 +278,43 @@
         h.setfield(box1, descr2, box3)
         h.setfield(box2, descr3, box3)
         h.replace_box(box1, box4)
-        assert h.getfield(box1, descr1) is None
-        assert h.getfield(box1, descr2) is None
         assert h.getfield(box4, descr1) is box2
         assert h.getfield(box4, descr2) is box3
         assert h.getfield(box2, descr3) is box3
+        h.setfield(box4, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box4
+        assert h.getfield(box2, descr3) is box4
+
+    def test_replace_box_twice(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box5, descr1) is box2
+        assert h.getfield(box5, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+        h.setfield(box5, descr1, box3)
+        assert h.getfield(box4, descr1) is box3
+
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box3, box4)
+        h.replace_box(box4, box5)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box5
+        assert h.getfield(box2, descr3) is box5
 
     def test_replace_box_array(self):
         h = HeapCache()
@@ -291,9 +325,6 @@
         h.setarrayitem(box3, descr2, index2, box1)
         h.setarrayitem(box2, descr3, index2, box3)
         h.replace_box(box1, box4)
-        assert h.getarrayitem(box1, descr1, index1) is None
-        assert h.getarrayitem(box1, descr2, index1) is None
-        assert h.arraylen(box1) is None
         assert h.arraylen(box4) is lengthbox1
         assert h.getarrayitem(box4, descr1, index1) is box2
         assert h.getarrayitem(box4, descr2, index1) is box3
@@ -304,6 +335,27 @@
         h.replace_box(lengthbox1, lengthbox2)
         assert h.arraylen(box4) is lengthbox2
 
+    def test_replace_box_array_twice(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        h.replace_box(box4, box5)
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box5, descr1, index1) is box2
+        assert h.getarrayitem(box5, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box5
+        assert h.getarrayitem(box3, descr2, index2) is box5
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        h.replace_box(lengthbox2, lengthbox3)
+        assert h.arraylen(box4) is lengthbox3
+
     def test_ll_arraycopy(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -103,8 +103,8 @@
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
-udir.join('pypy_decl.h').write("/* Will be filled later */")
-udir.join('pypy_macros.h').write("/* Will be filled later */")
+udir.join('pypy_decl.h').write("/* Will be filled later */\n")
+udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 globals().update(rffi_platform.configure(CConfig_constants))
 
 def copy_header_files(dstdir):
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -38,10 +38,19 @@
 	PyObject_VAR_HEAD
 } PyVarObject;
 
+#ifndef PYPY_DEBUG_REFCOUNT
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)ob))
 #define Py_DECREF(ob)   (Py_DecRef((PyObject *)ob))
 #define Py_XINCREF(ob)  (Py_IncRef((PyObject *)ob))
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)ob))
+#else
+#define Py_INCREF(ob)   (((PyObject *)ob)->ob_refcnt++)
+#define Py_DECREF(ob)  ((((PyObject *)ob)->ob_refcnt > 1) ? \
+			((PyObject *)ob)->ob_refcnt-- : (Py_DecRef((PyObject *)ob)))
+
+#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
+#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
+#endif
 
 #define Py_CLEAR(op)				\
         do {                            	\
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -110,6 +110,16 @@
     space.call_method(w_list, "reverse")
     return 0
 
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyList_GetSlice(space, w_list, low, high):
+    """Return a list of the objects in list containing the objects between low
+    and high.  Return NULL and set an exception if unsuccessful.  Analogous
+    to list[low:high].  Negative indices, as when slicing from Python, are not
+    supported."""
+    w_start = space.wrap(low)
+    w_stop = space.wrap(high)
+    return space.getslice(w_list, w_start, w_stop)
+
 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyList_SetSlice(space, w_list, low, high, w_sequence):
     """Set the slice of list between low and high to the contents of
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -381,6 +381,15 @@
     This is the equivalent of the Python expression hash(o)."""
     return space.int_w(space.hash(w_obj))
 
+ at cpython_api([PyObject], lltype.Signed, error=-1)
+def PyObject_HashNotImplemented(space, o):
+    """Set a TypeError indicating that type(o) is not hashable and return -1.
+    This function receives special treatment when stored in a tp_hash slot,
+    allowing a type to explicitly indicate to the interpreter that it is not
+    hashable.
+    """
+    raise OperationError(space.w_TypeError, space.wrap("unhashable type"))
+
 @cpython_api([PyObject], PyObject)
 def PyObject_Dir(space, w_o):
     """This is equivalent to the Python expression dir(o), returning a (possibly
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -7,7 +7,7 @@
     cpython_api, generic_cpy_call, PyObject, Py_ssize_t)
 from pypy.module.cpyext.typeobjectdefs import (
     unaryfunc, wrapperfunc, ternaryfunc, PyTypeObjectPtr, binaryfunc,
-    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc,
+    getattrfunc, getattrofunc, setattrofunc, lenfunc, ssizeargfunc, inquiry,
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc)
@@ -60,6 +60,16 @@
     args_w = space.fixedview(w_args)
     return generic_cpy_call(space, func_binary, w_self, args_w[0])
 
+def wrap_inquirypred(space, w_self, w_args, func):
+    func_inquiry = rffi.cast(inquiry, func)
+    check_num_args(space, w_args, 0)
+    args_w = space.fixedview(w_args)
+    res = generic_cpy_call(space, func_inquiry, w_self)
+    res = rffi.cast(lltype.Signed, res)
+    if res == -1:
+        space.fromcache(State).check_and_raise_exception()
+    return space.wrap(bool(res))
+
 def wrap_getattr(space, w_self, w_args, func):
     func_target = rffi.cast(getattrfunc, func)
     check_num_args(space, w_args, 1)
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -294,6 +294,26 @@
         w_errors = space.wrap(rffi.charp2str(errors))
     return space.call_method(w_str, 'encode', w_encoding, w_errors)
 
+ at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
+def PyString_AsDecodedObject(space, w_str, encoding, errors):
+    """Decode a string object by passing it to the codec registered
+    for encoding and return the result as Python object. encoding and
+    errors have the same meaning as the parameters of the same name in
+    the string encode() method.  The codec to be used is looked up
+    using the Python codec registry. Return NULL if an exception was
+    raised by the codec.
+
+    This function is not available in 3.x and does not have a PyBytes alias."""
+    if not PyString_Check(space, w_str):
+        PyErr_BadArgument(space)
+
+    w_encoding = w_errors = space.w_None
+    if encoding:
+        w_encoding = space.wrap(rffi.charp2str(encoding))
+    if errors:
+        w_errors = space.wrap(rffi.charp2str(errors))
+    return space.call_method(w_str, "decode", w_encoding, w_errors)
+
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1405,17 +1405,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
-def PyList_GetSlice(space, list, low, high):
-    """Return a list of the objects in list containing the objects between low
-    and high.  Return NULL and set an exception if unsuccessful.  Analogous
-    to list[low:high].  Negative indices, as when slicing from Python, are not
-    supported.
-
-    This function used an int for low and high. This might
-    require changes in your code for properly supporting 64-bit systems."""
-    raise NotImplementedError
-
 @cpython_api([Py_ssize_t], PyObject)
 def PyLong_FromSsize_t(space, v):
     """Return a new PyLongObject object from a C Py_ssize_t, or
@@ -1606,15 +1595,6 @@
     for PyObject_Str()."""
     raise NotImplementedError
 
- at cpython_api([PyObject], lltype.Signed, error=-1)
-def PyObject_HashNotImplemented(space, o):
-    """Set a TypeError indicating that type(o) is not hashable and return -1.
-    This function receives special treatment when stored in a tp_hash slot,
-    allowing a type to explicitly indicate to the interpreter that it is not
-    hashable.
-    """
-    raise NotImplementedError
-
 @cpython_api([], PyFrameObject)
 def PyEval_GetFrame(space):
     """Return the current thread state's frame, which is NULL if no frame is
@@ -1737,17 +1717,6 @@
     changes in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsDecodedObject(space, str, encoding, errors):
-    """Decode a string object by passing it to the codec registered for encoding and
-    return the result as Python object. encoding and errors have the same
-    meaning as the parameters of the same name in the string encode() method.
-    The codec to be used is looked up using the Python codec registry. Return NULL
-    if an exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Encode(space, s, size, encoding, errors):
     """Encode the char buffer of the given size by passing it to the codec
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -58,6 +58,11 @@
         w_t = api.PyList_AsTuple(w_l)
         assert space.unwrap(w_t) == (3, 2, 1)
 
+    def test_list_getslice(self, space, api):
+        w_l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
+        w_s = api.PyList_GetSlice(w_l, 1, 5)
+        assert space.unwrap(w_s) == [2, 1]
+
 class AppTestListObject(AppTestCpythonExtensionBase):
     def test_listobject(self):
         import sys
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -307,6 +307,13 @@
             space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
         )
 
+    def test_AsDecodedObject(self, space, api):
+        w_str = space.wrap('caf\xe9')
+        encoding = rffi.str2charp("latin-1")
+        w_res = api.PyString_AsDecodedObject(w_str, encoding, None)
+        rffi.free_charp(encoding)
+        assert space.unwrap(w_res) == u"caf\xe9"
+
     def test_eq(self, space, api):
         assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
         assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -488,3 +488,55 @@
         assert type(it) is type(iter([]))
         assert module.tp_iternext(it) == 1
         raises(StopIteration, module.tp_iternext, it)
+        
+    def test_bool(self):
+        module = self.import_extension('foo', [
+            ("newInt", "METH_VARARGS",
+             """
+                IntLikeObject *intObj;
+                long intval;
+                PyObject *name;
+
+                if (!PyArg_ParseTuple(args, "i", &intval))
+                    return NULL;
+
+                IntLike_Type.tp_as_number = &intlike_as_number;
+                intlike_as_number.nb_nonzero = intlike_nb_nonzero;
+                if (PyType_Ready(&IntLike_Type) < 0) return NULL;
+                intObj = PyObject_New(IntLikeObject, &IntLike_Type);
+                if (!intObj) {
+                    return NULL;
+                }
+
+                intObj->value = intval;
+                return (PyObject *)intObj;
+             """)],
+            """
+            typedef struct
+            {
+                PyObject_HEAD
+                int value;
+            } IntLikeObject;
+
+            static int
+            intlike_nb_nonzero(IntLikeObject *v)
+            {
+                if (v->value == -42) {
+                    PyErr_SetNone(PyExc_ValueError);
+                    return -1;
+                }
+                return v->value;
+            }
+
+            PyTypeObject IntLike_Type = {
+                PyObject_HEAD_INIT(0)
+                /*ob_size*/             0,
+                /*tp_name*/             "IntLike",
+                /*tp_basicsize*/        sizeof(IntLikeObject),
+            };
+            static PyNumberMethods intlike_as_number;
+            """)
+        assert not bool(module.newInt(0))
+        assert bool(module.newInt(1))
+        assert bool(module.newInt(-1))
+        raises(ValueError, bool, module.newInt(-42))
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,6 +5,7 @@
     interpleveldefs = {
         'debug_repr': 'interp_extras.debug_repr',
         'remove_invalidates': 'interp_extras.remove_invalidates',
+        'set_invalidation': 'interp_extras.set_invalidation',
     }
     appleveldefs = {}
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -10,6 +10,7 @@
 from pypy.module.micronumpy.interp_dtype import get_dtype_cache
 from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
      scalar_w, W_NDimArray, array)
+from pypy.module.micronumpy.interp_arrayops import where
 from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize, instantiate
 
@@ -35,6 +36,7 @@
 SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any",
                         "unegative", "flat", "tostring"]
 TWO_ARG_FUNCTIONS = ["dot", 'take']
+THREE_ARG_FUNCTIONS = ['where']
 
 class FakeSpace(object):
     w_ValueError = None
@@ -445,14 +447,25 @@
             arg = self.args[1].execute(interp)
             if not isinstance(arg, BaseArray):
                 raise ArgumentNotAnArray
-            if not isinstance(arg, BaseArray):
-                raise ArgumentNotAnArray
             if self.name == "dot":
                 w_res = arr.descr_dot(interp.space, arg)
             elif self.name == 'take':
                 w_res = arr.descr_take(interp.space, arg)
             else:
                 assert False # unreachable code
+        elif self.name in THREE_ARG_FUNCTIONS:
+            if len(self.args) != 3:
+                raise ArgumentMismatch
+            arg1 = self.args[1].execute(interp)
+            arg2 = self.args[2].execute(interp)
+            if not isinstance(arg1, BaseArray):
+                raise ArgumentNotAnArray
+            if not isinstance(arg2, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "where":
+                w_res = where(interp.space, arr, arg1, arg2)
+            else:
+                assert False
         else:
             raise WrongFunctionName
         if isinstance(w_res, BaseArray):
diff --git a/pypy/module/micronumpy/interp_arrayops.py b/pypy/module/micronumpy/interp_arrayops.py
--- a/pypy/module/micronumpy/interp_arrayops.py
+++ b/pypy/module/micronumpy/interp_arrayops.py
@@ -4,7 +4,7 @@
 from pypy.module.micronumpy import signature
 
 class WhereArray(VirtualArray):
-    def __init__(self, arr, x, y):
+    def __init__(self, space, arr, x, y):
         self.arr = arr
         self.x = x
         self.y = y
@@ -87,4 +87,4 @@
     arr = convert_to_array(space, w_arr)
     x = convert_to_array(space, w_x)
     y = convert_to_array(space, w_y)
-    return WhereArray(arr, x, y)
+    return WhereArray(space, arr, x, y)
diff --git a/pypy/module/micronumpy/interp_extras.py b/pypy/module/micronumpy/interp_extras.py
--- a/pypy/module/micronumpy/interp_extras.py
+++ b/pypy/module/micronumpy/interp_extras.py
@@ -1,5 +1,5 @@
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.module.micronumpy.interp_numarray import BaseArray
+from pypy.module.micronumpy.interp_numarray import BaseArray, get_numarray_cache
 
 
 @unwrap_spec(array=BaseArray)
@@ -13,3 +13,7 @@
     """
     del array.invalidates[:]
     return space.w_None
+
+ at unwrap_spec(arg=bool)
+def set_invalidation(space, arg):
+    get_numarray_cache(space).enable_invalidation = arg
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,9 +72,10 @@
             arr.force_if_needed()
         del self.invalidates[:]
 
-    def add_invalidates(self, other):
-        self.invalidates.append(other)
-
+    def add_invalidates(self, space, other):
+        if get_numarray_cache(space).enable_invalidation:
+            self.invalidates.append(other)
+        
     def descr__new__(space, w_subtype, w_size, w_dtype=None):
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
@@ -1583,3 +1584,10 @@
         arr.fill(space, space.wrap(False))
         return arr
     return space.wrap(False)
+
+class NumArrayCache(object):
+    def __init__(self, space):
+        self.enable_invalidation = True
+
+def get_numarray_cache(space):
+    return space.fromcache(NumArrayCache)
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -278,7 +278,7 @@
         else:
             w_res = Call1(self.func, self.name, w_obj.shape, calc_dtype,
                                          res_dtype, w_obj)
-        w_obj.add_invalidates(w_res)
+        w_obj.add_invalidates(space, w_res)
         return w_res
 
 
@@ -347,8 +347,8 @@
         w_res = Call2(self.func, self.name,
                       new_shape, calc_dtype,
                       res_dtype, w_lhs, w_rhs, out)
-        w_lhs.add_invalidates(w_res)
-        w_rhs.add_invalidates(w_res)
+        w_lhs.add_invalidates(space, w_res)
+        w_rhs.add_invalidates(space, w_res)
         if out:
             w_res.get_concrete()
         return w_res
diff --git a/pypy/module/micronumpy/test/test_arrayops.py b/pypy/module/micronumpy/test/test_arrayops.py
--- a/pypy/module/micronumpy/test/test_arrayops.py
+++ b/pypy/module/micronumpy/test/test_arrayops.py
@@ -7,3 +7,10 @@
         a = [1, 2, 3, 0, -3]
         a = where(array(a) > 0, ones(5), zeros(5))
         assert (a == [1, 1, 1, 0, 0]).all()
+
+    def test_where_invalidates(self):
+        from _numpypy import where, ones, zeros, array
+        a = array([1, 2, 3, 0, -3])
+        b = where(a > 0, ones(5), zeros(5))
+        a[0] = 0
+        assert (b == [1, 1, 1, 0, 0]).all()
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -270,3 +270,13 @@
         b -> 2
         """)
         assert interp.results[0].value == 3
+
+    def test_where(self):
+        interp = self.run('''
+        a = [1, 0, 3, 0]
+        b = [1, 1, 1, 1]
+        c = [0, 0, 0, 0]
+        d = where(a, b, c)
+        d -> 1
+        ''')
+        assert interp.results[0].value == 0
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -1830,6 +1830,19 @@
         a[a & 1 == 1] = array([8, 9, 10])
         assert (a == [[0, 8], [2, 9], [4, 10]]).all()
 
+    def test_array_indexing_bool_setitem_multidim(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 3], [15, 5], [15, 7], [15, 9]]).all()
+
+    def test_array_indexing_bool_setitem_2(self):
+        from _numpypy import arange
+        a = arange(10).reshape(5, 2)
+        a = a[::2]
+        a[a & 1 == 0] = 15
+        assert (a == [[15, 1], [15, 5], [15, 9]]).all()
+
     def test_copy_kwarg(self):
         from _numpypy import array
         x = array([1, 2, 3])
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1,7 +1,6 @@
 
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-
 class AppTestUfuncs(BaseNumpyAppTest):
     def test_ufunc_instance(self):
         from _numpypy import add, ufunc
@@ -149,7 +148,11 @@
         assert math.isnan(fmax(0, nan))
         assert math.isnan(fmax(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmax(nnan, nan)) == -1.0
+        # Since comparisons with nnan and nan all return false,
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmax(nnan, nan)) == math.copysign(1., nnan)
+
 
     def test_fmin(self):
         from _numpypy import fmin
@@ -165,7 +168,9 @@
         assert math.isnan(fmin(0, nan))
         assert math.isnan(fmin(nan, nan))
         # The numpy docs specify that the FIRST NaN should be used if both are NaN
-        assert math.copysign(1.0, fmin(nnan, nan)) == -1.0
+        # use copysign on both sides to sidestep bug in nan representaion
+        # on Microsoft win32
+        assert math.copysign(1., fmin(nnan, nan)) == math.copysign(1., nnan)
 
     def test_fmod(self):
         from _numpypy import fmod
diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py
--- a/pypy/module/mmap/test/test_mmap.py
+++ b/pypy/module/mmap/test/test_mmap.py
@@ -596,7 +596,7 @@
         import sys
         size = 0x14FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
@@ -618,7 +618,7 @@
         import sys
         size = 0x17FFFFFFF
         if sys.platform.startswith('win') or sys.platform == 'darwin':
-            self.skip('test requires %s bytes and a long time to run' % size)
+            skip('test requires %s bytes and a long time to run' % size)
 
         with open(self.tmpname, "w+b") as f:
             f.seek(size)
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assert=plain -rf
+addopts = --assert=reinterp -rf
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -40,7 +40,7 @@
 # In that case, do 5 bits at a time.  The potential drawback is that
 # a table of 2**5 intermediate results is computed.
 
-## FIVEARY_CUTOFF = 8   disabled for now
+FIVEARY_CUTOFF = 8
 
 
 def _mask_digit(x):
@@ -456,7 +456,7 @@
 
         # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
         # into helper function result = _help_mult(x, y, c)
-        if 1:   ## b.numdigits() <= FIVEARY_CUTOFF:
+        if b.numdigits() <= FIVEARY_CUTOFF:
             # Left-to-right binary exponentiation (HAC Algorithm 14.79)
             # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
             i = b.numdigits() - 1
@@ -469,30 +469,51 @@
                         z = _help_mult(z, a, c)
                     j >>= 1
                 i -= 1
-##        else:
-##            This code is disabled for now, because it assumes that
-##            SHIFT is a multiple of 5.  It could be fixed but it looks
-##            like it's more troubles than benefits...
-##
-##            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-##            # This is only useful in the case where c != None.
-##            # z still holds 1L
-##            table = [z] * 32
-##            table[0] = z
-##            for i in range(1, 32):
-##                table[i] = _help_mult(table[i-1], a, c)
-##            i = b.numdigits() - 1
-##            while i >= 0:
-##                bi = b.digit(i)
-##                j = SHIFT - 5
-##                while j >= 0:
-##                    index = (bi >> j) & 0x1f
-##                    for k in range(5):
-##                        z = _help_mult(z, z, c)
-##                    if index:
-##                        z = _help_mult(z, table[index], c)
-##                    j -= 5
-##                i -= 1
+        else:
+            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
+            # This is only useful in the case where c != None.
+            # z still holds 1L
+            table = [z] * 32
+            table[0] = z
+            for i in range(1, 32):
+                table[i] = _help_mult(table[i-1], a, c)
+            i = b.numdigits()
+            # Note that here SHIFT is not a multiple of 5.  The difficulty
+            # is to extract 5 bits at a time from 'b', starting from the
+            # most significant digits, so that at the end of the algorithm
+            # it falls exactly to zero.
+            # m  = max number of bits = i * SHIFT
+            # m+ = m rounded up to the next multiple of 5
+            # j  = (m+) % SHIFT = (m+) - (i * SHIFT)
+            # (computed without doing "i * SHIFT", which might overflow)
+            j = i % 5
+            if j != 0:
+                j = 5 - j
+            if not we_are_translated():
+                assert j == (i*SHIFT+4)//5*5 - i*SHIFT
+            #
+            accum = r_uint(0)
+            while True:
+                j -= 5
+                if j >= 0:
+                    index = (accum >> j) & 0x1f
+                else:
+                    # 'accum' does not have enough digit.
+                    # must get the next digit from 'b' in order to complete
+                    i -= 1
+                    if i < 0:
+                        break    # done
+                    bi = b.udigit(i)
+                    index = ((accum << (-j)) | (bi >> (j+SHIFT))) & 0x1f
+                    accum = bi
+                    j += SHIFT
+                #
+                for k in range(5):
+                    z = _help_mult(z, z, c)
+                if index:
+                    z = _help_mult(z, table[index], c)
+            #
+            assert j == -5
 
         if negativeOutput and z.sign != 0:
             z = z.sub(c)
diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -1234,7 +1234,7 @@
             pos += 1
             continue
 
-        if 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
             # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
             pos += 1
             oc2 = ord(s[pos])
@@ -1350,6 +1350,20 @@
     pos = 0
     while pos < size:
         oc = ord(s[pos])
+
+        if MAXUNICODE < 65536 and 0xD800 <= oc < 0xDC00 and pos + 1 < size:
+            # Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes
+            pos += 1
+            oc2 = ord(s[pos])
+
+            if 0xDC00 <= oc2 <= 0xDFFF:
+                ucs = (((oc & 0x03FF) << 10) | (oc2 & 0x03FF)) + 0x00010000
+                raw_unicode_escape_helper(result, ucs)
+                pos += 1
+                continue
+            # Fall through: isolated surrogates are copied as-is
+            pos -= 1
+
         if oc < 0x100:
             result.append(chr(oc))
         else:
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint.py
@@ -379,6 +379,18 @@
         for n, expected in [(37, 9), (1291, 931), (67889, 39464)]:
             v = two.pow(t, rbigint.fromint(n))
             assert v.toint() == expected
+        #
+        # more tests, comparing against CPython's answer
+        enabled = sample(range(5*32), 10)
+        for i in range(5*32):
+            t = t.mul(two)      # add one random bit
+            if random() >= 0.5:
+                t = t.add(rbigint.fromint(1))
+            if i not in enabled:
+                continue    # don't take forever
+            n = randint(1, sys.maxint)
+            v = two.pow(t, rbigint.fromint(n))
+            assert v.toint() == pow(2, t.tolong(), n)
 
     def test_pow_lln(self):
         x = 10L
diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py
--- a/pypy/rlib/test/test_runicode.py
+++ b/pypy/rlib/test/test_runicode.py
@@ -728,3 +728,18 @@
 
         res = interpret(f, [0x10140])
         assert res == 0x10140
+
+    def test_encode_surrogate_pair(self):
+        u = runicode.UNICHR(0xD800) + runicode.UNICHR(0xDC00)
+        if runicode.MAXUNICODE < 65536:
+            # Narrow unicode build, consider utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\U00010000'
+        else:
+            # Wide unicode build, don't merge utf16 surrogate pairs
+            assert runicode.unicode_encode_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'
+            assert runicode.unicode_encode_raw_unicode_escape(
+                u, len(u), True) == r'\ud800\udc00'
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1180,7 +1180,7 @@
         try:
             return self._lookup_adtmeth(field_name)
         except AttributeError:
-            raise AttributeError("%r instance has no field %r" % (self._T._name,
+            raise AttributeError("%r instance has no field %r" % (self._T,
                                                                   field_name))
 
     def __setattr__(self, field_name, val):
diff --git a/pypy/rpython/tool/rffi_platform.py b/pypy/rpython/tool/rffi_platform.py
--- a/pypy/rpython/tool/rffi_platform.py
+++ b/pypy/rpython/tool/rffi_platform.py
@@ -379,7 +379,7 @@
         self.name = name
 
     def prepare_code(self):
-        yield 'if ((%s) < 0) {' % (self.name,)
+        yield 'if ((%s) <= 0) {' % (self.name,)
         yield '    long long x = (long long)(%s);' % (self.name,)
         yield '    printf("value: %lld\\n", x);'
         yield '} else {'
@@ -401,7 +401,7 @@
     def prepare_code(self):
         yield '#ifdef %s' % self.macro
         yield 'dump("defined", 1);'
-        yield 'if ((%s) < 0) {' % (self.macro,)
+        yield 'if ((%s) <= 0) {' % (self.macro,)
         yield '    long long x = (long long)(%s);' % (self.macro,)
         yield '    printf("value: %lld\\n", x);'
         yield '} else {'
diff --git a/pypy/tool/compare_last_builds.py b/pypy/tool/compare_last_builds.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/compare_last_builds.py
@@ -0,0 +1,122 @@
+import os
+import urllib2
+import json
+import sys
+import md5
+
+wanted = sys.argv[1:]
+if not wanted:
+    wanted = ['default']
+base = "http://buildbot.pypy.org/json/builders/"
+
+cachedir = os.environ.get('PYPY_BUILDS_CACHE')
+if cachedir and not os.path.exists(cachedir):
+    os.makedirs(cachedir)
+
+
+
+def get_json(url, cache=cachedir):
+    return json.loads(get_data(url, cache))
+
+
+def get_data(url, cache=cachedir):
+    url = str(url)
+    if cache:
+        digest = md5.md5()
+        digest.update(url)
+        digest = digest.hexdigest()
+        cachepath = os.path.join(cachedir, digest)
+        if os.path.exists(cachepath):
+            with open(cachepath) as fp:
+                return fp.read()
+
+    print 'GET', url
+    fp = urllib2.urlopen(url)
+    try:
+        data = fp.read()
+        if cache:
+            with open(cachepath, 'wb') as cp:
+                cp.write(data)
+        return data
+    finally:
+        fp.close()
+
+def parse_log(log):
+    items = []
+    for v in log.splitlines(1):
+        if not v[0].isspace() and v[1].isspace():
+            items.append(v)
+    return sorted(items) #sort cause testrunner order is non-deterministic
+
+def gather_logdata(build):
+    logdata = get_data(str(build['log']) + '?as_text=1')
+    logdata = logdata.replace('</span><span class="stdout">', '')
+    logdata = logdata.replace('</span></pre>', '')
+    del build['log']
+    build['log'] = parse_log(logdata)
+
+
+def branch_mapping(l):
+    keep = 3 - len(wanted)
+    d = {}
+    for x in reversed(l):
+        gather_logdata(x)
+        if not x['log']:
+            continue
+        b = x['branch']
+        if b not in d:
+            d[b] = []
+        d[b].insert(0, x)
+        if len(d[b]) > keep:
+            d[b].pop()
+    return d
+
+def cleanup_build(d):
+    for a in 'times eta steps slave reason sourceStamp blame currentStep text'.split():
+        del d[a]
+
+    props = d.pop(u'logs')
+    for name, val in props:
+        if name == u'pytestLog':
+            d['log'] = val
+    props = d.pop(u'properties')
+    for name, val, _ in props:
+        if name == u'branch':
+            d['branch'] = val or 'default'
+    return d
+
+def collect_builds(d):
+    name = str(d['basedir'])
+    builds = d['cachedBuilds']
+    l = []
+    for build in builds:
+        d = get_json(base + '%s/builds/%s' % (name, build))
+        cleanup_build(d)
+        l.append(d)
+
+    l = [x for x in l if x['branch'] in wanted and 'log' in x]
+    d = branch_mapping(l)
+    return [x for lst in d.values() for x in lst]
+
+
+def only_linux32(d):
+    return d['own-linux-x86-32']
+
+
+own_builds = get_json(base, cache=False)['own-linux-x86-32']
+
+builds = collect_builds(own_builds)
+
+
+builds.sort(key=lambda x: (wanted.index(x['branch']), x['number']))
+logs = [x.pop('log') for x in builds]
+for b, s in zip(builds, logs):
+    b['resultset'] = len(s)
+import pprint
+pprint.pprint(builds)
+
+from difflib import Differ
+
+for x in Differ().compare(*logs):
+    if x[0]!=' ':
+        sys.stdout.write(x)
diff --git a/pypy/tool/pytest/pypy_test_failure_demo.py b/pypy/tool/pytest/pypy_test_failure_demo.py
--- a/pypy/tool/pytest/pypy_test_failure_demo.py
+++ b/pypy/tool/pytest/pypy_test_failure_demo.py
@@ -8,6 +8,10 @@
 def test_interp_func(space): 
     assert space.is_true(space.w_None) 
 
+def test_interp_reinterpret(space):
+    a = 1
+    assert a == 2
+
 class TestInterpTest: 
     def test_interp_method(self): 
         assert self.space.is_true(self.space.w_False) 
diff --git a/pypy/translator/c/src/cjkcodecs/cjkcodecs.h b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
--- a/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
@@ -210,15 +210,15 @@
 
 #define BEGIN_CODECS_LIST /* empty */
 #define _CODEC(name)                                                    \
-  static const MultibyteCodec _pypy_cjkcodec_##name;                    \
-  const MultibyteCodec *pypy_cjkcodec_##name(void) {                    \
+  static MultibyteCodec _pypy_cjkcodec_##name;                          \
+  MultibyteCodec *pypy_cjkcodec_##name(void) {                          \
     if (_pypy_cjkcodec_##name.codecinit != NULL) {                      \
       int r = _pypy_cjkcodec_##name.codecinit(_pypy_cjkcodec_##name.config); \
       assert(r == 0);                                                   \
     }                                                                   \
     return &_pypy_cjkcodec_##name;                                      \
   }                                                                     \
-  static const MultibyteCodec _pypy_cjkcodec_##name
+  static MultibyteCodec _pypy_cjkcodec_##name
 #define _STATEFUL_METHODS(enc)          \
     enc##_encode,                       \
     enc##_encode_init,                  \
diff --git a/pypy/translator/c/src/cjkcodecs/multibytecodec.h b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.h
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
@@ -131,7 +131,7 @@
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    const MultibyteCodec *pypy_cjkcodec_##name(void);
+    MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)

From noreply at buildbot.pypy.org  Thu Apr 26 10:57:44 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 10:57:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Optimization disabled; see 27558fbd3171.
Message-ID: <20120426085744.77BF982009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54763:bc2fd3dc86d9
Date: 2012-04-26 10:57 +0200
http://bitbucket.org/pypy/pypy/changeset/bc2fd3dc86d9/

Log:	Optimization disabled; see 27558fbd3171.

diff --git a/lib-python/modified-2.7/test/test_peepholer.py b/lib-python/modified-2.7/test/test_peepholer.py
--- a/lib-python/modified-2.7/test/test_peepholer.py
+++ b/lib-python/modified-2.7/test/test_peepholer.py
@@ -145,12 +145,15 @@
 
     def test_binary_subscr_on_unicode(self):
         # valid code get optimized
-        asm = dis_single('u"foo"[0]')
-        self.assertIn("(u'f')", asm)
-        self.assertNotIn('BINARY_SUBSCR', asm)
-        asm = dis_single('u"\u0061\uffff"[1]')
-        self.assertIn("(u'\\uffff')", asm)
-        self.assertNotIn('BINARY_SUBSCR', asm)
+        # XXX for now we always disable this optimization
+        # XXX see CPython's issue5057
+        if 0:
+            asm = dis_single('u"foo"[0]')
+            self.assertIn("(u'f')", asm)
+            self.assertNotIn('BINARY_SUBSCR', asm)
+            asm = dis_single('u"\u0061\uffff"[1]')
+            self.assertIn("(u'\\uffff')", asm)
+            self.assertNotIn('BINARY_SUBSCR', asm)
 
         # invalid code doesn't get optimized
         # out of range

From noreply at buildbot.pypy.org  Thu Apr 26 11:15:55 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 11:15:55 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Use pthread_join() instead of a lock.
 This makes sure that the threads
Message-ID: <20120426091555.A27C682009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54764:e9d12083d556
Date: 2012-04-26 11:15 +0200
http://bitbucket.org/pypy/pypy/changeset/e9d12083d556/

Log:	Use pthread_join() instead of a lock. This makes sure that the
	threads are really down before returning.

diff --git a/pypy/translator/stm/src_stm/rpyintf.c b/pypy/translator/stm/src_stm/rpyintf.c
--- a/pypy/translator/stm/src_stm/rpyintf.c
+++ b/pypy/translator/stm/src_stm/rpyintf.c
@@ -3,17 +3,12 @@
 #include "src_stm/fifo.c"
 
 
-/* this lock is acquired when we start running transactions, and
-   released only when we are finished. */
-static pthread_mutex_t mutex_unfinished = PTHREAD_MUTEX_INITIALIZER;
-
 /* this mutex is used to ensure non-conflicting accesses to global
    data in run_thread(). */
 static pthread_mutex_t mutex_global = PTHREAD_MUTEX_INITIALIZER;
 
 /* this lock is acquired if and only if there are no tasks pending,
-   i.e. the linked list stm_g_first_transaction ... stm_g_last_transaction is
-   empty and both pointers are NULL. */
+   i.e. the fifo stm_g_pending is empty. */
 static pthread_mutex_t mutex_no_tasks_pending = PTHREAD_MUTEX_INITIALIZER;
 
 /* some global data put there by run_all_transactions(). */
@@ -110,8 +105,6 @@
     }
 
   pypy_g__stm_thread_stopping();
-  if (stm_g_num_waiting_threads == 0)   /* only the last thread to leave */
-    pthread_mutex_unlock(&mutex_unfinished);
   pthread_mutex_unlock(&mutex_global);
   return NULL;
 }
@@ -119,28 +112,42 @@
 void stm_run_all_transactions(void *initial_transaction,
                               long num_threads)
 {
-  int i;
+  long i;
+  pthread_t *th = malloc(num_threads * sizeof(pthread_t*));
+  if (th == NULL)
+    {
+      /* XXX turn into a nice exception */
+      fprintf(stderr, "out of memory: too many threads?\n");
+      exit(1);
+    }
+
   fifo_init(&stm_g_pending);
   fifo_append(&stm_g_pending, initial_transaction);
   stm_g_num_threads = (int)num_threads;
   stm_g_num_waiting_threads = 0;
   stm_g_finished = 0;
 
-  pthread_mutex_lock(&mutex_unfinished);
-
-  for (i=0; i<(int)num_threads; i++)
+  for (i=0; i<num_threads; i++)
     {
-      pthread_t th;
-      int status = pthread_create(&th, NULL, run_thread, NULL);
+      int status = pthread_create(&th[i], NULL, run_thread, NULL);
       if (status != 0)
         {
           /* XXX turn into a nice exception */
-          fprintf(stderr, "fatal error: cannot create threads\n");
+          fprintf(stderr, "fatal error: cannot create thread %ld/%ld\n",
+                  i, num_threads);
           exit(1);
         }
-      pthread_detach(th);
     }
 
-  pthread_mutex_lock(&mutex_unfinished);
-  pthread_mutex_unlock(&mutex_unfinished);
+  for (i=0; i<num_threads; i++)
+    {
+      void *retval = NULL;
+      int ret = pthread_join(th[i], &retval);
+      if (ret != 0 || retval != NULL)
+        {
+          /* XXX? */
+          fprintf(stderr, "warning: thread %ld/%ld exited with %d (%p)\n",
+                  i, num_threads, ret, retval);
+        }
+    }
 }

From noreply at buildbot.pypy.org  Thu Apr 26 18:08:28 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 18:08:28 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Close this branch,
	now that stm-gc seems to work.
Message-ID: <20120426160828.3B63882009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r54765:9ab33061def3
Date: 2012-04-26 18:08 +0200
http://bitbucket.org/pypy/pypy/changeset/9ab33061def3/

Log:	Close this branch, now that stm-gc seems to work.


From noreply at buildbot.pypy.org  Thu Apr 26 18:39:22 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 18:39:22 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: A draft.
Message-ID: <20120426163922.208DE82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4196:6bba6778891a
Date: 2012-04-26 18:39 +0200
http://bitbucket.org/pypy/extradoc/changeset/6bba6778891a/

Log:	A draft.

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/stm-apr2012.rst
@@ -0,0 +1,48 @@
+STM update (and thanks everybody)
+=================================
+
+A short update on the Software Transactional Memory (STM) side.  I have
+now reached the point where the basics seem to work, and the integration
+with the Garbage Collection subsystem is --- not done by far, but at
+least "not crashing in my simple tests and not leaking memory too
+quickly".  (It is leaking a bit of memory though, and it is never
+calling ``__del__`` so far.)
+
+If you want to play with it, you can download `this binary`_ (you need to
+put it in a place with the paths ``lib-python`` and ``lib_pypy``, for
+example from a regular `nightly tarball`_ or from a full checkout).  It
+is for Linux 32-bit; I didn't do fixes for other platforms for now.
+It was compiled from the `stm-gc`_ branch on the 25th of April.
+It runs e.g. the `modified version of richards`_.
+
+.. _`this binary`: http://wyvern.cs.uni-duesseldorf.de/~arigo/pypy-stm-22fccf3c9b5e.tar.bz2
+.. _`nightly tarball`: http://buildbot.pypy.org/nightly/trunk/
+.. _`stm-gc`: https://bitbucket.org/pypy/pypy/src/stm-gc
+.. _`modified version of richards`: https://bitbucket.org/pypy/pypy/raw/stm-gc/pypy/translator/stm/test/richards.py
+
+It exposes the same interface as the transaction_ module, except it
+doesn't support epoll right now, so it cannot be used to play with `a
+branch of Twisted`_; but that's coming soon.  For now you can use it to
+get multi-core usage on purely computational programs, like PyPy's own
+``translate.py``, for which I did a tweak `in rpython/rtyper.py`_ (lines
+273-281 are all that I needed to add).
+
+.. _transaction: https://bitbucket.org/pypy/pypy/raw/stm-gc/lib_pypy/transaction.py
+.. _`a branch of Twisted`: svn://svn.twistedmatrix.com/svn/Twisted/branches/stm-5526
+.. _`in rpython/rtyper.py`: https://bitbucket.org/pypy/pypy/src/stm-gc/pypy/rpython/rtyper.py#cl-249
+
+The performance is not great, even taking into account that it has no
+JIT so far, but it seems to scale; at least, it does scale on my
+2-real-cores, 4-hyperthreaded-cores laptop, roughly as expected.
+
+And...
+
+...a bit thank you to everyone who contributed some money to support
+this!  As you see on the PyPy_ site, we got more than 6700$ so far in
+only 5 or 6 weeks.  Thanks to that, my contract started last Monday, and
+I am now paid a small salary via the `Software Freedom Conservancy`_
+(thanks Bradley M. Kuhn for organizational support from the SFC).
+Again, thank you everybody!
+
+.. _PyPy: http://pypy.org/
+.. _`Software Freedom Conservancy`: http://sfconservancy.org/

From noreply at buildbot.pypy.org  Thu Apr 26 18:42:28 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 18:42:28 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: A precision.
Message-ID: <20120426164228.E8A3D82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4197:979caca9d6ed
Date: 2012-04-26 18:42 +0200
http://bitbucket.org/pypy/extradoc/changeset/979caca9d6ed/

Log:	A precision.

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
--- a/blog/draft/stm-apr2012.rst
+++ b/blog/draft/stm-apr2012.rst
@@ -20,7 +20,9 @@
 .. _`stm-gc`: https://bitbucket.org/pypy/pypy/src/stm-gc
 .. _`modified version of richards`: https://bitbucket.org/pypy/pypy/raw/stm-gc/pypy/translator/stm/test/richards.py
 
-It exposes the same interface as the transaction_ module, except it
+It exposes the same interface as the pure Python transaction_ module
+(except of course that it's not using the naive implementation linked
+above, but its own).  A difference is that it
 doesn't support epoll right now, so it cannot be used to play with `a
 branch of Twisted`_; but that's coming soon.  For now you can use it to
 get multi-core usage on purely computational programs, like PyPy's own

From noreply at buildbot.pypy.org  Thu Apr 26 18:44:19 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 18:44:19 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: More precisions.
Message-ID: <20120426164419.D0F9082009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4198:28b41dd266d6
Date: 2012-04-26 18:44 +0200
http://bitbucket.org/pypy/extradoc/changeset/28b41dd266d6/

Log:	More precisions.

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
--- a/blog/draft/stm-apr2012.rst
+++ b/blog/draft/stm-apr2012.rst
@@ -26,8 +26,9 @@
 doesn't support epoll right now, so it cannot be used to play with `a
 branch of Twisted`_; but that's coming soon.  For now you can use it to
 get multi-core usage on purely computational programs, like PyPy's own
-``translate.py``, for which I did a tweak `in rpython/rtyper.py`_ (lines
-273-281 are all that I needed to add).
+``translate.py`` --- for that I did a tweak `in rpython/rtyper.py`_ (lines
+273-281 are all that I needed to add), but there are a few more places
+in the whole ``translate.py`` that could be similarly enhanced.
 
 .. _transaction: https://bitbucket.org/pypy/pypy/raw/stm-gc/lib_pypy/transaction.py
 .. _`a branch of Twisted`: svn://svn.twistedmatrix.com/svn/Twisted/branches/stm-5526

From noreply at buildbot.pypy.org  Thu Apr 26 18:44:50 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 26 Apr 2012 18:44:50 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Duh
Message-ID: <20120426164450.3694882009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4199:e08b3337d75b
Date: 2012-04-26 18:44 +0200
http://bitbucket.org/pypy/extradoc/changeset/e08b3337d75b/

Log:	Duh

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
--- a/blog/draft/stm-apr2012.rst
+++ b/blog/draft/stm-apr2012.rst
@@ -40,7 +40,7 @@
 
 And...
 
-...a bit thank you to everyone who contributed some money to support
+...a big thank you to everyone who contributed some money to support
 this!  As you see on the PyPy_ site, we got more than 6700$ so far in
 only 5 or 6 weeks.  Thanks to that, my contract started last Monday, and
 I am now paid a small salary via the `Software Freedom Conservancy`_

From noreply at buildbot.pypy.org  Thu Apr 26 20:26:20 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 26 Apr 2012 20:26:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: add validate_fd for windows and test it
Message-ID: <20120426182620.B4D1C82009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54766:31add8845fcc
Date: 2012-04-26 19:21 +0300
http://bitbucket.org/pypy/pypy/changeset/31add8845fcc/

Log:	add validate_fd for windows and test it

diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -1,9 +1,11 @@
 import os
-from pypy.rpython.lltypesystem.rffi import CConstant, CExternVariable, INT
+from pypy.rpython.lltypesystem.rffi import (CConstant, CExternVariable,
+        INT, CCHARPP)
 from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.rarithmetic import intmask
 from pypy.rlib.objectmodel import specialize
+from pypy.rlib import jit
 
 class CConstantErrno(CConstant):
     # these accessors are used when calling get_errno() or set_errno()
@@ -18,9 +20,69 @@
     def __setitem__(self, index, value):
         assert index == 0
         ll2ctypes.TLS.errno = value
+if os.name == 'nt':
+    separate_module_sources =['''
+        /* Lifted completely from CPython 3.3 Modules/posix_module.c */
+        #include <malloc.h> /* for _msize */
+        typedef struct {
+            intptr_t osfhnd;
+            char osfile;
+        } my_ioinfo;
+        extern __declspec(dllimport) char * __pioinfo[];
+        #define IOINFO_L2E 5
+        #define IOINFO_ARRAY_ELTS   (1 << IOINFO_L2E)
+        #define IOINFO_ARRAYS 64
+        #define _NHANDLE_           (IOINFO_ARRAYS * IOINFO_ARRAY_ELTS)
+        #define FOPEN 0x01
+        #define _NO_CONSOLE_FILENO (intptr_t)-2
 
+        /* This function emulates what the windows CRT
+            does to validate file handles */
+        int
+        _PyVerify_fd(int fd)
+        {
+            const int i1 = fd >> IOINFO_L2E;
+            const int i2 = fd & ((1 << IOINFO_L2E) - 1);
+
+            static size_t sizeof_ioinfo = 0;
+
+            /* Determine the actual size of the ioinfo structure,
+             * as used by the CRT loaded in memory
+             */
+            if (sizeof_ioinfo == 0 && __pioinfo[0] != NULL) {
+                sizeof_ioinfo = _msize(__pioinfo[0]) / IOINFO_ARRAY_ELTS;
+            }
+            if (sizeof_ioinfo == 0) {
+                /* This should not happen... */
+                goto fail;
+            }
+
+            /* See that it isn't a special CLEAR fileno */
+                if (fd != _NO_CONSOLE_FILENO) {
+                /* Microsoft CRT would check that 0<=fd<_nhandle but we can't do that.  Instead
+                 * we check pointer validity and other info
+                 */
+                if (0 <= i1 && i1 < IOINFO_ARRAYS && __pioinfo[i1] != NULL) {
+                    /* finally, check that the file is open */
+                    my_ioinfo* info = (my_ioinfo*)(__pioinfo[i1] + i2 * sizeof_ioinfo);
+                    if (info->osfile & FOPEN) {
+                        return 1;
+                    }
+                }
+            }
+          fail:
+            errno = EBADF;
+            return 0;
+        }
+    ''',]
+    export_symbols = ['_PyVerify_fd']
+else:
+    separate_module_sources = []
+    export_symbols = []
 errno_eci = ExternalCompilationInfo(
-    includes=['errno.h']
+    includes=['errno.h','stdio.h'],
+    separate_module_sources = separate_module_sources,
+    export_symbols = export_symbols,
 )
 
 _get_errno, _set_errno = CExternVariable(INT, 'errno', errno_eci,
@@ -35,6 +97,21 @@
 def set_errno(errno):
     _set_errno(rffi.cast(INT, errno))
 
+if os.name == 'nt':
+    _validate_fd = rffi.llexternal(
+        "_PyVerify_fd", [rffi.INT], rffi.INT,
+        compilation_info=errno_eci,
+        )
+    @jit.dont_look_inside
+    def validate_fd(fd):
+        if not _validate_fd(fd):
+            raise OSError(get_errno(), 'Bad file descriptor')
+else:
+    def _validate_fd(fd):
+        return 1
+
+    def validate_fd(fd):
+        return 1
 
 def closerange(fd_low, fd_high):
     # this behaves like os.closerange() from Python 2.6.
diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -131,3 +131,13 @@
                 os.rmdir(self.ufilename)
             except Exception:
                 pass
+
+    def test_validate_fd(self):
+        assert rposix._validate_fd(0) == 1
+        fid = open(str(udir.join('validate_test.txt')), 'w')
+        fd = fid.fileno()
+        assert rposix._validate_fd(fd) == 1
+        fid.close()
+        assert rposix._validate_fd(fd) == 0
+
+

From noreply at buildbot.pypy.org  Thu Apr 26 20:26:22 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 26 Apr 2012 20:26:22 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: simplify test (amaury_)
Message-ID: <20120426182622.0508782009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54767:19263abaa526
Date: 2012-04-26 21:19 +0300
http://bitbucket.org/pypy/pypy/changeset/19263abaa526/

Log:	simplify test (amaury_)

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -265,10 +265,6 @@
 
         if option.runappdirect:
             py.test.skip("works with internals of _file impl on py.py")
-        if os.name=='nt':
-            mode = 'rb'
-        else:
-            mode = 'r'
         state = [0]
         def read(fd, n=None):
             if fd != 42:
@@ -282,7 +278,7 @@
             return ''
         os.read = read
         stdin = W_File(cls.space)
-        stdin.file_fdopen(42, mode, 1)
+        stdin.file_fdopen(42, 'rb', 1)
         stdin.name = '<stdin>'
         cls.w_stream = stdin
 

From noreply at buildbot.pypy.org  Thu Apr 26 20:26:23 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 26 Apr 2012 20:26:23 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge from default
Message-ID: <20120426182623.5109182009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54768:6740d398e7e2
Date: 2012-04-26 21:24 +0300
http://bitbucket.org/pypy/pypy/changeset/6740d398e7e2/

Log:	merge from default

diff --git a/lib-python/modified-2.7/test/test_peepholer.py b/lib-python/modified-2.7/test/test_peepholer.py
--- a/lib-python/modified-2.7/test/test_peepholer.py
+++ b/lib-python/modified-2.7/test/test_peepholer.py
@@ -145,12 +145,15 @@
 
     def test_binary_subscr_on_unicode(self):
         # valid code get optimized
-        asm = dis_single('u"foo"[0]')
-        self.assertIn("(u'f')", asm)
-        self.assertNotIn('BINARY_SUBSCR', asm)
-        asm = dis_single('u"\u0061\uffff"[1]')
-        self.assertIn("(u'\\uffff')", asm)
-        self.assertNotIn('BINARY_SUBSCR', asm)
+        # XXX for now we always disable this optimization
+        # XXX see CPython's issue5057
+        if 0:
+            asm = dis_single('u"foo"[0]')
+            self.assertIn("(u'f')", asm)
+            self.assertNotIn('BINARY_SUBSCR', asm)
+            asm = dis_single('u"\u0061\uffff"[1]')
+            self.assertIn("(u'\\uffff')", asm)
+            self.assertNotIn('BINARY_SUBSCR', asm)
 
         # invalid code doesn't get optimized
         # out of range
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
--- a/pypy/doc/cppyy.rst
+++ b/pypy/doc/cppyy.rst
@@ -80,7 +80,7 @@
         void SetMyInt(int i) { m_myint = i; }
 
     public:
-       int m_myint;
+        int m_myint;
     };
 
 Then, generate the bindings using ``genreflex`` (part of ROOT), and compile the
@@ -174,10 +174,10 @@
 
     $ cat MyAdvanced.xml
     <lcgdict>
-       <class pattern="Base?" />
-       <class name="Derived" />
-       <class name="std::string" />
-       <function name="BaseFactory" />
+        <class pattern="Base?" />
+        <class name="Derived" />
+        <class name="std::string" />
+        <function name="BaseFactory" />
     </lcgdict>
 
 .. _`selection file`: http://root.cern.ch/drupal/content/generating-reflex-dictionaries
@@ -353,6 +353,9 @@
   using classes that themselves are templates (etc.) in the arguments.
   All classes must already exist in the loaded reflection info.
 
+* **typedefs**: Are simple python references to the actual classes to which
+  they refer.
+
 * **unary operators**: Are supported if a python equivalent exists, and if the
   operator is defined in the C++ class.
 
@@ -370,6 +373,107 @@
 Only that one specific method can not be used.
 
 
+Templates
+=========
+
+A bit of special care needs to be taken for the use of templates.
+For a templated class to be completely available, it must be guaranteed that
+said class is fully instantiated, and hence all executable C++ code is
+generated and compiled in.
+The easiest way to fulfill that guarantee is by explicit instantiation in the
+header file that is handed to ``genreflex``.
+The following example should make that clear::
+
+    $ cat MyTemplate.h
+    #include <vector>
+
+    class MyClass {
+    public:
+        MyClass(int i = -99) : m_i(i) {}
+        MyClass(const MyClass& s) : m_i(s.m_i) {}
+        MyClass& operator=(const MyClass& s) { m_i = s.m_i; return *this; }
+        ~MyClass() {}
+        int m_i;
+    };
+
+    template class std::vector<MyClass>;
+
+If you know for certain that all symbols will be linked in from other sources,
+you can also declare the explicit template instantiation ``extern``.
+
+Unfortunately, this is not enough for gcc.
+The iterators, if they are going to be used, need to be instantiated as well,
+as do the comparison operators on those iterators, as these live in an
+internal namespace, rather than in the iterator classes.
+One way to handle this, is to deal with this once in a macro, then reuse that
+macro for all ``vector`` classes.
+Thus, the header above needs this, instead of just the explicit instantiation
+of the ``vector<MyClass>``::
+
+    #define STLTYPES_EXPLICIT_INSTANTIATION_DECL(STLTYPE, TTYPE)                      \
+    template class std::STLTYPE< TTYPE >;                                             \
+    template class __gnu_cxx::__normal_iterator<TTYPE*, std::STLTYPE< TTYPE > >;      \
+    template class __gnu_cxx::__normal_iterator<const TTYPE*, std::STLTYPE< TTYPE > >;\
+    namespace __gnu_cxx {                                                             \
+    template bool operator==(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    template bool operator!=(const std::STLTYPE< TTYPE >::iterator&,                  \
+                             const std::STLTYPE< TTYPE >::iterator&);                 \
+    }
+
+    STLTYPES_EXPLICIT_INSTANTIATION_DECL(vector, MyClass)
+
+Then, still for gcc, the selection file needs to contain the full hierarchy as
+well as the global overloads for comparisons for the iterators::
+
+    $ cat MyTemplate.xml
+    <lcgdict>
+        <class pattern="std::vector<*>" />
+        <class pattern="__gnu_cxx::__normal_iterator<*>" />
+        <class pattern="__gnu_cxx::new_allocator<*>" />
+        <class pattern="std::_Vector_base<*>" />
+        <class pattern="std::_Vector_base<*>::_Vector_impl" />
+        <class pattern="std::allocator<*>" />
+        <function name="__gnu_cxx::operator=="/>
+        <function name="__gnu_cxx::operator!="/>
+
+        <class name="MyClass" />
+    </lcgdict>
+
+Run the normal ``genreflex`` and compilation steps::
+
+    $ genreflex MyTemplate.h --selection=MyTemplate.xm
+    $ g++ -fPIC -rdynamic -O2 -shared -I$ROOTSYS/include MyTemplate_rflx.cpp -o libTemplateDict.so
+
+Note: this is a dirty corner that clearly could do with some automation,
+even if the macro already helps.
+Such automation is planned.
+In fact, in the cling world, the backend can perform the template
+instantations and generate the reflection info on the fly, and none of the
+above will any longer be necessary.
+
+Subsequent use should be as expected.
+Note the meta-class style of "instantiating" the template::
+
+    >>>> import cppyy
+    >>>> cppyy.load_reflection_info("libTemplateDict.so")
+    >>>> std = cppyy.gbl.std
+    >>>> MyClass = cppyy.gbl.MyClass
+    >>>> v = std.vector(MyClass)()
+    >>>> v += [MyClass(1), MyClass(2), MyClass(3)]
+    >>>> for m in v:
+    ....     print m.m_i,
+    ....
+    1 2 3
+    >>>>
+
+Other templates work similarly.
+The arguments to the template instantiation can either be a string with the
+full list of arguments, or the explicit classes.
+The latter makes for easier code writing if the classes passed to the
+instantiation are themselves templates.
+
+
 The fast lane
 =============
 
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -304,14 +304,19 @@
                     # produce compatible pycs.
                     if (self.space.isinstance_w(w_obj, self.space.w_unicode) and
                         self.space.isinstance_w(w_const, self.space.w_unicode)):
-                        unistr = self.space.unicode_w(w_const)
-                        if len(unistr) == 1:
-                            ch = ord(unistr[0])
-                        else:
-                            ch = 0
-                        if (ch > 0xFFFF or
-                            (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
-                            return subs
+                        #unistr = self.space.unicode_w(w_const)
+                        #if len(unistr) == 1:
+                        #    ch = ord(unistr[0])
+                        #else:
+                        #    ch = 0
+                        #if (ch > 0xFFFF or
+                        #    (MAXUNICODE == 0xFFFF and 0xD800 <= ch <= 0xDFFF)):
+                        # --XXX-- for now we always disable optimization of
+                        # u'...'[constant] because the tests above are not
+                        # enough to fix issue5057 (CPython has the same
+                        # problem as of April 24, 2012).
+                        # See test_const_fold_unicode_subscr
+                        return subs
 
                     return ast.Const(w_const, subs.lineno, subs.col_offset)
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -844,7 +844,8 @@
         return u"abc"[0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        if 0:   # xxx later?
+            assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
 
         # getitem outside of the BMP should not be optimized
         source = """def f():
@@ -854,12 +855,20 @@
         assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
                           ops.RETURN_VALUE: 1}
 
+        source = """def f():
+        return u"\U00012345abcdef"[3]
+        """
+        counts = self.count_instructions(source)
+        assert counts == {ops.LOAD_CONST: 2, ops.BINARY_SUBSCR: 1,
+                          ops.RETURN_VALUE: 1}
+
         monkeypatch.setattr(optimize, "MAXUNICODE", 0xFFFF)
         source = """def f():
         return u"\uE01F"[0]
         """
         counts = self.count_instructions(source)
-        assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
+        if 0:   # xxx later?
+            assert counts == {ops.LOAD_CONST: 1, ops.RETURN_VALUE: 1}
         monkeypatch.undo()
 
         # getslice is not yet optimized.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -112,8 +112,8 @@
 """.split()
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
-udir.join('pypy_decl.h').write("/* Will be filled later */")
-udir.join('pypy_macros.h').write("/* Will be filled later */")
+udir.join('pypy_decl.h').write("/* Will be filled later */\n")
+udir.join('pypy_macros.h').write("/* Will be filled later */\n")
 globals().update(rffi_platform.configure(CConfig_constants))
 
 def copy_header_files(dstdir):
diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -79,13 +79,13 @@
 else:
     separate_module_sources = []
     export_symbols = []
-eci = ExternalCompilationInfo(
+errno_eci = ExternalCompilationInfo(
     includes=['errno.h','stdio.h'],
     separate_module_sources = separate_module_sources,
     export_symbols = export_symbols,
 )
 
-_get_errno, _set_errno = CExternVariable(INT, 'errno', eci,
+_get_errno, _set_errno = CExternVariable(INT, 'errno', errno_eci,
                                          CConstantErrno, sandboxsafe=True,
                                          _nowrapper=True, c_type='int')
 # the default wrapper for set_errno is not suitable for use in critical places
@@ -100,7 +100,7 @@
 if os.name == 'nt':
     _validate_fd = rffi.llexternal(
         "_PyVerify_fd", [rffi.INT], rffi.INT,
-        compilation_info=eci,
+        compilation_info=errno_eci,
         )
     @jit.dont_look_inside
     def validate_fd(fd):
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1167,7 +1167,7 @@
         try:
             return self._lookup_adtmeth(field_name)
         except AttributeError:
-            raise AttributeError("%r instance has no field %r" % (self._T._name,
+            raise AttributeError("%r instance has no field %r" % (self._T,
                                                                   field_name))
 
     def __setattr__(self, field_name, val):
diff --git a/pypy/rpython/tool/rffi_platform.py b/pypy/rpython/tool/rffi_platform.py
--- a/pypy/rpython/tool/rffi_platform.py
+++ b/pypy/rpython/tool/rffi_platform.py
@@ -379,7 +379,7 @@
         self.name = name
 
     def prepare_code(self):
-        yield 'if ((%s) < 0) {' % (self.name,)
+        yield 'if ((%s) <= 0) {' % (self.name,)
         yield '    long long x = (long long)(%s);' % (self.name,)
         yield '    printf("value: %lld\\n", x);'
         yield '} else {'
@@ -401,7 +401,7 @@
     def prepare_code(self):
         yield '#ifdef %s' % self.macro
         yield 'dump("defined", 1);'
-        yield 'if ((%s) < 0) {' % (self.macro,)
+        yield 'if ((%s) <= 0) {' % (self.macro,)
         yield '    long long x = (long long)(%s);' % (self.macro,)
         yield '    printf("value: %lld\\n", x);'
         yield '} else {'
diff --git a/pypy/tool/compare_last_builds.py b/pypy/tool/compare_last_builds.py
new file mode 100644
--- /dev/null
+++ b/pypy/tool/compare_last_builds.py
@@ -0,0 +1,122 @@
+import os
+import urllib2
+import json
+import sys
+import md5
+
+wanted = sys.argv[1:]
+if not wanted:
+    wanted = ['default']
+base = "http://buildbot.pypy.org/json/builders/"
+
+cachedir = os.environ.get('PYPY_BUILDS_CACHE')
+if cachedir and not os.path.exists(cachedir):
+    os.makedirs(cachedir)
+
+
+
+def get_json(url, cache=cachedir):
+    return json.loads(get_data(url, cache))
+
+
+def get_data(url, cache=cachedir):
+    url = str(url)
+    if cache:
+        digest = md5.md5()
+        digest.update(url)
+        digest = digest.hexdigest()
+        cachepath = os.path.join(cachedir, digest)
+        if os.path.exists(cachepath):
+            with open(cachepath) as fp:
+                return fp.read()
+
+    print 'GET', url
+    fp = urllib2.urlopen(url)
+    try:
+        data = fp.read()
+        if cache:
+            with open(cachepath, 'wb') as cp:
+                cp.write(data)
+        return data
+    finally:
+        fp.close()
+
+def parse_log(log):
+    items = []
+    for v in log.splitlines(1):
+        if not v[0].isspace() and v[1].isspace():
+            items.append(v)
+    return sorted(items) #sort cause testrunner order is non-deterministic
+
+def gather_logdata(build):
+    logdata = get_data(str(build['log']) + '?as_text=1')
+    logdata = logdata.replace('</span><span class="stdout">', '')
+    logdata = logdata.replace('</span></pre>', '')
+    del build['log']
+    build['log'] = parse_log(logdata)
+
+
+def branch_mapping(l):
+    keep = 3 - len(wanted)
+    d = {}
+    for x in reversed(l):
+        gather_logdata(x)
+        if not x['log']:
+            continue
+        b = x['branch']
+        if b not in d:
+            d[b] = []
+        d[b].insert(0, x)
+        if len(d[b]) > keep:
+            d[b].pop()
+    return d
+
+def cleanup_build(d):
+    for a in 'times eta steps slave reason sourceStamp blame currentStep text'.split():
+        del d[a]
+
+    props = d.pop(u'logs')
+    for name, val in props:
+        if name == u'pytestLog':
+            d['log'] = val
+    props = d.pop(u'properties')
+    for name, val, _ in props:
+        if name == u'branch':
+            d['branch'] = val or 'default'
+    return d
+
+def collect_builds(d):
+    name = str(d['basedir'])
+    builds = d['cachedBuilds']
+    l = []
+    for build in builds:
+        d = get_json(base + '%s/builds/%s' % (name, build))
+        cleanup_build(d)
+        l.append(d)
+
+    l = [x for x in l if x['branch'] in wanted and 'log' in x]
+    d = branch_mapping(l)
+    return [x for lst in d.values() for x in lst]
+
+
+def only_linux32(d):
+    return d['own-linux-x86-32']
+
+
+own_builds = get_json(base, cache=False)['own-linux-x86-32']
+
+builds = collect_builds(own_builds)
+
+
+builds.sort(key=lambda x: (wanted.index(x['branch']), x['number']))
+logs = [x.pop('log') for x in builds]
+for b, s in zip(builds, logs):
+    b['resultset'] = len(s)
+import pprint
+pprint.pprint(builds)
+
+from difflib import Differ
+
+for x in Differ().compare(*logs):
+    if x[0]!=' ':
+        sys.stdout.write(x)
diff --git a/pypy/translator/c/src/cjkcodecs/cjkcodecs.h b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
--- a/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
+++ b/pypy/translator/c/src/cjkcodecs/cjkcodecs.h
@@ -210,15 +210,15 @@
 
 #define BEGIN_CODECS_LIST /* empty */
 #define _CODEC(name)                                                    \
-  static const MultibyteCodec _pypy_cjkcodec_##name;                    \
-  const MultibyteCodec *pypy_cjkcodec_##name(void) {                    \
+  static MultibyteCodec _pypy_cjkcodec_##name;                          \
+  MultibyteCodec *pypy_cjkcodec_##name(void) {                          \
     if (_pypy_cjkcodec_##name.codecinit != NULL) {                      \
       int r = _pypy_cjkcodec_##name.codecinit(_pypy_cjkcodec_##name.config); \
       assert(r == 0);                                                   \
     }                                                                   \
     return &_pypy_cjkcodec_##name;                                      \
   }                                                                     \
-  static const MultibyteCodec _pypy_cjkcodec_##name
+  static MultibyteCodec _pypy_cjkcodec_##name
 #define _STATEFUL_METHODS(enc)          \
     enc##_encode,                       \
     enc##_encode_init,                  \
diff --git a/pypy/translator/c/src/cjkcodecs/multibytecodec.h b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
--- a/pypy/translator/c/src/cjkcodecs/multibytecodec.h
+++ b/pypy/translator/c/src/cjkcodecs/multibytecodec.h
@@ -131,7 +131,7 @@
 /* list of codecs defined in the .c files */
 
 #define DEFINE_CODEC(name)                              \
-    const MultibyteCodec *pypy_cjkcodec_##name(void);
+    MultibyteCodec *pypy_cjkcodec_##name(void);
 
 // _codecs_cn
 DEFINE_CODEC(gb2312)

From noreply at buildbot.pypy.org  Thu Apr 26 23:07:56 2012
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 26 Apr 2012 23:07:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix jitviewer for simple targets which
 don't set sys_executable.
Message-ID: <20120426210756.C39CC82009@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r54769:f6fbfecb93fd
Date: 2012-04-26 23:07 +0200
http://bitbucket.org/pypy/pypy/changeset/f6fbfecb93fd/

Log:	Fix jitviewer for simple targets which don't set sys_executable.

diff --git a/pypy/jit/backend/llsupport/asmmemmgr.py b/pypy/jit/backend/llsupport/asmmemmgr.py
--- a/pypy/jit/backend/llsupport/asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/asmmemmgr.py
@@ -277,6 +277,8 @@
             from pypy.jit.backend.hlinfo import highleveljitinfo
             if highleveljitinfo.sys_executable:
                 debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable)
+            else:
+                debug_print('SYS_EXECUTABLE', '??')
             #
             HEX = '0123456789ABCDEF'
             dump = []

From noreply at buildbot.pypy.org  Fri Apr 27 00:23:05 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 27 Apr 2012 00:23:05 +0200 (CEST)
Subject: [pypy-commit] buildbot win32-cleanup: allow shell 9 step to run on
	windows
Message-ID: <20120426222305.7116382009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup
Changeset: r638:789283b42c44
Date: 2012-04-27 01:18 +0300
http://bitbucket.org/pypy/buildbot/changeset/789283b42c44/

Log:	allow shell 9 step to run on windows

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -314,9 +314,13 @@
                 logfiles={'pytestLog': 'pypyjit.log'}))
             #
             # "new" test_pypy_c
+            if platform == 'windows':
+                cmd = r'pypy\translator\goal\pypy-c'
+            else:
+                cmd = 'pypy/translator/goal/pypy-c'
             self.addStep(PytestCmd(
                 description="pypyjit tests",
-                command=["pypy/translator/goal/pypy-c", "pypy/test_all.py",
+                command=[cmd, "pypy/test_all.py",
                          "--resultlog=pypyjit_new.log",
                          "pypy/module/pypyjit/test_pypy_c"],
                 logfiles={'pytestLog': 'pypyjit_new.log'}))

From noreply at buildbot.pypy.org  Fri Apr 27 00:23:06 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 27 Apr 2012 00:23:06 +0200 (CEST)
Subject: [pypy-commit] buildbot win32-cleanup: remove non-functioning
	buildbots
Message-ID: <20120426222306.8D70682009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup
Changeset: r639:d6760789ad4d
Date: 2012-04-27 01:22 +0300
http://bitbucket.org/pypy/buildbot/changeset/d6760789ad4d/

Log:	remove non-functioning buildbots

diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -184,7 +184,7 @@
     ##         JITLINUX32,                # on tannit32, uses 1 core
     ##         JITLINUX64,                # on tannit64, uses 1 core
     ##         OJITLINUX32,               # on tannit32, uses 1 core
-    ##         JITWIN32,                  # on bigboard
+    ##         JITWIN32,                  # on aurora
     ##         STACKLESSAPPLVLFREEBSD64,  # on headless
     ##         JITMACOSX64,               # on mvt's machine
     ##         ], hour=4, minute=0),
@@ -225,7 +225,7 @@
             APPLVLLINUX32,             # on tannit32, uses 1 core
             APPLVLLINUX64,             # on tannit64, uses 1 core
             #
-            JITWIN32,                  # on bigboard
+            JITWIN32,                  # on aurora
             #JITFREEBSD64,              # on headless
             JITMACOSX64,               # on mvt's machine
             ], branch=None, hour=3, minute=0),
@@ -348,7 +348,7 @@
                    'category' : 'mac64',
                    },
                   {"name": WIN32,
-                   "slavenames": ["aurora", "SalsaSalsa", "snakepit32", "bigboard"],
+                   "slavenames": ["aurora", "SalsaSalsa"],
                    "builddir": WIN32,
                    "factory": pypyOwnTestFactoryWin,
                    "category": 'win32'
@@ -360,13 +360,13 @@
                    "category": 'win32'
                   },
                   {"name": APPLVLWIN32,
-                   "slavenames": ["aurora", "SalsaSalsa", "snakepit32", "bigboard"],
+                   "slavenames": ["aurora", "SalsaSalsa"],
                    "builddir": APPLVLWIN32,
                    "factory": pypyTranslatedAppLevelTestFactoryWin,
                    "category": "win32"
                   },
                   {"name" : JITWIN32,
-                   "slavenames": ["aurora", "SalsaSalsa", "snakepit32", "bigboard"],
+                   "slavenames": ["aurora", "SalsaSalsa"],
                    'builddir' : JITWIN32,
                    'factory' : pypyJITTranslatedTestFactoryWin,
                    'category' : 'win32',

From noreply at buildbot.pypy.org  Fri Apr 27 00:34:39 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 27 Apr 2012 00:34:39 +0200 (CEST)
Subject: [pypy-commit] buildbot win32-cleanup: whoops
Message-ID: <20120426223439.40B4182009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup
Changeset: r640:6a30885d7740
Date: 2012-04-27 01:34 +0300
http://bitbucket.org/pypy/buildbot/changeset/6a30885d7740/

Log:	whoops

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -314,7 +314,7 @@
                 logfiles={'pytestLog': 'pypyjit.log'}))
             #
             # "new" test_pypy_c
-            if platform == 'windows':
+            if platform == 'win32':
                 cmd = r'pypy\translator\goal\pypy-c'
             else:
                 cmd = 'pypy/translator/goal/pypy-c'

From noreply at buildbot.pypy.org  Fri Apr 27 08:38:19 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 27 Apr 2012 08:38:19 +0200 (CEST)
Subject: [pypy-commit] buildbot win32-cleanup: close branch befor merge
Message-ID: <20120427063819.0F6D382009@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: win32-cleanup
Changeset: r641:34a513c1e44a
Date: 2012-04-27 08:35 +0200
http://bitbucket.org/pypy/buildbot/changeset/34a513c1e44a/

Log:	close branch befor merge


From noreply at buildbot.pypy.org  Fri Apr 27 08:38:20 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 27 Apr 2012 08:38:20 +0200 (CEST)
Subject: [pypy-commit] buildbot default: merge from win32-cleanup
Message-ID: <20120427063820.3D0AF82009@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r642:64eeefddd046
Date: 2012-04-27 08:36 +0200
http://bitbucket.org/pypy/buildbot/changeset/64eeefddd046/

Log:	merge from win32-cleanup

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -314,9 +314,13 @@
                 logfiles={'pytestLog': 'pypyjit.log'}))
             #
             # "new" test_pypy_c
+            if platform == 'win32':
+                cmd = r'pypy\translator\goal\pypy-c'
+            else:
+                cmd = 'pypy/translator/goal/pypy-c'
             self.addStep(PytestCmd(
                 description="pypyjit tests",
-                command=["pypy/translator/goal/pypy-c", "pypy/test_all.py",
+                command=[cmd, "pypy/test_all.py",
                          "--resultlog=pypyjit_new.log",
                          "pypy/module/pypyjit/test_pypy_c"],
                 logfiles={'pytestLog': 'pypyjit_new.log'}))
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -184,7 +184,7 @@
     ##         JITLINUX32,                # on tannit32, uses 1 core
     ##         JITLINUX64,                # on tannit64, uses 1 core
     ##         OJITLINUX32,               # on tannit32, uses 1 core
-    ##         JITWIN32,                  # on bigboard
+    ##         JITWIN32,                  # on aurora
     ##         STACKLESSAPPLVLFREEBSD64,  # on headless
     ##         JITMACOSX64,               # on mvt's machine
     ##         ], hour=4, minute=0),
@@ -225,7 +225,7 @@
             APPLVLLINUX32,             # on tannit32, uses 1 core
             APPLVLLINUX64,             # on tannit64, uses 1 core
             #
-            JITWIN32,                  # on bigboard
+            JITWIN32,                  # on aurora
             #JITFREEBSD64,              # on headless
             JITMACOSX64,               # on mvt's machine
             ], branch=None, hour=3, minute=0),
@@ -348,7 +348,7 @@
                    'category' : 'mac64',
                    },
                   {"name": WIN32,
-                   "slavenames": ["aurora", "SalsaSalsa", "snakepit32", "bigboard"],
+                   "slavenames": ["aurora", "SalsaSalsa"],
                    "builddir": WIN32,
                    "factory": pypyOwnTestFactoryWin,
                    "category": 'win32'
@@ -360,13 +360,13 @@
                    "category": 'win32'
                   },
                   {"name": APPLVLWIN32,
-                   "slavenames": ["aurora", "SalsaSalsa", "snakepit32", "bigboard"],
+                   "slavenames": ["aurora", "SalsaSalsa"],
                    "builddir": APPLVLWIN32,
                    "factory": pypyTranslatedAppLevelTestFactoryWin,
                    "category": "win32"
                   },
                   {"name" : JITWIN32,
-                   "slavenames": ["aurora", "SalsaSalsa", "snakepit32", "bigboard"],
+                   "slavenames": ["aurora", "SalsaSalsa"],
                    'builddir' : JITWIN32,
                    'factory' : pypyJITTranslatedTestFactoryWin,
                    'category' : 'win32',

From noreply at buildbot.pypy.org  Fri Apr 27 08:56:08 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Apr 2012 08:56:08 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Precision w.r.t. the supported
	platforms.
Message-ID: <20120427065608.2BB9082009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4200:1fc3d5240cd4
Date: 2012-04-27 08:35 +0200
http://bitbucket.org/pypy/extradoc/changeset/1fc3d5240cd4/

Log:	Precision w.r.t. the supported platforms.

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
--- a/blog/draft/stm-apr2012.rst
+++ b/blog/draft/stm-apr2012.rst
@@ -11,9 +11,11 @@
 If you want to play with it, you can download `this binary`_ (you need to
 put it in a place with the paths ``lib-python`` and ``lib_pypy``, for
 example from a regular `nightly tarball`_ or from a full checkout).  It
-is for Linux 32-bit; I didn't do fixes for other platforms for now.
-It was compiled from the `stm-gc`_ branch on the 25th of April.
+is for Linux x86 32-bit.
+This version was compiled from the `stm-gc`_ branch on the 25th of April.
 It runs e.g. the `modified version of richards`_.
+This branch could also be translated for Linux x86-64,
+but I didn't do fixes for other platforms for now.
 
 .. _`this binary`: http://wyvern.cs.uni-duesseldorf.de/~arigo/pypy-stm-22fccf3c9b5e.tar.bz2
 .. _`nightly tarball`: http://buildbot.pypy.org/nightly/trunk/

From noreply at buildbot.pypy.org  Fri Apr 27 09:12:24 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 27 Apr 2012 09:12:24 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: explain
Message-ID: <20120427071224.2E10E82009@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r4201:e2615b4dd640
Date: 2012-04-27 09:12 +0200
http://bitbucket.org/pypy/extradoc/changeset/e2615b4dd640/

Log:	explain

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
--- a/blog/draft/stm-apr2012.rst
+++ b/blog/draft/stm-apr2012.rst
@@ -10,7 +10,8 @@
 
 If you want to play with it, you can download `this binary`_ (you need to
 put it in a place with the paths ``lib-python`` and ``lib_pypy``, for
-example from a regular `nightly tarball`_ or from a full checkout).  It
+example inside the main directory from a regular `nightly tarball`_
+or from a full checkout).  It
 is for Linux x86 32-bit.
 This version was compiled from the `stm-gc`_ branch on the 25th of April.
 It runs e.g. the `modified version of richards`_.

From noreply at buildbot.pypy.org  Fri Apr 27 09:46:42 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Apr 2012 09:46:42 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Expand here and there.
Message-ID: <20120427074642.B6C4782009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4202:7f5d0a8f1a66
Date: 2012-04-27 09:46 +0200
http://bitbucket.org/pypy/extradoc/changeset/7f5d0a8f1a66/

Log:	Expand here and there.

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
--- a/blog/draft/stm-apr2012.rst
+++ b/blog/draft/stm-apr2012.rst
@@ -1,47 +1,75 @@
 STM update (and thanks everybody)
 =================================
 
-A short update on the Software Transactional Memory (STM) side.  I have
-now reached the point where the basics seem to work, and the integration
-with the Garbage Collection subsystem is --- not done by far, but at
-least "not crashing in my simple tests and not leaking memory too
-quickly".  (It is leaking a bit of memory though, and it is never
-calling ``__del__`` so far.)
+A short update on the Software Transactional Memory (STM) side.  Let me
+remind you that the work is to add STM internally into PyPy, with the
+goal of letting the user's programs run on multiple cores after a minor
+adaptation.  (The goal is not to expose STM to the user's program.)  I
+will soon write some official documentation that explains in more
+details exactly what you get.  For now you can read the previous blog__
+posts__, and you can also find technical details in the `call for
+donation`_ itself; or directly look at how I adapted the examples linked
+to later in this post.
+
+.. _`call for donation`: http://pypy.org/tmdonate.html
+.. __: http://morepypy.blogspot.com/2012/03/call-for-donations-for-software.html
+.. __: http://morepypy.blogspot.com/2012/01/transactional-memory-ii.html
+
+I have now reached the point where the basics seem to work.  There is no
+integration with the JIT so far; moreover the integration with the
+Garbage Collection subsystem is not finished right now, but at least
+"not crashing in my simple tests and not leaking memory too quickly".
+(It means that it is never calling ``__del__`` so far, although it
+releases memory; and when entering transactional mode or when going to
+the next transaction, all live objects become immortal.  This should
+still let most not-too-long-running programs work.)
 
 If you want to play with it, you can download `this binary`_ (you need to
 put it in a place with the paths ``lib-python`` and ``lib_pypy``, for
 example inside the main directory from a regular `nightly tarball`_
-or from a full checkout).  It
-is for Linux x86 32-bit.
-This version was compiled from the `stm-gc`_ branch on the 25th of April.
-It runs e.g. the `modified version of richards`_.
-This branch could also be translated for Linux x86-64,
-but I didn't do fixes for other platforms for now.
+or from a full checkout).
+This version was compiled for Linux x86 32-bit from the `stm-gc`_ branch
+on the 25th of April.  It runs e.g. the modified version of richards_.
+This branch could also be translated for Linux x86-64, but not for
+other OSes nor other CPUs for now.
 
 .. _`this binary`: http://wyvern.cs.uni-duesseldorf.de/~arigo/pypy-stm-22fccf3c9b5e.tar.bz2
 .. _`nightly tarball`: http://buildbot.pypy.org/nightly/trunk/
 .. _`stm-gc`: https://bitbucket.org/pypy/pypy/src/stm-gc
-.. _`modified version of richards`: https://bitbucket.org/pypy/pypy/raw/stm-gc/pypy/translator/stm/test/richards.py
+.. _richards: https://bitbucket.org/pypy/pypy/raw/stm-gc/pypy/translator/stm/test/richards.py
 
-It exposes the same interface as the pure Python transaction_ module
-(except of course that it's not using the naive implementation linked
-above, but its own).  A difference is that it
-doesn't support epoll right now, so it cannot be used to play with `a
-branch of Twisted`_; but that's coming soon.  For now you can use it to
-get multi-core usage on purely computational programs, like PyPy's own
-``translate.py`` --- for that I did a tweak `in rpython/rtyper.py`_ (lines
-273-281 are all that I needed to add), but there are a few more places
-in the whole ``translate.py`` that could be similarly enhanced.
+The resulting ``pypy-stm`` exposes the same interface as the pure Python
+transaction_ module, which is an emulator (running on CPython or any
+version of PyPy) which can be used to play around and prepare your
+programs.  See the comments in there.  A difference is that the real
+``pypy-stm`` doesn't support epoll right now, so it cannot be used yet
+to play with `a branch of Twisted`_ that was already adapted (thanks
+Jean-Paul Calderone); but that's coming soon.  For now you can use it to
+get multi-core usage on purely computational programs.
+
+I did for example adapt PyPy's own ``translate.py``: see the tweak `in
+rpython/rtyper.py`_.  Lines 273-281 are all that I needed to add, and
+they are mostly a "simplification and parallelization" of the lines
+above.  There are a few more places in the whole ``translate.py`` that
+could be similarly modified, but overall it is just that: a few places.
+I did not measure performance, but I checked that it is capable of using
+multiple cores in the RTyping step of translation, with --- as expected
+--- some still-reasonable number of conflicts, particularly at the
+beginning when shared data structures are still being built.
 
 .. _transaction: https://bitbucket.org/pypy/pypy/raw/stm-gc/lib_pypy/transaction.py
 .. _`a branch of Twisted`: svn://svn.twistedmatrix.com/svn/Twisted/branches/stm-5526
 .. _`in rpython/rtyper.py`: https://bitbucket.org/pypy/pypy/src/stm-gc/pypy/rpython/rtyper.py#cl-249
 
-The performance is not great, even taking into account that it has no
-JIT so far, but it seems to scale; at least, it does scale on my
-2-real-cores, 4-hyperthreaded-cores laptop, roughly as expected.
+On a few smaller, more regular examples like richards_, I did measure
+the performance.  It is not great, even taking into account that it has
+no JIT so far; but it is well within the 2-to-5-times slower goal, and
+it does seem to scale.  At least, it scales roughly as expected on my
+2-real-cores, 4-hyperthreaded-cores laptop (which means that using two
+cores is not quite twice as fast as using only one, and using all four
+cores is again not twice faster but only 20-30%).
 
-And...
+And finally...
 
 ...a big thank you to everyone who contributed some money to support
 this!  As you see on the PyPy_ site, we got more than 6700$ so far in

From noreply at buildbot.pypy.org  Fri Apr 27 09:56:51 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Apr 2012 09:56:51 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Update docs
Message-ID: <20120427075651.29E5E82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54770:1c1ee4c2d200
Date: 2012-04-27 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/1c1ee4c2d200/

Log:	Update docs

diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py
--- a/lib_pypy/transaction.py
+++ b/lib_pypy/transaction.py
@@ -113,10 +113,16 @@
     is not shared between multiple concurrently-running transactions.
     It can be accessed without conflicts.
 
-    It can be used for purely transaction-local data.
+    It can be used for purely transaction-local data that needs to be
+    stored in a single global object.  As long as the data is not needed
+    after the transaction, e.g. because it is removed before the
+    transaction ends, using a "local" instance avoids conflicts which
+    would otherwise systematically trigger.
 
-    It can also be used for long-living caches that store values that
-    are (1) not too costly to compute and (2) not too memory-hungry,
-    because they will end up being computed and stored once per actual
-    thread.
+    Values that remain in a "local" instance after the end of a
+    transaction are visible in the next transaction that happens to be
+    executed on the same thread.  This can be used for long-living
+    caches that store values that are (1) not too costly to compute,
+    because they will end up being computed once per thread; and (2) not
+    too memory-hungry, because of the replicated storage.
     """

From noreply at buildbot.pypy.org  Fri Apr 27 10:28:06 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Apr 2012 10:28:06 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Comment.
Message-ID: <20120427082806.9F0C182009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54771:84017bfb4a89
Date: 2012-04-27 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/84017bfb4a89/

Log:	Comment.

diff --git a/pypy/translator/stm/src_stm/lists.c b/pypy/translator/stm/src_stm/lists.c
--- a/pypy/translator/stm/src_stm/lists.c
+++ b/pypy/translator/stm/src_stm/lists.c
@@ -232,7 +232,12 @@
 {
   /* XXX it would be a good idea to try to compactify the list,
      by removing duplicate entries.  We are only interested in
-     the set of entries present, and the order doesn't matter. */
+     the set of entries present, and the order doesn't matter.
+     See for example if, in practice, there are many consecutive
+     duplicate entries; if so, add a check here.  Otherwise, or
+     in addition, we need a general de-duplicator logic called
+     at every local_collection().
+  */
   if (oreclist->size == oreclist->alloc)
     _oreclist_grow(oreclist);
   oreclist->items[oreclist->size++] = newitem;

From noreply at buildbot.pypy.org  Fri Apr 27 11:19:06 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Apr 2012 11:19:06 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Clarify and reword.
Message-ID: <20120427091906.6EA4882009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4203:076d5ffc08ba
Date: 2012-04-27 11:18 +0200
http://bitbucket.org/pypy/extradoc/changeset/076d5ffc08ba/

Log:	Clarify and reword.

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
--- a/blog/draft/stm-apr2012.rst
+++ b/blog/draft/stm-apr2012.rst
@@ -63,11 +63,13 @@
 
 On a few smaller, more regular examples like richards_, I did measure
 the performance.  It is not great, even taking into account that it has
-no JIT so far; but it is well within the 2-to-5-times slower goal, and
-it does seem to scale.  At least, it scales roughly as expected on my
-2-real-cores, 4-hyperthreaded-cores laptop (which means that using two
-cores is not quite twice as fast as using only one, and using all four
-cores is again not twice faster but only 20-30%).
+no JIT so far.  Running pypy-stm with one thread is roughly 5 times
+slower than running a regular PyPy with no JIT (it used to be better in
+previous versions, but they didn't have any GC; nevertheless, I need to
+investigate).  However, it does seem to scale.  At least, it scales
+roughly as expected on my 2-real-cores, 4-hyperthreaded-cores laptop
+(i.e. for N between 1 and 4, the N-threaded pypy-stm performs similarly
+to N independent pypy-stm's running one thread each).
 
 And finally...
 

From noreply at buildbot.pypy.org  Fri Apr 27 11:50:15 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 27 Apr 2012 11:50:15 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: revert some useless changes
Message-ID: <20120427095015.582BB82009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54772:7ec68c9430e2
Date: 2012-04-27 12:23 +0300
http://bitbucket.org/pypy/pypy/changeset/7ec68c9430e2/

Log:	revert some useless changes

diff --git a/pypy/jit/backend/llsupport/asmmemmgr.py b/pypy/jit/backend/llsupport/asmmemmgr.py
--- a/pypy/jit/backend/llsupport/asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/asmmemmgr.py
@@ -277,6 +277,8 @@
             from pypy.jit.backend.hlinfo import highleveljitinfo
             if highleveljitinfo.sys_executable:
                 debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable)
+            else:
+                debug_print('SYS_EXECUTABLE', '??')
             #
             HEX = '0123456789ABCDEF'
             dump = []
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -10,7 +10,6 @@
 from pypy.translator.platform import platform
 
 import sys
-import os
 import weakref
 import py
 
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -24,7 +24,7 @@
     from pypy.module.thread import ll_thread as thread
 
     eci = ExternalCompilationInfo(
-        includes = ["windows.h"],
+        includes = ['windows.h'],
         post_include_bits = ["BOOL pypy_timemodule_setCtrlHandler(HANDLE event);"],
         separate_module_sources=['''
             static HANDLE interrupt_event;
diff --git a/pypy/module/select/interp_select.py b/pypy/module/select/interp_select.py
--- a/pypy/module/select/interp_select.py
+++ b/pypy/module/select/interp_select.py
@@ -53,6 +53,7 @@
             except (OverflowError, ValueError):
                 raise OperationError(space.w_ValueError,
                                      space.wrap("math range error"))
+
         try:
             retval = rpoll.poll(self.fddict, timeout)
         except rpoll.PollError, e:
diff --git a/pypy/translator/c/src/ll_math.c b/pypy/translator/c/src/ll_math.c
--- a/pypy/translator/c/src/ll_math.c
+++ b/pypy/translator/c/src/ll_math.c
@@ -20,7 +20,7 @@
 #define PyPy_IS_INFINITY(X) ((X) &&                                   \
                              (Py_FORCE_DOUBLE(X)*0.5 == Py_FORCE_DOUBLE(X)))
 #endif
-#define PyPy_NAN fabs(HUGE_VAL * 0.)
+#define PyPy_NAN (HUGE_VAL * 0.)
 
 /* The following copyright notice applies to the original
    implementations of acosh, asinh and atanh. */

From noreply at buildbot.pypy.org  Fri Apr 27 11:50:16 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 27 Apr 2012 11:50:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20120427095016.8ED2982009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54773:ff5ad277dbe2
Date: 2012-04-27 12:38 +0300
http://bitbucket.org/pypy/pypy/changeset/ff5ad277dbe2/

Log:	fix test

diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -133,6 +133,8 @@
                 pass
 
     def test_validate_fd(self):
+        if os.name != 'nt':
+            py.test.skip('relevant for windows only')
         assert rposix._validate_fd(0) == 1
         fid = open(str(udir.join('validate_test.txt')), 'w')
         fd = fid.fileno()

From noreply at buildbot.pypy.org  Fri Apr 27 11:50:17 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 27 Apr 2012 11:50:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge some trivial changes from
	win32-cleanup2
Message-ID: <20120427095017.DFB3282009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54774:b702c733839a
Date: 2012-04-27 12:49 +0300
http://bitbucket.org/pypy/pypy/changeset/b702c733839a/

Log:	merge some trivial changes from win32-cleanup2

diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -157,13 +157,15 @@
         raises(IOError, _multiprocessing.Connection, -15)
 
     def test_byte_order(self):
+        import socket
+        if not 'fromfd' in dir(socket):
+            skip('No fromfd in socket')
         # The exact format of net strings (length in network byte
         # order) is important for interoperation with others
         # implementations.
         rhandle, whandle = self.make_pair()
         whandle.send_bytes("abc")
         whandle.send_bytes("defg")
-        import socket
         sock = socket.fromfd(rhandle.fileno(),
                              socket.AF_INET, socket.SOCK_STREAM)
         data1 = sock.recv(7)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -198,7 +198,10 @@
         import nt
         r = ExpandEnvironmentStrings(u"%windir%\\test")
         assert isinstance(r, unicode)
-        assert r == nt.environ["WINDIR"] + "\\test"
+        if 'WINDIR' in nt.environ.keys():
+            assert r == nt.environ["WINDIR"] + "\\test"
+        else:
+            assert r == nt.environ["windir"] + "\\test"
 
     def test_long_key(self):
         from _winreg import (
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -572,7 +572,7 @@
                 if i < length and format[i] == '#':
                     # not documented by python
                     i += 1
-                if i >= length or format[i] not in "aAbBcdfHIjmMpSUwWxXyYzZ%":
+                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
                     raise OperationError(space.w_ValueError,
                                          space.wrap("invalid format string"))
             i += 1
diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -134,7 +134,7 @@
 
     def test_validate_fd(self):
         if os.name != 'nt':
-            py.test.skip('relevant for windows only')
+            skip('relevant for windows only')
         assert rposix._validate_fd(0) == 1
         fid = open(str(udir.join('validate_test.txt')), 'w')
         fd = fid.fileno()

From noreply at buildbot.pypy.org  Fri Apr 27 14:02:49 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 27 Apr 2012 14:02:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: copy os.environ into subprocess
Message-ID: <20120427120249.7C4EA82009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54775:dc8fb1054c92
Date: 2012-04-27 15:02 +0300
http://bitbucket.org/pypy/pypy/changeset/dc8fb1054c92/

Log:	copy os.environ into subprocess

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -54,7 +54,8 @@
             cmdline += ['--jit', ','.join(jitcmdline)]
         cmdline.append(str(self.filepath))
         #
-        env={'PYPYLOG': self.log_string + ':' + str(logfile)}
+        env = os.environ.copy()
+        env['PYPYLOG'] = self.log_string + ':' + str(logfile)
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,

From noreply at buildbot.pypy.org  Fri Apr 27 14:07:45 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 27 Apr 2012 14:07:45 +0200 (CEST)
Subject: [pypy-commit] pypy win32-cleanup2: merge default into branch
Message-ID: <20120427120745.8C4FE82009@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-cleanup2
Changeset: r54776:8c915f9df26c
Date: 2012-04-27 15:07 +0300
http://bitbucket.org/pypy/pypy/changeset/8c915f9df26c/

Log:	merge default into branch

diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -54,7 +54,8 @@
             cmdline += ['--jit', ','.join(jitcmdline)]
         cmdline.append(str(self.filepath))
         #
-        env={'PYPYLOG': self.log_string + ':' + str(logfile)}
+        env = os.environ.copy()
+        env['PYPYLOG'] = self.log_string + ':' + str(logfile)
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -133,6 +133,8 @@
                 pass
 
     def test_validate_fd(self):
+        if os.name != 'nt':
+            skip('relevant for windows only')
         assert rposix._validate_fd(0) == 1
         fid = open(str(udir.join('validate_test.txt')), 'w')
         fd = fid.fileno()

From noreply at buildbot.pypy.org  Fri Apr 27 17:45:39 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Apr 2012 17:45:39 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Bah.
Message-ID: <20120427154539.C6CB78208A@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54778:c25ae0621f3d
Date: 2012-04-27 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/c25ae0621f3d/

Log:	Bah.

diff --git a/pypy/translator/stm/src_stm/rpyintf.c b/pypy/translator/stm/src_stm/rpyintf.c
--- a/pypy/translator/stm/src_stm/rpyintf.c
+++ b/pypy/translator/stm/src_stm/rpyintf.c
@@ -150,4 +150,5 @@
                   i, num_threads, ret, retval);
         }
     }
+  free(th);
 }

From noreply at buildbot.pypy.org  Fri Apr 27 17:45:38 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Apr 2012 17:45:38 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Minor tweak.
Message-ID: <20120427154538.8FECD82009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54777:3d31bcaf6477
Date: 2012-04-27 14:48 +0200
http://bitbucket.org/pypy/pypy/changeset/3d31bcaf6477/

Log:	Minor tweak.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -19,10 +19,10 @@
 
 class Transaction(object):
     _next_transaction = None
-    _scheduled = False         # debugging
-    retry_counter = 0
+    #_scheduled = False         # for debugging
 
     def run(self):
+        # subclasses can access 'self.retry_counter' here
         raise NotImplementedError
 
 

From noreply at buildbot.pypy.org  Fri Apr 27 18:24:26 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 27 Apr 2012 18:24:26 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Typoish
Message-ID: <20120427162426.4AA5082009@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4204:8c4ce58d78b6
Date: 2012-04-27 18:24 +0200
http://bitbucket.org/pypy/extradoc/changeset/8c4ce58d78b6/

Log:	Typoish

diff --git a/blog/draft/stm-apr2012.rst b/blog/draft/stm-apr2012.rst
--- a/blog/draft/stm-apr2012.rst
+++ b/blog/draft/stm-apr2012.rst
@@ -17,7 +17,7 @@
 
 I have now reached the point where the basics seem to work.  There is no
 integration with the JIT so far; moreover the integration with the
-Garbage Collection subsystem is not finished right now, but at least
+Garbage Collection subsystem is not finished right now, but at least it is
 "not crashing in my simple tests and not leaking memory too quickly".
 (It means that it is never calling ``__del__`` so far, although it
 releases memory; and when entering transactional mode or when going to

From notifications-noreply at bitbucket.org  Fri Apr 27 21:44:25 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Fri, 27 Apr 2012 19:44:25 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120427194425.6586.13947@bitbucket15.managed.contegix.com>

You have received a notification from Christopher Houdeshell.

Hi, I forked pypy. My fork is at https://bitbucket.org/choudeshell/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Sat Apr 28 00:34:15 2012
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Sat, 28 Apr 2012 00:34:15 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-issue1137: add tests for behavior of
 objects with either or both int and index slots.
Message-ID: <20120427223415.1D3C1820B1@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpypy-issue1137
Changeset: r54781:c1a1e296db16
Date: 2012-04-28 00:33 +0200
http://bitbucket.org/pypy/pypy/changeset/c1a1e296db16/

Log:	add tests for behavior of objects with either or both int and index
	slots.

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -329,6 +329,49 @@
         assert a[1, 3] == 8
         assert a.T[1, 2] == 11
 
+    def test_getitem_obj_index(self):
+        from _numpypy import arange
+
+        class CustomIndexObject(object):
+            def __init__(self, index):
+                self.index = index
+            def __index__(self):
+                return self.index
+
+        a = arange(10)
+
+        assert a[CustomIndexObject(1)] == 1
+
+    def test_getitem_obj_prefer_index_to_int(self):
+        from _numpypy import arange
+
+        class CustomIndexIntObject(object):
+            def __init__(self, index, value):
+                self.index = index
+                self.value = value
+            def __index__(self):
+                return self.index
+            def __int__(self):
+                return self.value
+
+        a = arange(10)
+
+
+        assert a[CustomIndexIntObject(0, 1)] == 0
+
+    def test_getitem_obj_int(self):
+        from _numpypy import arange
+
+        class CustomIntObject(object):
+            def __init__(self, value):
+                self.value = value
+            def __index__(self):
+                return self.value
+
+        a = arange(10)
+
+        assert a[CustomIntObject(1)] == 1
+
     def test_setitem(self):
         from _numpypy import array
         a = array(range(5))
@@ -348,6 +391,52 @@
         for i in xrange(5):
             assert a[i] == i
 
+    def test_setitem_obj_index(self):
+        from _numpypy import arange
+
+        class CustomIndexObject(object):
+            def __init__(self, index):
+                self.index = index
+            def __index__(self):
+                return self.index
+
+        a = arange(10)
+
+        a[CustomIndexObject(1)] = 100
+        assert a[1] == 100
+
+    def test_setitem_obj_prefer_index_to_int(self):
+        from _numpypy import arange
+
+        class CustomIndexIntObject(object):
+            def __init__(self, index, value):
+                self.index = index
+                self.value = value
+            def __index__(self):
+                return self.index
+            def __int__(self):
+                return self.value
+
+        a = arange(10)
+
+        a[CustomIndexIntObject(0, 1)] = 100
+        assert a[0] == 100
+
+    def test_setitem_obj_int(self):
+        from _numpypy import arange
+
+        class CustomIntObject(object):
+            def __init__(self, index):
+                self.index = index
+            def __index__(self):
+                return self.index
+
+        a = arange(10)
+
+        a[CustomIntObject(1)] = 100
+
+        assert a[1] == 100
+
     def test_setslice_array(self):
         from _numpypy import array
         a = array(range(5))

From noreply at buildbot.pypy.org  Sat Apr 28 11:27:47 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Apr 2012 11:27:47 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120428092747.182F5820C1@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54783:cfdeabad2c3e
Date: 2012-04-28 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/cfdeabad2c3e/

Log:	hg merge default

diff --git a/lib-python/modified-2.7/test/test_peepholer.py b/lib-python/modified-2.7/test/test_peepholer.py
--- a/lib-python/modified-2.7/test/test_peepholer.py
+++ b/lib-python/modified-2.7/test/test_peepholer.py
@@ -145,12 +145,15 @@
 
     def test_binary_subscr_on_unicode(self):
         # valid code get optimized
-        asm = dis_single('u"foo"[0]')
-        self.assertIn("(u'f')", asm)
-        self.assertNotIn('BINARY_SUBSCR', asm)
-        asm = dis_single('u"\u0061\uffff"[1]')
-        self.assertIn("(u'\\uffff')", asm)
-        self.assertNotIn('BINARY_SUBSCR', asm)
+        # XXX for now we always disable this optimization
+        # XXX see CPython's issue5057
+        if 0:
+            asm = dis_single('u"foo"[0]')
+            self.assertIn("(u'f')", asm)
+            self.assertNotIn('BINARY_SUBSCR', asm)
+            asm = dis_single('u"\u0061\uffff"[1]')
+            self.assertIn("(u'\\uffff')", asm)
+            self.assertNotIn('BINARY_SUBSCR', asm)
 
         # invalid code doesn't get optimized
         # out of range
diff --git a/pypy/jit/backend/llsupport/asmmemmgr.py b/pypy/jit/backend/llsupport/asmmemmgr.py
--- a/pypy/jit/backend/llsupport/asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/asmmemmgr.py
@@ -277,6 +277,8 @@
             from pypy.jit.backend.hlinfo import highleveljitinfo
             if highleveljitinfo.sys_executable:
                 debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable)
+            else:
+                debug_print('SYS_EXECUTABLE', '??')
             #
             HEX = '0123456789ABCDEF'
             dump = []
diff --git a/pypy/rlib/rposix.py b/pypy/rlib/rposix.py
--- a/pypy/rlib/rposix.py
+++ b/pypy/rlib/rposix.py
@@ -1,9 +1,11 @@
 import os
-from pypy.rpython.lltypesystem.rffi import CConstant, CExternVariable, INT
+from pypy.rpython.lltypesystem.rffi import (CConstant, CExternVariable,
+        INT, CCHARPP)
 from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.rarithmetic import intmask
 from pypy.rlib.objectmodel import specialize
+from pypy.rlib import jit
 
 class CConstantErrno(CConstant):
     # these accessors are used when calling get_errno() or set_errno()
@@ -18,9 +20,69 @@
     def __setitem__(self, index, value):
         assert index == 0
         ll2ctypes.TLS.errno = value
+if os.name == 'nt':
+    separate_module_sources =['''
+        /* Lifted completely from CPython 3.3 Modules/posix_module.c */
+        #include <malloc.h> /* for _msize */
+        typedef struct {
+            intptr_t osfhnd;
+            char osfile;
+        } my_ioinfo;
+        extern __declspec(dllimport) char * __pioinfo[];
+        #define IOINFO_L2E 5
+        #define IOINFO_ARRAY_ELTS   (1 << IOINFO_L2E)
+        #define IOINFO_ARRAYS 64
+        #define _NHANDLE_           (IOINFO_ARRAYS * IOINFO_ARRAY_ELTS)
+        #define FOPEN 0x01
+        #define _NO_CONSOLE_FILENO (intptr_t)-2
 
+        /* This function emulates what the windows CRT
+            does to validate file handles */
+        int
+        _PyVerify_fd(int fd)
+        {
+            const int i1 = fd >> IOINFO_L2E;
+            const int i2 = fd & ((1 << IOINFO_L2E) - 1);
+
+            static size_t sizeof_ioinfo = 0;
+
+            /* Determine the actual size of the ioinfo structure,
+             * as used by the CRT loaded in memory
+             */
+            if (sizeof_ioinfo == 0 && __pioinfo[0] != NULL) {
+                sizeof_ioinfo = _msize(__pioinfo[0]) / IOINFO_ARRAY_ELTS;
+            }
+            if (sizeof_ioinfo == 0) {
+                /* This should not happen... */
+                goto fail;
+            }
+
+            /* See that it isn't a special CLEAR fileno */
+                if (fd != _NO_CONSOLE_FILENO) {
+                /* Microsoft CRT would check that 0<=fd<_nhandle but we can't do that.  Instead
+                 * we check pointer validity and other info
+                 */
+                if (0 <= i1 && i1 < IOINFO_ARRAYS && __pioinfo[i1] != NULL) {
+                    /* finally, check that the file is open */
+                    my_ioinfo* info = (my_ioinfo*)(__pioinfo[i1] + i2 * sizeof_ioinfo);
+                    if (info->osfile & FOPEN) {
+                        return 1;
+                    }
+                }
+            }
+          fail:
+            errno = EBADF;
+            return 0;
+        }
+    ''',]
+    export_symbols = ['_PyVerify_fd']
+else:
+    separate_module_sources = []
+    export_symbols = []
 errno_eci = ExternalCompilationInfo(
-    includes=['errno.h']
+    includes=['errno.h','stdio.h'],
+    separate_module_sources = separate_module_sources,
+    export_symbols = export_symbols,
 )
 
 _get_errno, _set_errno = CExternVariable(INT, 'errno', errno_eci,
@@ -35,6 +97,21 @@
 def set_errno(errno):
     _set_errno(rffi.cast(INT, errno))
 
+if os.name == 'nt':
+    _validate_fd = rffi.llexternal(
+        "_PyVerify_fd", [rffi.INT], rffi.INT,
+        compilation_info=errno_eci,
+        )
+    @jit.dont_look_inside
+    def validate_fd(fd):
+        if not _validate_fd(fd):
+            raise OSError(get_errno(), 'Bad file descriptor')
+else:
+    def _validate_fd(fd):
+        return 1
+
+    def validate_fd(fd):
+        return 1
 
 def closerange(fd_low, fd_high):
     # this behaves like os.closerange() from Python 2.6.
diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -131,3 +131,13 @@
                 os.rmdir(self.ufilename)
             except Exception:
                 pass
+
+    def test_validate_fd(self):
+        assert rposix._validate_fd(0) == 1
+        fid = open(str(udir.join('validate_test.txt')), 'w')
+        fd = fid.fileno()
+        assert rposix._validate_fd(fd) == 1
+        fid.close()
+        assert rposix._validate_fd(fd) == 0
+
+
diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py
--- a/pypy/rpython/annlowlevel.py
+++ b/pypy/rpython/annlowlevel.py
@@ -488,6 +488,8 @@
     else:
         TO = PTR
     if not hasattr(object, '_carry_around_for_tests'):
+        if object is None:
+            return lltype.nullptr(PTR.TO)
         assert not hasattr(object, '_TYPE')
         object._carry_around_for_tests = True
         object._TYPE = TO
@@ -557,6 +559,8 @@
     """NOT_RPYTHON: hack. Reverse the hacking done in cast_object_to_ptr()."""
     if isinstance(lltype.typeOf(ptr), lltype.Ptr):
         ptr = ptr._as_obj()
+        if ptr is None:
+            return None
     if not isinstance(ptr, Class):
         raise NotImplementedError("cast_base_ptr_to_instance: casting %r to %r"
                                   % (ptr, Class))
diff --git a/pypy/rpython/test/test_llann.py b/pypy/rpython/test/test_llann.py
--- a/pypy/rpython/test/test_llann.py
+++ b/pypy/rpython/test/test_llann.py
@@ -9,6 +9,7 @@
 from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
 from pypy.rpython.annlowlevel import PseudoHighLevelCallable
 from pypy.rpython.annlowlevel import llhelper, cast_instance_to_base_ptr
+from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.rpython.annlowlevel import base_ptr_lltype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.test.test_llinterp import interpret
@@ -502,7 +503,10 @@
             self.y = y
 
     def f(x, y):
-        a = A(x, y)
+        if x > 20:
+            a = None
+        else:
+            a = A(x, y)
         a1 = cast_instance_to_base_ptr(a)
         return a1
 
@@ -510,3 +514,30 @@
     assert typeOf(res) == base_ptr_lltype()
     assert fishllattr(res, 'x') == 5
     assert fishllattr(res, 'y') == 10
+
+    res = interpret(f, [25, 10])
+    assert res == nullptr(base_ptr_lltype().TO)
+
+
+def test_cast_base_ptr_to_instance():
+    class A:
+        def __init__(self, x, y):
+            self.x = x
+            self.y = y
+
+    def f(x, y):
+        if x > 20:
+            a = None
+        else:
+            a = A(x, y)
+        a1 = cast_instance_to_base_ptr(a)
+        b = cast_base_ptr_to_instance(A, a1)
+        return a is b
+
+    assert f(5, 10) is True
+    assert f(25, 10) is True
+
+    res = interpret(f, [5, 10])
+    assert res is True
+    res = interpret(f, [25, 10])
+    assert res is True

From noreply at buildbot.pypy.org  Sat Apr 28 11:27:45 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Apr 2012 11:27:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add tests,
 and write the missing corners (non-translated casts
Message-ID: <20120428092745.8F245820B1@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54782:3260900bdd46
Date: 2012-04-28 10:08 +0200
http://bitbucket.org/pypy/pypy/changeset/3260900bdd46/

Log:	Add tests, and write the missing corners (non-translated casts
	from/to None)

diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py
--- a/pypy/rpython/annlowlevel.py
+++ b/pypy/rpython/annlowlevel.py
@@ -488,6 +488,8 @@
     else:
         TO = PTR
     if not hasattr(object, '_carry_around_for_tests'):
+        if object is None:
+            return lltype.nullptr(PTR.TO)
         assert not hasattr(object, '_TYPE')
         object._carry_around_for_tests = True
         object._TYPE = TO
@@ -557,6 +559,8 @@
     """NOT_RPYTHON: hack. Reverse the hacking done in cast_object_to_ptr()."""
     if isinstance(lltype.typeOf(ptr), lltype.Ptr):
         ptr = ptr._as_obj()
+        if ptr is None:
+            return None
     if not isinstance(ptr, Class):
         raise NotImplementedError("cast_base_ptr_to_instance: casting %r to %r"
                                   % (ptr, Class))
diff --git a/pypy/rpython/test/test_llann.py b/pypy/rpython/test/test_llann.py
--- a/pypy/rpython/test/test_llann.py
+++ b/pypy/rpython/test/test_llann.py
@@ -9,6 +9,7 @@
 from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
 from pypy.rpython.annlowlevel import PseudoHighLevelCallable
 from pypy.rpython.annlowlevel import llhelper, cast_instance_to_base_ptr
+from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.rpython.annlowlevel import base_ptr_lltype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.test.test_llinterp import interpret
@@ -502,7 +503,10 @@
             self.y = y
 
     def f(x, y):
-        a = A(x, y)
+        if x > 20:
+            a = None
+        else:
+            a = A(x, y)
         a1 = cast_instance_to_base_ptr(a)
         return a1
 
@@ -510,3 +514,30 @@
     assert typeOf(res) == base_ptr_lltype()
     assert fishllattr(res, 'x') == 5
     assert fishllattr(res, 'y') == 10
+
+    res = interpret(f, [25, 10])
+    assert res == nullptr(base_ptr_lltype().TO)
+
+
+def test_cast_base_ptr_to_instance():
+    class A:
+        def __init__(self, x, y):
+            self.x = x
+            self.y = y
+
+    def f(x, y):
+        if x > 20:
+            a = None
+        else:
+            a = A(x, y)
+        a1 = cast_instance_to_base_ptr(a)
+        b = cast_base_ptr_to_instance(A, a1)
+        return a is b
+
+    assert f(5, 10) is True
+    assert f(25, 10) is True
+
+    res = interpret(f, [5, 10])
+    assert res is True
+    res = interpret(f, [25, 10])
+    assert res is True

From noreply at buildbot.pypy.org  Sat Apr 28 11:27:48 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Apr 2012 11:27:48 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Start a ThreadLocal object. See comments.
Message-ID: <20120428092748.5819D820B1@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54784:588d378678ed
Date: 2012-04-28 10:17 +0200
http://bitbucket.org/pypy/pypy/changeset/588d378678ed/

Log:	Start a ThreadLocal object. See comments.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -3,6 +3,7 @@
 from pypy.rpython.annlowlevel import llhelper, cast_instance_to_base_ptr
 from pypy.rpython.annlowlevel import base_ptr_lltype, cast_base_ptr_to_instance
 from pypy.rlib.objectmodel import keepalive_until_here, we_are_translated
+from pypy.rlib.objectmodel import specialize
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.nonconst import NonConstant
 from pypy.translator.stm.stmgcintf import StmOperations
@@ -219,3 +220,43 @@
 
 def _stm_thread_stopping():
     llop.stm_thread_stopping(lltype.Void)
+
+
+class ThreadLocal(object):
+    """
+    A thread-local container.  Use only for one or a few static places,
+    e.g. the ExecutionContext in the PyPy interpreter; and store any
+    number of stuff on the ExecutionContext instead.  The point of this
+    is to have proper GC support: if at the end of a transaction some
+    objects are only reachable via a ThreadLocal object, then these
+    objects don't need to be turned GLOBAL.  It avoids the overhead of
+    STM, notably the two copies that are needed for every transaction
+    that changes a GLOBAL object.
+    """
+    STMTHREADLOCAL = lltype.Struct('StmThreadLocal',
+                                   ('content', base_ptr_lltype()),
+                                   hints={'stm_thread_local': True})
+
+    def __init__(self, Class):
+        assert not we_are_translated(), (
+            "You can only have a small number of ThreadLocal() instances"
+            " built during translation.")
+        self.Class = Class
+        self.threadlocal = lltype.malloc(self.STMTHREADLOCAL, immortal=True)
+
+    def _freeze_(self):
+        return True      # but the thread-local value can be read and written
+
+    @specialize.arg(0)
+    def getvalue(self):
+        """Read the thread-local value.
+        It can be either None (the default) or an instance of self.Class."""
+        ptr = self.threadlocal.content
+        return cast_base_ptr_to_instance(self.Class, ptr)
+
+    @specialize.arg(0)
+    def setvalue(self, value):
+        """Write the thread-local value."""
+        assert value is None or isinstance(value, self.Class)
+        ptr = cast_instance_to_base_ptr(value)
+        self.threadlocal.content = ptr
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -117,3 +117,20 @@
         def run(self):
             return [FooBar() for i in range(10)]
     py.test.raises(MyException, rstm.run_all_transactions, DoInOrder())
+
+def test_threadlocal():
+    # not testing the thread-local factor, but only the general interface
+    class Point:
+        def __init__(self, x, y):
+            self.x = x
+            self.y = y
+    p1 = Point(10, 2)
+    p2 = Point(-1, 0)
+    tl = rstm.ThreadLocal(Point)
+    assert tl.getvalue() is None
+    tl.setvalue(p1)
+    assert tl.getvalue() is p1
+    tl.setvalue(p2)
+    assert tl.getvalue() is p2
+    tl.setvalue(None)
+    assert tl.getvalue() is None

From noreply at buildbot.pypy.org  Sat Apr 28 11:27:49 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Apr 2012 11:27:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: In case of typo,
 the flow object space complained typically like that:
Message-ID: <20120428092749.937CD820B1@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54785:a91776704352
Date: 2012-04-28 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/a91776704352/

Log:	In case of typo, the flow object space complained typically like
	that:

	 found an operation that always raises AttributeError: generated
	by a constant operation: getattr

	That's very very uninformative. Display at least the arguments of
	the getattr operation.

diff --git a/pypy/objspace/flow/operation.py b/pypy/objspace/flow/operation.py
--- a/pypy/objspace/flow/operation.py
+++ b/pypy/objspace/flow/operation.py
@@ -350,8 +350,8 @@
                     result = op(*args)
                 except Exception, e:
                     etype = e.__class__
-                    msg = "generated by a constant operation:  %s" % (
-                        name)
+                    msg = "generated by a constant operation:\n\t%s%r" % (
+                        name, tuple(args))
                     raise OperationThatShouldNotBePropagatedError(
                         self.wrap(etype), self.wrap(msg))
                 else:

From noreply at buildbot.pypy.org  Sat Apr 28 11:27:50 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Apr 2012 11:27:50 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120428092750.CACCB820B1@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54786:e865723a53c3
Date: 2012-04-28 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/e865723a53c3/

Log:	hg merge default

diff --git a/pypy/objspace/flow/operation.py b/pypy/objspace/flow/operation.py
--- a/pypy/objspace/flow/operation.py
+++ b/pypy/objspace/flow/operation.py
@@ -350,8 +350,8 @@
                     result = op(*args)
                 except Exception, e:
                     etype = e.__class__
-                    msg = "generated by a constant operation:  %s" % (
-                        name)
+                    msg = "generated by a constant operation:\n\t%s%r" % (
+                        name, tuple(args))
                     raise OperationThatShouldNotBePropagatedError(
                         self.wrap(etype), self.wrap(msg))
                 else:

From noreply at buildbot.pypy.org  Sat Apr 28 11:27:52 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Apr 2012 11:27:52 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Comment,
 and kill outdated method (see start_transaction() in
Message-ID: <20120428092752.0FFCA820B1@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54787:7d05feb1fc7c
Date: 2012-04-28 10:58 +0200
http://bitbucket.org/pypy/pypy/changeset/7d05feb1fc7c/

Log:	Comment, and kill outdated method (see start_transaction() in the
	class StmShadowStackRootWalker).

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -146,6 +146,9 @@
 def _stm_run_transaction(transactionptr, retry_counter):
     #
     # Tell the GC we are starting a transaction
+    # (at this point, we have no stack root at all; the following
+    # call will clear any remaining garbage from the shadowstack,
+    # in case of an aborted transaction)
     llop.stm_start_transaction(lltype.Void)
     #
     # Now we can use the GC
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -198,9 +198,3 @@
         self.rootstackhook(collect_stack_root, arg,
                            stackgcdata.root_stack_base,
                            stackgcdata.root_stack_top)
-
-    def clear_current_stack_roots(self):
-        """Used when we start a transaction: there might be garbage left
-        behind by the previous aborted transaction."""
-        stackgcdata = self.stackgcdata
-        stackgcdata.root_stack_top = stackgcdata.root_stack_base

From noreply at buildbot.pypy.org  Sat Apr 28 11:27:53 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Apr 2012 11:27:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120428092753.990D7820B1@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54788:e47c9e33ca61
Date: 2012-04-28 11:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e47c9e33ca61/

Log:	merge heads

diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -157,13 +157,15 @@
         raises(IOError, _multiprocessing.Connection, -15)
 
     def test_byte_order(self):
+        import socket
+        if not 'fromfd' in dir(socket):
+            skip('No fromfd in socket')
         # The exact format of net strings (length in network byte
         # order) is important for interoperation with others
         # implementations.
         rhandle, whandle = self.make_pair()
         whandle.send_bytes("abc")
         whandle.send_bytes("defg")
-        import socket
         sock = socket.fromfd(rhandle.fileno(),
                              socket.AF_INET, socket.SOCK_STREAM)
         data1 = sock.recv(7)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -198,7 +198,10 @@
         import nt
         r = ExpandEnvironmentStrings(u"%windir%\\test")
         assert isinstance(r, unicode)
-        assert r == nt.environ["WINDIR"] + "\\test"
+        if 'WINDIR' in nt.environ.keys():
+            assert r == nt.environ["WINDIR"] + "\\test"
+        else:
+            assert r == nt.environ["windir"] + "\\test"
 
     def test_long_key(self):
         from _winreg import (
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -185,6 +185,33 @@
 
         assert dtype("float") is dtype(float)
 
+    def test_index_int8(self):
+        from _numpypy import array, int8
+
+        a = array(range(10), dtype=int8)
+        b = array([0] * 10, dtype=int8)
+        for idx in b: a[idx] += 1
+
+    def test_index_int16(self):
+        from _numpypy import array, int16
+
+        a = array(range(10), dtype=int16)
+        b = array([0] * 10, dtype=int16)
+        for idx in b: a[idx] += 1
+
+    def test_index_int32(self):
+        from _numpypy import array, int32
+
+        a = array(range(10), dtype=int32)
+        b = array([0] * 10, dtype=int32)
+        for idx in b: a[idx] += 1
+
+    def test_index_int64(self):
+        from _numpypy import array, int64
+
+        a = array(range(10), dtype=int64)
+        b = array([0] * 10, dtype=int64)
+        for idx in b: a[idx] += 1
 
 class AppTestTypes(BaseNumpyAppTest):    
     def test_abstract_types(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -54,7 +54,8 @@
             cmdline += ['--jit', ','.join(jitcmdline)]
         cmdline.append(str(self.filepath))
         #
-        env={'PYPYLOG': self.log_string + ':' + str(logfile)}
+        env = os.environ.copy()
+        env['PYPYLOG'] = self.log_string + ':' + str(logfile)
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -572,7 +572,7 @@
                 if i < length and format[i] == '#':
                     # not documented by python
                     i += 1
-                if i >= length or format[i] not in "aAbBcdfHIjmMpSUwWxXyYzZ%":
+                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
                     raise OperationError(space.w_ValueError,
                                          space.wrap("invalid format string"))
             i += 1
diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -133,6 +133,8 @@
                 pass
 
     def test_validate_fd(self):
+        if os.name != 'nt':
+            skip('relevant for windows only')
         assert rposix._validate_fd(0) == 1
         fid = open(str(udir.join('validate_test.txt')), 'w')
         fd = fid.fileno()

From noreply at buildbot.pypy.org  Sat Apr 28 15:01:30 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat, 28 Apr 2012 15:01:30 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: debugging issue1080
Message-ID: <20120428130130.5171F820B1@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: dynamic-specialized-tuple
Changeset: r54789:5529f254d0b3
Date: 2012-04-28 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/5529f254d0b3/

Log:	debugging issue1080

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5119,6 +5119,25 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_issue1080_basic(self):
+        ops = """
+        []
+        p69 = same_as(ConstPtr(myptr))
+        quasiimmut_field(p69, descr=quasiimmutdescr)
+        guard_not_invalidated() []        
+        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
+        guard_value(p71, -4247) []
+        jump()
+        """
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        guard_value(p72, -4247) []        
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+        
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7813,6 +7813,51 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_issue1080_infinitie_loop_virtual(self):
+        ops = """
+        [p10]
+        p52 = getfield_gc(p10, descr=nextdescr) # inst_storage
+        p54 = getarrayitem_gc(p52, 0, descr=arraydescr)
+        p69 = getfield_gc_pure(p54, descr=otherdescr) # inst_w_function
+
+        quasiimmut_field(p69, descr=quasiimmutdescr)
+        guard_not_invalidated() []        
+        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
+        guard_value(p71, -4247) []
+
+        p106 = new_with_vtable(ConstClass(node_vtable))
+        p108 = new_array(3, descr=arraydescr)
+        p110 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p110, ConstPtr(myptr2), descr=otherdescr) # inst_w_function
+        setarrayitem_gc(p108, 0, p110, descr=arraydescr)
+        setfield_gc(p106, p108, descr=nextdescr) # inst_storage
+        jump(p106)
+        """
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr2), descr=quasifielddescr)
+        guard_value(p72, -4247) []        
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+        
+    def test_issue1080_infinitie_loop_simple(self):
+        ops = """
+        [p69]
+        quasiimmut_field(p69, descr=quasiimmutdescr)
+        guard_not_invalidated() []        
+        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
+        guard_value(p71, -4247) []
+        jump(ConstPtr(myptr))
+        """
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        guard_value(p72, -4247) []        
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 

From noreply at buildbot.pypy.org  Sat Apr 28 15:26:33 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat, 28 Apr 2012 15:26:33 +0200 (CEST)
Subject: [pypy-commit] pypy dynamic-specialized-tuple: Backed out changeset
 5529f254d0b3, wrong branch
Message-ID: <20120428132633.10F6D820B1@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: dynamic-specialized-tuple
Changeset: r54790:0bbab927514e
Date: 2012-04-28 15:06 +0200
http://bitbucket.org/pypy/pypy/changeset/0bbab927514e/

Log:	Backed out changeset 5529f254d0b3, wrong branch

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5119,25 +5119,6 @@
         """
         self.optimize_loop(ops, expected)
 
-    def test_issue1080_basic(self):
-        ops = """
-        []
-        p69 = same_as(ConstPtr(myptr))
-        quasiimmut_field(p69, descr=quasiimmutdescr)
-        guard_not_invalidated() []        
-        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
-        guard_value(p71, -4247) []
-        jump()
-        """
-        expected = """
-        []
-        p72 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
-        guard_value(p72, -4247) []        
-        jump()
-        """
-        self.optimize_loop(ops, expected)
-        
-
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7813,51 +7813,6 @@
         """
         self.optimize_loop(ops, expected)
 
-    def test_issue1080_infinitie_loop_virtual(self):
-        ops = """
-        [p10]
-        p52 = getfield_gc(p10, descr=nextdescr) # inst_storage
-        p54 = getarrayitem_gc(p52, 0, descr=arraydescr)
-        p69 = getfield_gc_pure(p54, descr=otherdescr) # inst_w_function
-
-        quasiimmut_field(p69, descr=quasiimmutdescr)
-        guard_not_invalidated() []        
-        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
-        guard_value(p71, -4247) []
-
-        p106 = new_with_vtable(ConstClass(node_vtable))
-        p108 = new_array(3, descr=arraydescr)
-        p110 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p110, ConstPtr(myptr2), descr=otherdescr) # inst_w_function
-        setarrayitem_gc(p108, 0, p110, descr=arraydescr)
-        setfield_gc(p106, p108, descr=nextdescr) # inst_storage
-        jump(p106)
-        """
-        expected = """
-        []
-        p72 = getfield_gc(ConstPtr(myptr2), descr=quasifielddescr)
-        guard_value(p72, -4247) []        
-        jump()
-        """
-        self.optimize_loop(ops, expected)
-        
-    def test_issue1080_infinitie_loop_simple(self):
-        ops = """
-        [p69]
-        quasiimmut_field(p69, descr=quasiimmutdescr)
-        guard_not_invalidated() []        
-        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
-        guard_value(p71, -4247) []
-        jump(ConstPtr(myptr))
-        """
-        expected = """
-        []
-        p72 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
-        guard_value(p72, -4247) []        
-        jump()
-        """
-        self.optimize_loop(ops, expected)
-
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 

From noreply at buildbot.pypy.org  Sat Apr 28 15:26:34 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat, 28 Apr 2012 15:26:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: debugging issue1080
Message-ID: <20120428132634.5DEA4820B1@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54791:68e5f2fc92c1
Date: 2012-04-28 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/68e5f2fc92c1/

Log:	debugging issue1080

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5090,6 +5090,25 @@
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
 
+    def test_issue1080_basic(self):
+        ops = """
+        []
+        p69 = same_as(ConstPtr(myptr))
+        quasiimmut_field(p69, descr=quasiimmutdescr)
+        guard_not_invalidated() []
+        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
+        guard_value(p71, -4247) []
+        jump()
+        """
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        guard_value(p72, -4247) []
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
+
 
 ##class TestOOtype(BaseTestOptimizeBasic, OOtypeMixin):
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7813,6 +7813,51 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_issue1080_infinitie_loop_virtual(self):
+        ops = """
+        [p10]
+        p52 = getfield_gc(p10, descr=nextdescr) # inst_storage
+        p54 = getarrayitem_gc(p52, 0, descr=arraydescr)
+        p69 = getfield_gc_pure(p54, descr=otherdescr) # inst_w_function
+
+        quasiimmut_field(p69, descr=quasiimmutdescr)
+        guard_not_invalidated() []
+        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
+        guard_value(p71, -4247) []
+
+        p106 = new_with_vtable(ConstClass(node_vtable))
+        p108 = new_array(3, descr=arraydescr)
+        p110 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p110, ConstPtr(myptr2), descr=otherdescr) # inst_w_function
+        setarrayitem_gc(p108, 0, p110, descr=arraydescr)
+        setfield_gc(p106, p108, descr=nextdescr) # inst_storage
+        jump(p106)
+        """
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr2), descr=quasifielddescr)
+        guard_value(p72, -4247) []
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_issue1080_infinitie_loop_simple(self):
+        ops = """
+        [p69]
+        quasiimmut_field(p69, descr=quasiimmutdescr)
+        guard_not_invalidated() []
+        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
+        guard_value(p71, -4247) []
+        jump(ConstPtr(myptr))
+        """
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        guard_value(p72, -4247) []
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 

From noreply at buildbot.pypy.org  Sat Apr 28 15:34:21 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat, 28 Apr 2012 15:34:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: these should result in InvalidLoops
Message-ID: <20120428133421.8F305820B1@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54792:3206e05abe93
Date: 2012-04-28 15:33 +0200
http://bitbucket.org/pypy/pypy/changeset/3206e05abe93/

Log:	these should result in InvalidLoops

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5090,26 +5090,6 @@
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
 
-    def test_issue1080_basic(self):
-        ops = """
-        []
-        p69 = same_as(ConstPtr(myptr))
-        quasiimmut_field(p69, descr=quasiimmutdescr)
-        guard_not_invalidated() []
-        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
-        guard_value(p71, -4247) []
-        jump()
-        """
-        expected = """
-        []
-        p72 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
-        guard_value(p72, -4247) []
-        jump()
-        """
-        self.optimize_loop(ops, expected)
-
-
-
 ##class TestOOtype(BaseTestOptimizeBasic, OOtypeMixin):
 
 ##    def test_instanceof(self):
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7833,13 +7833,9 @@
         setfield_gc(p106, p108, descr=nextdescr) # inst_storage
         jump(p106)
         """
-        expected = """
-        []
-        p72 = getfield_gc(ConstPtr(myptr2), descr=quasifielddescr)
-        guard_value(p72, -4247) []
-        jump()
-        """
-        self.optimize_loop(ops, expected)
+        py.test.raises(InvalidLoop, self.optimize_loop,
+                       ops, expected)
+        
 
     def test_issue1080_infinitie_loop_simple(self):
         ops = """
@@ -7856,7 +7852,8 @@
         guard_value(p72, -4247) []
         jump()
         """
-        self.optimize_loop(ops, expected)
+        py.test.raises(InvalidLoop, self.optimize_loop,
+                       ops, expected)
 
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Sat Apr 28 16:16:39 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 28 Apr 2012 16:16:39 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: use some more nice relative
	imports
Message-ID: <20120428141639.95264820B1@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r166:4d72d90ca555
Date: 2012-03-24 16:10 +0100
http://bitbucket.org/pypy/pyrepl/changeset/4d72d90ca555/

Log:	use some more nice relative imports

diff --git a/pyrepl/unix_console.py b/pyrepl/unix_console.py
--- a/pyrepl/unix_console.py
+++ b/pyrepl/unix_console.py
@@ -22,10 +22,10 @@
 import termios, select, os, struct, errno
 import signal, re, time, sys
 from fcntl import ioctl
-from pyrepl import curses
-from pyrepl.fancy_termios import tcgetattr, tcsetattr
-from pyrepl.console import Console, Event
-from pyrepl import unix_eventqueue
+from . import curses
+from .fancy_termios import tcgetattr, tcsetattr
+from .console import Console, Event
+from .unix_eventqueue import EventQueue
 
 class InvalidTerminal(RuntimeError):
     pass
@@ -98,6 +98,7 @@
         else:
             self.output_fd = f_out.fileno()
         
+
         self.pollob = poll()
         self.pollob.register(self.input_fd, POLLIN)
         curses.setupterm(term, self.output_fd)
@@ -161,7 +162,7 @@
 
         self.__move = self.__move_short
 
-        self.event_queue = unix_eventqueue.EventQueue(self.input_fd)
+        self.event_queue = EventQueue(self.input_fd)
         self.partial_char = b''
         self.cursor_visible = 1
 

From noreply at buildbot.pypy.org  Sat Apr 28 16:16:40 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 28 Apr 2012 16:16:40 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: initial tracing for unic console
Message-ID: <20120428141640.C1E3B820B1@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r167:2e5ce6fd51fc
Date: 2012-03-24 16:26 +0100
http://bitbucket.org/pypy/pyrepl/changeset/2e5ce6fd51fc/

Log:	initial tracing for unic console

diff --git a/pyrepl/trace.py b/pyrepl/trace.py
new file mode 100644
--- /dev/null
+++ b/pyrepl/trace.py
@@ -0,0 +1,17 @@
+import os
+
+trace_filename = os.environ.get("PYREPL_TRACE")
+
+if trace_filename is not None:
+    trace_file = open(trace_filename, 'a')
+else:
+    trace_file = None
+
+def trace(line, *k, **kw):
+    if trace_file is None:
+        return
+    if k or kw:
+        line = line.format(*k, **kw)
+    trace_file.write(line+'\n')
+    trace_file.flush()
+
diff --git a/pyrepl/unix_console.py b/pyrepl/unix_console.py
--- a/pyrepl/unix_console.py
+++ b/pyrepl/unix_console.py
@@ -26,6 +26,7 @@
 from .fancy_termios import tcgetattr, tcsetattr
 from .console import Console, Event
 from .unix_eventqueue import EventQueue
+from .trace import trace
 
 class InvalidTerminal(RuntimeError):
     pass
@@ -410,6 +411,7 @@
         self.event_queue.insert(Event('resize', None))
 
     def push_char(self, char):
+        trace('push char {char!r}', char=char)
         self.partial_char += char
         try:
             c = self.partial_char.decode(self.encoding)

From noreply at buildbot.pypy.org  Sat Apr 28 16:55:04 2012
From: noreply at buildbot.pypy.org (dripton)
Date: Sat, 28 Apr 2012 16:55:04 +0200 (CEST)
Subject: [pypy-commit] pyrepl default: (hpaulj) Fix startup_hook in readline
 to work like CPython. Fixes issue950
Message-ID: <20120428145504.50DA2820B1@wyvern.cs.uni-duesseldorf.de>

Author: David Ripton <dripton at ripton.net>
Branch: 
Changeset: r168:d92286eb65ec
Date: 2012-01-28 19:17 -0500
http://bitbucket.org/pypy/pyrepl/changeset/d92286eb65ec/

Log:	(hpaulj) Fix startup_hook in readline to work like CPython. Fixes
	issue950

diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -37,16 +37,16 @@
     for i in range(256):
         c = unichr(i)
         if not uc_map.has_key(c):
-            uc_map[c] = u'\\%03o'%i 
+            uc_map[c] = u'\\%03o'%i
     return uc_map
 
 # disp_str proved to be a bottleneck for large inputs, so it's been
 # rewritten in C; it's not required though.
 try:
     raise ImportError # currently it's borked by the unicode support
-    
+
     from _pyrepl_utils import disp_str, init_unctrl_map
-    
+
     init_unctrl_map(_make_unctrl_map())
 
     del init_unctrl_map
@@ -118,7 +118,7 @@
      (r'\C-x\C-u', 'upcase-region'),
      (r'\C-y', 'yank'),
      (r'\C-z', 'suspend'),
-     
+
      (r'\M-b', 'backward-word'),
      (r'\M-c', 'capitalize-word'),
      (r'\M-d', 'kill-word'),
@@ -303,7 +303,7 @@
 
     def process_prompt(self, prompt):
         """ Process the prompt.
-        
+
         This means calculate the length of the prompt. The character \x01
         and \x02 are used to bracket ANSI control sequences and need to be
         excluded from the length calculation.  So also a copy of the prompt
@@ -372,7 +372,7 @@
         while p >= 0 and b[p] <> '\n':
             p -= 1
         return p + 1
-    
+
     def eol(self, p=None):
         """Return the 0-based index of the line break following p most
         immediately.
@@ -463,7 +463,7 @@
         """This function is called to allow post command cleanup."""
         if getattr(cmd, "kills_digit_arg", 1):
             if self.arg is not None:
-                self.dirty = 1                
+                self.dirty = 1
             self.arg = None
 
     def prepare(self):
@@ -575,13 +575,15 @@
     def push_char(self, char):
         self.console.push_char(char)
         self.handle1(0)
-    
-    def readline(self, returns_unicode=False):
+
+    def readline(self, returns_unicode=False, startup_hook=None):
         """Read a line.  The implementation of this method also shows
         how to drive Reader if you want more control over the event
         loop."""
         self.prepare()
         try:
+            if startup_hook is not None:
+                startup_hook()
             self.refresh()
             while not self.finished:
                 self.handle1()
diff --git a/pyrepl/readline.py b/pyrepl/readline.py
--- a/pyrepl/readline.py
+++ b/pyrepl/readline.py
@@ -193,10 +193,8 @@
             reader = self.get_reader()
         except _error:
             return _old_raw_input(prompt)
-        if self.startup_hook is not None:
-            self.startup_hook()
         reader.ps1 = prompt
-        return reader.readline()
+        return reader.readline(reader, startup_hook=self.startup_hook)
 
     def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False):
         """Read an input on possibly multiple lines, asking for more
@@ -390,7 +388,7 @@
     global _old_raw_input
     if _old_raw_input is not None:
         return # don't run _setup twice
-    
+
     try:
         f_in = sys.stdin.fileno()
         f_out = sys.stdout.fileno()

From noreply at buildbot.pypy.org  Sat Apr 28 16:55:05 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 28 Apr 2012 16:55:05 +0200 (CEST)
Subject: [pypy-commit] pyrepl default: Change the default mapping of the
	keys Home and End
Message-ID: <20120428145505.732BB820B1@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r169:bf330a330af4
Date: 2012-04-20 11:37 +0200
http://bitbucket.org/pypy/pyrepl/changeset/bf330a330af4/

Log:	Change the default mapping of the keys Home and End to go to the
	start/end of the current line instead of the start/end of the whole
	multi-line block.

diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -152,8 +152,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal

From noreply at buildbot.pypy.org  Sat Apr 28 16:55:06 2012
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Sat, 28 Apr 2012 16:55:06 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: merge from default
Message-ID: <20120428145506.8FA14820B1@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r170:71e5bead2693
Date: 2012-04-28 16:54 +0200
http://bitbucket.org/pypy/pyrepl/changeset/71e5bead2693/

Log:	merge from default

diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -50,9 +50,9 @@
 # rewritten in C; it's not required though.
 try:
     raise ImportError # currently it's borked by the unicode support
-    
+
     from _pyrepl_utils import disp_str, init_unctrl_map
-    
+
     init_unctrl_map(_make_unctrl_map())
 
     del init_unctrl_map
@@ -127,7 +127,7 @@
      (r'\C-x\C-u', 'upcase-region'),
      (r'\C-y', 'yank'),
      (r'\C-z', 'suspend'),
-     
+
      (r'\M-b', 'backward-word'),
      (r'\M-c', 'capitalize-word'),
      (r'\M-d', 'kill-word'),
@@ -161,8 +161,8 @@
      (r'\<delete>', 'delete'),
      (r'\<backspace>', 'backspace'),
      (r'\M-\<backspace>', 'backward-kill-word'),
-     (r'\<end>', 'end'),
-     (r'\<home>', 'home'),
+     (r'\<end>', 'end-of-line'),         # was 'end'
+     (r'\<home>', 'beginning-of-line'),  # was 'home'
      (r'\<f1>', 'help'),
      (r'\EOF', 'end'),  # the entries in the terminfo database for xterms
      (r'\EOH', 'home'), # seem to be wrong.  this is a less than ideal
@@ -313,7 +313,7 @@
 
     def process_prompt(self, prompt):
         """ Process the prompt.
-        
+
         This means calculate the length of the prompt. The character \x01
         and \x02 are used to bracket ANSI control sequences and need to be
         excluded from the length calculation.  So also a copy of the prompt
@@ -382,7 +382,7 @@
         while p >= 0 and b[p] != '\n':
             p -= 1
         return p + 1
-    
+
     def eol(self, p=None):
         """Return the 0-based index of the line break following p most
         immediately.
@@ -473,7 +473,7 @@
         """This function is called to allow post command cleanup."""
         if getattr(cmd, "kills_digit_arg", 1):
             if self.arg is not None:
-                self.dirty = 1                
+                self.dirty = 1
             self.arg = None
 
     def prepare(self):
@@ -592,13 +592,15 @@
     def push_char(self, char):
         self.console.push_char(char)
         self.handle1(0)
-    
-    def readline(self, returns_unicode=False):
+
+    def readline(self, returns_unicode=False, startup_hook=None):
         """Read a line.  The implementation of this method also shows
         how to drive Reader if you want more control over the event
         loop."""
         self.prepare()
         try:
+            if startup_hook is not None:
+                startup_hook()
             self.refresh()
             while not self.finished:
                 self.handle1()
diff --git a/pyrepl/readline.py b/pyrepl/readline.py
--- a/pyrepl/readline.py
+++ b/pyrepl/readline.py
@@ -193,10 +193,8 @@
             reader = self.get_reader()
         except _error:
             return _old_raw_input(prompt)
-        if self.startup_hook is not None:
-            self.startup_hook()
         reader.ps1 = prompt
-        return reader.readline()
+        return reader.readline(reader, startup_hook=self.startup_hook)
 
     def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False):
         """Read an input on possibly multiple lines, asking for more
@@ -390,7 +388,7 @@
     global _old_raw_input
     if _old_raw_input is not None:
         return # don't run _setup twice
-    
+
     try:
         f_in = sys.stdin.fileno()
         f_out = sys.stdout.fileno()
diff --git a/pyrepl/unix_console.py b/pyrepl/unix_console.py
--- a/pyrepl/unix_console.py
+++ b/pyrepl/unix_console.py
@@ -420,7 +420,12 @@
                    e.args[4] == 'unexpected end of data':
                 pass
             else:
-                raise
+                # was: "raise".  But it crashes pyrepl, and by extension the
+                # pypy currently running, in which we are e.g. in the middle
+                # of some debugging session.  Argh.  Instead just print an
+                # error message to stderr and continue running, for now.
+                self.partial_char = ''
+                sys.stderr.write('\n%s: %s\n' % (e.__class__.__name__, e))
         else:
             self.partial_char = b''
             self.event_queue.push(c)

From noreply at buildbot.pypy.org  Sat Apr 28 20:07:07 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat, 28 Apr 2012 20:07:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: an rpython-level version of this test
 to clairify when this kind of traces occure
Message-ID: <20120428180707.84368820B1@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54793:ef6a5e8fe353
Date: 2012-04-28 20:05 +0200
http://bitbucket.org/pypy/pypy/changeset/ef6a5e8fe353/

Log:	an rpython-level version of this test to clairify when this kind of
	traces occure

diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -8,7 +8,7 @@
 from pypy.jit.metainterp.quasiimmut import get_current_qmut_instance
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.jit.codewriter.policy import StopAtXPolicy
-from pypy.rlib.jit import JitDriver, dont_look_inside, unroll_safe
+from pypy.rlib.jit import JitDriver, dont_look_inside, unroll_safe, promote
 
 
 def test_get_current_qmut_instance():
@@ -506,6 +506,27 @@
             "guard_not_invalidated": 2
         })
 
+    def test_issue1080(self):
+        myjitdriver = JitDriver(greens=[], reds=["n", "sa", "a"])
+        class Foo(object):
+            _immutable_fields_ = ["x?"]
+            def __init__(self, x):
+                self.x = x
+        one, two = Foo(1), Foo(2)
+        def main(n):
+            sa = 0
+            a = one
+            while n:
+                myjitdriver.jit_merge_point(n=n, sa=sa, a=a)
+                sa += a.x
+                if a.x == 1:
+                    a = two
+                elif a.x == 2:
+                    a = one                    
+                n -= 1
+            return sa
+        res = self.meta_interp(main, [10])
+        assert res == main(10)
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass

From noreply at buildbot.pypy.org  Sat Apr 28 20:26:28 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat, 28 Apr 2012 20:26:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: reenable virtualstate matching debug
	prints
Message-ID: <20120428182628.0B285820B1@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54794:de6a09192681
Date: 2012-04-28 20:09 +0200
http://bitbucket.org/pypy/pypy/changeset/de6a09192681/

Log:	reenable virtualstate matching debug prints

diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -526,8 +526,8 @@
         args = jumpop.getarglist()
         modifier = VirtualStateAdder(self.optimizer)
         virtual_state = modifier.get_virtual_state(args)
-        #debug_start('jit-log-virtualstate')
-        #virtual_state.debug_print("Looking for ")
+        debug_start('jit-log-virtualstate')
+        virtual_state.debug_print("Looking for ")
 
         for target in cell_token.target_tokens:
             if not target.virtual_state:
@@ -536,10 +536,10 @@
             extra_guards = []
 
             bad = {}
-            #debugmsg = 'Did not match '
+            debugmsg = 'Did not match '
             if target.virtual_state.generalization_of(virtual_state, bad):
                 ok = True
-                #debugmsg = 'Matched '
+                debugmsg = 'Matched '
             else:
                 try:
                     cpu = self.optimizer.cpu
@@ -548,13 +548,13 @@
                                                          extra_guards)
 
                     ok = True
-                    #debugmsg = 'Guarded to match '
+                    debugmsg = 'Guarded to match '
                 except InvalidLoop:
                     pass
-            #target.virtual_state.debug_print(debugmsg, bad)
+            target.virtual_state.debug_print(debugmsg, bad)
 
             if ok:
-                #debug_stop('jit-log-virtualstate')
+                debug_stop('jit-log-virtualstate')
 
                 values = [self.getvalue(arg)
                           for arg in jumpop.getarglist()]
@@ -581,7 +581,7 @@
                     jumpop.setdescr(cell_token.target_tokens[0])
                     self.optimizer.send_extra_operation(jumpop)
                 return True
-        #debug_stop('jit-log-virtualstate')
+        debug_stop('jit-log-virtualstate')
         return False
 
 class ValueImporter(object):

From noreply at buildbot.pypy.org  Sat Apr 28 20:26:29 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat, 28 Apr 2012 20:26:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add some comments on why loops become
	invalid
Message-ID: <20120428182629.4F541820B1@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54795:18980028c2dc
Date: 2012-04-28 20:25 +0200
http://bitbucket.org/pypy/pypy/changeset/18980028c2dc/

Log:	Add some comments on why loops become invalid

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -191,10 +191,13 @@
         # GUARD_OVERFLOW, then the loop is invalid.
         lastop = self.last_emitted_operation
         if lastop is None:
-            raise InvalidLoop
+            raise InvalidLoop('An INT_xxx_OVF was proven not to overflow but' +
+                              'guarded with GUARD_OVERFLOW')
         opnum = lastop.getopnum()
         if opnum not in (rop.INT_ADD_OVF, rop.INT_SUB_OVF, rop.INT_MUL_OVF):
-            raise InvalidLoop
+            raise InvalidLoop('An INT_xxx_OVF was proven not to overflow but' +
+                              'guarded with GUARD_OVERFLOW')
+                             
         self.emit_operation(op)
 
     def optimize_INT_ADD_OVF(self, op):
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -208,7 +208,8 @@
             box = value.box
             assert isinstance(box, Const)
             if not box.same_constant(constbox):
-                raise InvalidLoop
+                raise InvalidLoop('A GURAD_{VALUE,TRUE,FALSE} was proven to' +
+                                  'always fail')
             return
         if emit_operation:
             self.emit_operation(op)
@@ -220,7 +221,7 @@
         if value.is_null():
             return
         elif value.is_nonnull():
-            raise InvalidLoop
+            raise InvalidLoop('A GUARD_ISNULL was proven to always fail')
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
@@ -229,7 +230,7 @@
         if value.is_nonnull():
             return
         elif value.is_null():
-            raise InvalidLoop
+            raise InvalidLoop('A GUARD_NONNULL was proven to always fail')
         self.emit_operation(op)
         value.make_nonnull(op)
 
@@ -278,7 +279,7 @@
         if realclassbox is not None:
             if realclassbox.same_constant(expectedclassbox):
                 return
-            raise InvalidLoop
+            raise InvalidLoop('A GUARD_CLASS was proven to always fail')
         if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
@@ -301,7 +302,8 @@
     def optimize_GUARD_NONNULL_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
-            raise InvalidLoop
+            raise InvalidLoop('A GUARD_NONNULL_CLASS was proven to always ' +
+                              'fail')
         self.optimize_GUARD_CLASS(op)
 
     def optimize_CALL_LOOPINVARIANT(self, op):
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -378,7 +378,10 @@
             #final_virtual_state.debug_print("Bad virtual state at end of loop, ",
             #                                bad)
             #debug_stop('jit-log-virtualstate')
-            raise InvalidLoop
+            raise InvalidLoop('The virtual state at the end of the peeled ' +
+                              'loop is not compatible with the virtual ' +
+                              'state at the start of the loop which makes ' +
+                              'it impossible to close the loop')
             
         #debug_stop('jit-log-virtualstate')
 

From noreply at buildbot.pypy.org  Sat Apr 28 22:54:57 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat, 28 Apr 2012 22:54:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: Some more explanations of InvalidLoop
Message-ID: <20120428205457.4B8CB820B1@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54796:8cb0aa4c2211
Date: 2012-04-28 22:54 +0200
http://bitbucket.org/pypy/pypy/changeset/8cb0aa4c2211/

Log:	Some more explanations of InvalidLoop

diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -315,7 +315,10 @@
         try:
             jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         except BadVirtualState:
-            raise InvalidLoop
+            raise InvalidLoop('The state of the optimizer at the end of ' +
+                              'peeled loop is inconsistent with the ' +
+                              'VirtualState at the begining of the peeled ' +
+                              'loop')
         jumpop.initarglist(jumpargs)
 
         # Inline the short preamble at the end of the loop
@@ -325,7 +328,11 @@
         for i in range(len(short_inputargs)):
             if short_inputargs[i] in args:
                 if args[short_inputargs[i]] != jmp_to_short_args[i]:
-                    raise InvalidLoop
+                    raise InvalidLoop('The short preamble wants the ' +
+                                      'same box passed to multiple of its ' +
+                                      'inputargs, but the jump at the ' +
+                                      'end of this bridge does not do that.')
+                                      
             args[short_inputargs[i]] = jmp_to_short_args[i]
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         for op in self.short[1:]:
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -27,11 +27,15 @@
         if self.generalization_of(other, renum, {}):
             return
         if renum[self.position] != other.position:
-            raise InvalidLoop
+            raise InvalidLoop('The numbering of the virtual states does not ' +
+                              'match. This means that two virtual fields ' +
+                              'have been set to the same Box in one of the ' +
+                              'virtual states but not in the other.')
         self._generate_guards(other, box, cpu, extra_guards)
 
     def _generate_guards(self, other, box, cpu, extra_guards):
-        raise InvalidLoop
+        raise InvalidLoop('Generating guards for making the VirtualStates ' +
+                          'at hand match have not been implemented')
 
     def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
@@ -346,10 +350,12 @@
 
     def _generate_guards(self, other, box, cpu, extra_guards):
         if not isinstance(other, NotVirtualStateInfo):
-            raise InvalidLoop
+            raise InvalidLoop('The VirtualStates does not match as a ' +
+                              'virtual appears where a pointer is needed ' +
+                              'and it is too late to force it.')
 
         if self.lenbound or other.lenbound:
-            raise InvalidLoop
+            raise InvalidLoop('The array length bounds does not match.')
 
         if self.level == LEVEL_KNOWNCLASS and \
            box.nonnull() and \
@@ -400,7 +406,8 @@
             return
 
         # Remaining cases are probably not interesting
-        raise InvalidLoop
+        raise InvalidLoop('Generating guards for making the VirtualStates ' +
+                          'at hand match have not been implemented')
         if self.level == LEVEL_CONSTANT:
             import pdb; pdb.set_trace()
             raise NotImplementedError

From noreply at buildbot.pypy.org  Sun Apr 29 09:24:07 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 09:24:07 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Start to write support code around the
 GC for getting at thread-local
Message-ID: <20120429072407.BF3B982F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54797:133049a5ba84
Date: 2012-04-29 09:21 +0200
http://bitbucket.org/pypy/pypy/changeset/133049a5ba84/

Log:	Start to write support code around the GC for getting at thread-
	local data. This, and (by the way) also regular GC pointers in non-
	GC prebuilt structs, are not supported yet by the GC; it comes next.

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -86,14 +86,17 @@
 #     code (see tldict_lookup()).
 #
 # Invariant: between two transactions, all objects visible from the current
-# thread are always GLOBAL.  In particular:
+# thread are always GLOBAL, with the exception of the ones only reachable
+# via thread-local data structures, which remain LOCAL.  In particular:
 #
-#   - The LOCAL object of a thread are not visible at all from other threads.
+#   - The LOCAL objects of a thread are not visible at all from other threads.
 #     This means that in transactional mode there is *no* pointer from a
 #     GLOBAL object directly to a LOCAL object.
 #
 #   - At the end of enter_transactional_mode(), and at the beginning of
 #     leave_transactional_mode(), *all* objects everywhere are GLOBAL.
+#     (With the possible exception of the thread-local ones from the main
+#     thread.)
 #
 # Collection: for now we have only local_collection(), which ignores all
 # GLOBAL objects.
@@ -110,9 +113,11 @@
 #
 #   - A special case is the end-of-transaction collection, done by the same
 #     local_collection() with a twist: all pointers to a LOCAL COPY object
-#     are replaced with copies to the corresponding GLOBAL original.  When
-#     it is done, we mark all surviving LOCAL objects as GLOBAL too, and we
-#     are back to the situation where this thread sees only GLOBAL objects.
+#     are replaced with copies to the corresponding GLOBAL original.
+#     We additionally mark all surviving LOCAL objects as GLOBAL,
+#     unless they are only reachable via thread-locals.
+#     Then we are back to the situation where this thread sees only
+#     GLOBAL objects (again, with the exception of thread-locals).
 #     What we leave to the C code to do "as a finishing touch" is to copy
 #     transactionally the content of the LOCAL COPY objects back over the
 #     GLOBAL originals; before this is done, the transaction can be aborted
@@ -127,7 +132,7 @@
 #   - if GCFLAG_WAS_COPIED, it points to the GLOBAL original.
 #
 #   - if GCFLAG_HAS_SHADOW, to the shadow object outside the nursery.
-#     (It is not used on any other nursery object.)
+#     (It is not used on other nursery objects before collection.)
 #
 #   - it contains the 'next' object of the 'mt_global_turned_local' list.
 #
@@ -163,6 +168,7 @@
     #needs_write_barrier = "stm"
     prebuilt_gc_objects_are_static_roots = False
     malloc_zero_filled = True    # xxx?
+    handles_thread_locals = True
 
     HDR = lltype.Struct('header', ('tid', lltype.Signed),
                                   ('version', llmemory.Address))
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -168,6 +168,7 @@
         a_random_address = llmemory.cast_ptr_to_adr(foo)
         gcdata.static_root_start = a_random_address      # patched in finish()
         gcdata.static_root_nongcend = a_random_address   # patched in finish()
+        gcdata.static_root_thrlocend = a_random_address  # patched in finish()
         gcdata.static_root_end = a_random_address        # patched in finish()
         gcdata.max_type_id = 13                          # patched in finish()
         gcdata.typeids_z = a_random_address              # patched in finish()
@@ -207,6 +208,9 @@
             'static_root_nongcend',
             annmodel.SomeAddress())
         data_classdef.generalize_attr(
+            'static_root_thrlocend',
+            annmodel.SomeAddress())
+        data_classdef.generalize_attr(
             'static_root_end',
             annmodel.SomeAddress())
         data_classdef.generalize_attr(
@@ -583,9 +587,11 @@
         r_gcdata = self.translator.rtyper.getrepr(s_gcdata)
         ll_instance = rmodel.inputconst(r_gcdata, self.gcdata).value
 
+        lb = self.layoutbuilder
         addresses_of_static_ptrs = (
-            self.layoutbuilder.addresses_of_static_ptrs_in_nongc +
-            self.layoutbuilder.addresses_of_static_ptrs)
+            lb.addresses_of_static_ptrs_in_nongc +
+            lb.addresses_of_static_getters_thrloc +
+            lb.addresses_of_static_ptrs)
         log.info("found %s static roots" % (len(addresses_of_static_ptrs), ))
         ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address),
                                                len(addresses_of_static_ptrs),
@@ -593,9 +599,23 @@
 
         for i in range(len(addresses_of_static_ptrs)):
             ll_static_roots_inside[i] = addresses_of_static_ptrs[i]
-        ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address))
-        ll_instance.inst_static_root_nongcend = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs_in_nongc)
-        ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(addresses_of_static_ptrs)
+
+        ll_instance.inst_static_root_start = (
+            llmemory.cast_ptr_to_adr(ll_static_roots_inside) +
+            llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)))
+        ll_instance.inst_static_root_nongcend = (
+            ll_instance.inst_static_root_start +
+            sizeofaddr * len(lb.addresses_of_static_ptrs_in_nongc))
+        ll_instance.inst_static_root_thrlocend = (
+            ll_instance.inst_static_root_nongcend +
+            sizeofaddr * len(lb.addresses_of_static_getters_thrloc))
+        ll_instance.inst_static_root_end = (
+            ll_instance.inst_static_root_thrlocend +
+            sizeofaddr * len(lb.addresses_of_static_ptrs))
+        assert ll_instance.inst_static_root_end == (
+            ll_instance.inst_static_root_start +
+            sizeofaddr * len(addresses_of_static_ptrs))
+
         newgcdependencies = []
         newgcdependencies.append(ll_static_roots_inside)
         ll_instance.inst_max_type_id = len(group.members)
@@ -607,6 +627,15 @@
             ll_typeids_z[i] = typeids_z[i]
         ll_instance.inst_typeids_z = llmemory.cast_ptr_to_adr(ll_typeids_z)
         newgcdependencies.append(ll_typeids_z)
+        #
+        if lb.addresses_of_static_getters_thrloc:
+            annhelper = annlowlevel.MixLevelHelperAnnotator(
+                self.translator.rtyper)
+            for getter in lb.addresses_of_static_getters_thrloc:
+                annhelper.getgraph(getter.ptr._obj._callable, [],
+                                   annmodel.SomeAddress())
+            annhelper.finish()
+        #
         return newgcdependencies
 
     def get_finish_tables(self):
@@ -1374,7 +1403,8 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc):
+                   collect_static_in_prebuilt_gc,
+                   collect_static_in_prebuilt_threadlocal=None):
         gcdata = self.gcdata
         gc = self.gc
         if collect_static_in_prebuilt_nongc:
@@ -1385,8 +1415,18 @@
                 if gc.points_to_valid_gc_object(result):
                     collect_static_in_prebuilt_nongc(gc, result)
                 addr += sizeofaddr
+        if collect_static_in_prebuilt_threadlocal:
+            addr = gcdata.static_root_nongcend
+            end = gcdata.static_root_thrlocend
+            while addr != end:
+                fadr = addr.address[0]
+                fptr = llmemory.cast_adr_to_ptr(fadr, gctypelayout.GETTERFN)
+                result = fptr()
+                if gc.points_to_valid_gc_object(result):
+                    collect_static_in_prebuilt_threadlocal(gc, result)
+                addr += sizeofaddr
         if collect_static_in_prebuilt_gc:
-            addr = gcdata.static_root_nongcend
+            addr = gcdata.static_root_thrlocend
             end = gcdata.static_root_end
             while addr != end:
                 result = addr.address[0]
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -267,9 +267,14 @@
         # prebuilt structures.  It should list all the locations that could
         # possibly point to a GC heap object.
         # this lists contains pointers in GcStructs and GcArrays
+        # (note that on some GCs, we don't need this, so it is empty)
         self.addresses_of_static_ptrs = []
         # this lists contains pointers in raw Structs and Arrays
         self.addresses_of_static_ptrs_in_nongc = []
+        # this lists contains addresses of functions that just return
+        # the thread-local address of a GC pointer from a raw Struct
+        # with the 'stm_thread_local' hint.
+        self.addresses_of_static_getters_thrloc = []
         # for debugging, the following list collects all the prebuilt
         # GcStructs and GcArrays
         self.all_prebuilt_gc = []
@@ -425,24 +430,28 @@
         adr = llmemory.cast_ptr_to_adr(value._as_ptr())
         if TYPE._gckind == "gc":
             if gc.prebuilt_gc_objects_are_static_roots or gc.DEBUG:
-                appendto = self.addresses_of_static_ptrs
+                append = self.addresses_of_static_ptrs.append
             else:
                 return
         elif hasattr(TYPE, "_hints") and TYPE._hints.get('stm_thread_local'):
-            # The exception data's value object is skipped: it's a thread-
-            # local data structure.  We assume that objects are stored
-            # only temporarily there, so it is always cleared at the point
-            # where we collect the roots.
-            if TYPE._name == 'ExcData':
-                return
-            # Some other thread-local data don't have any GC pointers
-            # themselves.  These are fine.
-            assert list(gc_pointers_inside(value, adr)) == []
-            return
+            assert gc.handles_thread_locals
+            def append(a):
+                def getter():
+                    return a
+                from pypy.annotation import model as annmodel
+                from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
+                annhelper = MixLevelHelperAnnotator(self.translator.rtyper)
+                fptr = annhelper.delayedfunction(getter, [],
+                                                 annmodel.SomeAddress())
+                annhelper.finish()
+                adr = llmemory.cast_ptr_to_adr(fptr)
+                self.addresses_of_static_getters_thrloc.append(adr)
         else:
-            appendto = self.addresses_of_static_ptrs_in_nongc
+            append = self.addresses_of_static_ptrs_in_nongc.append
         for a in gc_pointers_inside(value, adr, mutable_only=True):
-            appendto.append(a)
+            append(a)
+
+GETTERFN = lltype.Ptr(lltype.FuncType([], llmemory.Address))
 
 # ____________________________________________________________
 #

From noreply at buildbot.pypy.org  Sun Apr 29 09:35:24 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 09:35:24 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: A failing test.
Message-ID: <20120429073524.53A3482F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54798:eb12e38ed389
Date: 2012-04-29 09:34 +0200
http://bitbucket.org/pypy/pypy/changeset/eb12e38ed389/

Log:	A failing test.

diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py
--- a/pypy/translator/stm/test/test_ztranslated.py
+++ b/pypy/translator/stm/test/test_ztranslated.py
@@ -63,3 +63,31 @@
         t, cbuilder = self.compile(entry_point, backendopt=True)
         data = cbuilder.cmdexec('')
         assert '12\n12\n' in data, "got: %r" % (data,)
+
+    def test_prebuilt_nongc(self):
+        class DoNothing(rstm.Transaction):
+            def run(self):
+                pass
+        from pypy.rpython.lltypesystem import lltype
+        R = lltype.GcStruct('R', ('x', lltype.Signed))
+        S1 = lltype.Struct('S1', ('r', lltype.Ptr(R)))
+        s1 = lltype.malloc(S1, immortal=True, flavor='raw')
+        S2 = lltype.Struct('S2', ('r', lltype.Ptr(R)),
+                           hints={'stm_thread_local': True})
+        s2 = lltype.malloc(S2, immortal=True, flavor='raw')
+        def do_stuff():
+            rstm.run_all_transactions(DoNothing())
+            print s1.r.x
+            print s2.r.x
+        do_stuff._dont_inline_ = True
+        def main(argv):
+            s1.r = lltype.malloc(R)
+            s1.r.x = 42
+            s2.r = lltype.malloc(R)
+            s2.r.x = 43
+            do_stuff()
+            return 0
+        #
+        t, cbuilder = self.compile(main)
+        data = cbuilder.cmdexec('')
+        assert '42\n43\n' in data, "got: %r" % (data,)

From noreply at buildbot.pypy.org  Sun Apr 29 10:06:01 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 10:06:01 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix the first half of the test.
Message-ID: <20120429080601.512D382F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54799:605a0a476290
Date: 2012-04-29 10:04 +0200
http://bitbucket.org/pypy/pypy/changeset/605a0a476290/

Log:	Fix the first half of the test.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -238,6 +238,9 @@
         # they are simply added to 'pending'.
         self.collect_roots_from_stack()
         #
+        # Find the roots that are living in raw structures.
+        self.collect_from_raw_structures()
+        #
         # Also find the roots that are the local copy of GCFLAG_WAS_COPIED
         # objects.
         if not self.in_main_thread:
@@ -382,6 +385,10 @@
         self.gc.root_walker.walk_current_stack_roots(
             StmGCTLS._trace_drag_out1, self)
 
+    def collect_from_raw_structures(self):
+        self.gc.root_walker.walk_current_nongc_roots(
+            StmGCTLS._trace_drag_out1, self)
+
     def trace_and_drag_out_of_nursery(self, obj):
         # This is called to fix the references inside 'obj', to ensure that
         # they are global.  If necessary, the referenced objects are copied
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -133,6 +133,8 @@
 class FakeRootWalker:
     def walk_current_stack_roots(self, *args):
         pass     # no stack roots in this test file
+    def walk_current_nongc_roots(self, *args):
+        pass     # no nongc roots in this test file
 
 
 class StmGCTests:
@@ -848,3 +850,17 @@
         assert a == sr1_adr
         a = self.gc.stm_normalize_global(tr1_adr)
         assert a == sr1_adr
+
+    def test_prebuilt_nongc(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        NONGC = lltype.Struct('NONGC', ('s', lltype.Ptr(S)))
+        nongc = lltype.malloc(NONGC, immortal=True, flavor='raw')
+        self.gc.root_walker.prebuilt_nongc = [(nongc, 's')]
+        #
+        s, _ = self.malloc(S, globl=False)      # a local object
+        nongc.s = s
+        self.gc.collect(0)                      # keeps LOCAL
+        s = nongc.s                             # reload, it moved
+        s_adr = llmemory.cast_ptr_to_adr(s)
+        self.checkflags(s_adr, False, False)    # check it survived
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -25,19 +25,36 @@
     pass
 
 class FakeRootWalker:
-    def walk_current_stack_roots(self, callback, arg):
+    current_stack = ()
+    prebuilt_nongc = ()
+
+    def collect_list(self, lst):
         A = lltype.Array(llmemory.Address)
-        roots = lltype.malloc(A, len(self.current_stack), flavor='raw')
-        for i in range(len(self.current_stack)):
-            roots[i] = llmemory.cast_ptr_to_adr(self.current_stack[i])
-        for i in range(len(self.current_stack)):
+        roots = lltype.malloc(A, len(lst), flavor='raw')
+        for i in range(len(lst)):
+            roots[i] = llmemory.cast_ptr_to_adr(lst[i])
+        for i in range(len(lst)):
             root = lltype.direct_ptradd(lltype.direct_arrayitems(roots), i)
             root = llmemory.cast_ptr_to_adr(root)
+            yield root
+        for i in range(len(lst)):
+            P = lltype.typeOf(lst[i])
+            lst[i] = llmemory.cast_adr_to_ptr(roots[i], P)
+        lltype.free(roots, flavor='raw')
+
+    def walk_current_stack_roots(self, callback, arg):
+        for root in self.collect_list(self.current_stack):
             callback(arg, root)
-        for i in range(len(self.current_stack)):
-            P = lltype.typeOf(self.current_stack[i])
-            self.current_stack[i] = llmemory.cast_adr_to_ptr(roots[i], P)
-        lltype.free(roots, flavor='raw')
+
+    def collect_field_list(self, lst):
+        for structptr, field in lst:
+            root = lltype.direct_fieldptr(structptr, field)
+            root = llmemory.cast_ptr_to_adr(root)
+            yield root
+
+    def walk_current_nongc_roots(self, callback, arg):
+        for root in self.collect_field_list(self.prebuilt_nongc):
+            callback(arg, root)
 
 class FakeGC:
     from pypy.rpython.memory.support import AddressDict, null_address_dict
diff --git a/pypy/rpython/memory/gctransform/stmframework.py b/pypy/rpython/memory/gctransform/stmframework.py
--- a/pypy/rpython/memory/gctransform/stmframework.py
+++ b/pypy/rpython/memory/gctransform/stmframework.py
@@ -198,3 +198,15 @@
         self.rootstackhook(collect_stack_root, arg,
                            stackgcdata.root_stack_base,
                            stackgcdata.root_stack_top)
+
+    @specialize.argtype(2)
+    def walk_current_nongc_roots(self, collect_nongc_root, arg):
+        gcdata = self.gcdata
+        gc = self.gc
+        addr = gcdata.static_root_start
+        end = gcdata.static_root_nongcend
+        while addr != end:
+            result = addr.address[0]
+            if gc.points_to_valid_gc_object(result):
+                collect_nongc_root(arg, result)
+            addr += sizeofaddr

From noreply at buildbot.pypy.org  Sun Apr 29 10:53:39 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 10:53:39 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Copy the logic for thread-locals. The
 last new test is the one failing
Message-ID: <20120429085339.1571682F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54800:4c6a625119f9
Date: 2012-04-29 10:33 +0200
http://bitbucket.org/pypy/pypy/changeset/4c6a625119f9/

Log:	Copy the logic for thread-locals. The last new test is the one
	failing because it needs slightly special care.

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -238,7 +238,8 @@
         # they are simply added to 'pending'.
         self.collect_roots_from_stack()
         #
-        # Find the roots that are living in raw structures.
+        # Find the roots that are living in raw structures, and in
+        # thread-local raw structures.
         self.collect_from_raw_structures()
         #
         # Also find the roots that are the local copy of GCFLAG_WAS_COPIED
@@ -388,6 +389,8 @@
     def collect_from_raw_structures(self):
         self.gc.root_walker.walk_current_nongc_roots(
             StmGCTLS._trace_drag_out1, self)
+        self.gc.root_walker.walk_current_thrloc_roots(
+            StmGCTLS._trace_drag_out1, self)
 
     def trace_and_drag_out_of_nursery(self, obj):
         # This is called to fix the references inside 'obj', to ensure that
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -135,6 +135,8 @@
         pass     # no stack roots in this test file
     def walk_current_nongc_roots(self, *args):
         pass     # no nongc roots in this test file
+    def walk_current_thrloc_roots(self, *args):
+        pass     # no thread-local roots in this test file
 
 
 class StmGCTests:
@@ -851,7 +853,7 @@
         a = self.gc.stm_normalize_global(tr1_adr)
         assert a == sr1_adr
 
-    def test_prebuilt_nongc(self):
+    def test_prebuilt_nongc_main(self):
         from pypy.rpython.memory.gc.test import test_stmtls
         self.gc.root_walker = test_stmtls.FakeRootWalker()
         NONGC = lltype.Struct('NONGC', ('s', lltype.Ptr(S)))
@@ -863,4 +865,50 @@
         self.gc.collect(0)                      # keeps LOCAL
         s = nongc.s                             # reload, it moved
         s_adr = llmemory.cast_ptr_to_adr(s)
-        self.checkflags(s_adr, False, False)    # check it survived
+        self.checkflags(s_adr, False, False)    # check it survived; local
+
+    def test_prebuilt_nongc_enterleave(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        NONGC = lltype.Struct('NONGC', ('s', lltype.Ptr(S)))
+        nongc = lltype.malloc(NONGC, immortal=True, flavor='raw')
+        self.gc.root_walker.prebuilt_nongc = [(nongc, 's')]
+        #
+        s, _ = self.malloc(S, globl=False)      # a local object
+        nongc.s = s
+        self.gc.enter_transactional_mode()      # forces it to become GLOBAL
+        self.gc.leave_transactional_mode()
+        s = nongc.s                             # reload, it moved
+        s_adr = llmemory.cast_ptr_to_adr(s)
+        self.checkflags(s_adr, True, False)     # check it survived; global
+
+    def test_prebuilt_threadlocal_main(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        THREADLOCAL = lltype.Struct('THREADLOCAL', ('s', lltype.Ptr(S)),
+                                    hints={'stm_thread_local': True})
+        threadlocal = lltype.malloc(THREADLOCAL, immortal=True, flavor='raw')
+        self.gc.root_walker.prebuilt_threadlocal = [(threadlocal, 's')]
+        #
+        s, _ = self.malloc(S, globl=False)      # a local object
+        threadlocal.s = s
+        self.gc.collect(0)                      # keeps LOCAL
+        s = threadlocal.s                       # reload, it moved
+        s_adr = llmemory.cast_ptr_to_adr(s)
+        self.checkflags(s_adr, False, False)    # check it survived; local
+
+    def test_prebuilt_threadlocal_enterleave(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        THREADLOCAL = lltype.Struct('THREADLOCAL', ('s', lltype.Ptr(S)),
+                                    hints={'stm_thread_local': True})
+        threadlocal = lltype.malloc(THREADLOCAL, immortal=True, flavor='raw')
+        self.gc.root_walker.prebuilt_threadlocal = [(threadlocal, 's')]
+        #
+        s, _ = self.malloc(S, globl=False)     # a local object
+        threadlocal.s = s
+        self.gc.enter_transactional_mode()     # 's' remains LOCAL (difference)
+        self.gc.leave_transactional_mode()
+        s = threadlocal.s                      # reload, it moved
+        s_adr = llmemory.cast_ptr_to_adr(s)
+        self.checkflags(s_adr, False, False)   # check it survived; still local
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -27,6 +27,7 @@
 class FakeRootWalker:
     current_stack = ()
     prebuilt_nongc = ()
+    prebuilt_threadlocal = ()
 
     def collect_list(self, lst):
         A = lltype.Array(llmemory.Address)
@@ -56,6 +57,10 @@
         for root in self.collect_field_list(self.prebuilt_nongc):
             callback(arg, root)
 
+    def walk_current_thrloc_roots(self, callback, arg):
+        for root in self.collect_field_list(self.prebuilt_threadlocal):
+            callback(arg, root)
+
 class FakeGC:
     from pypy.rpython.memory.support import AddressDict, null_address_dict
     AddressStack = get_address_stack()

From noreply at buildbot.pypy.org  Sun Apr 29 12:31:56 2012
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 29 Apr 2012 12:31:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: log the cause of InvalidLoop
Message-ID: <20120429103156.8404A82F50@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r54801:20af87d3217b
Date: 2012-04-29 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/20af87d3217b/

Log:	log the cause of InvalidLoop

diff --git a/pypy/jit/metainterp/jitexc.py b/pypy/jit/metainterp/jitexc.py
--- a/pypy/jit/metainterp/jitexc.py
+++ b/pypy/jit/metainterp/jitexc.py
@@ -12,7 +12,6 @@
     """
     _go_through_llinterp_uncaught_ = True     # ugh
 
-
 def _get_standard_error(rtyper, Class):
     exdata = rtyper.getexceptiondata()
     clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class)
diff --git a/pypy/jit/metainterp/optimize.py b/pypy/jit/metainterp/optimize.py
--- a/pypy/jit/metainterp/optimize.py
+++ b/pypy/jit/metainterp/optimize.py
@@ -5,3 +5,9 @@
     """Raised when the optimize*.py detect that the loop that
     we are trying to build cannot possibly make sense as a
     long-running loop (e.g. it cannot run 2 complete iterations)."""
+
+    def __init__(self, msg='?'):
+        debug_start("jit-abort")
+        debug_print(msg)
+        debug_stop("jit-abort")
+        self.msg = msg

From noreply at buildbot.pypy.org  Sun Apr 29 13:20:37 2012
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Sun, 29 Apr 2012 13:20:37 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-issue1137: got the Custom*Object classes
 into setup_class, but it's not clean.
Message-ID: <20120429112037.931E782F50@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpypy-issue1137
Changeset: r54802:7714c8bfaaec
Date: 2012-04-29 13:19 +0200
http://bitbucket.org/pypy/pypy/changeset/7714c8bfaaec/

Log:	got the Custom*Object classes into setup_class, but it's not clean.

	now setup_class has a copy of BaseNumpyAppTest.setup_class in it :(

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -8,6 +8,12 @@
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
+# needed only for the verbatim copy of setup_class from test_base
+from pypy.conftest import gettestobjspace
+from pypy.module.micronumpy.interp_dtype import nonnative_byteorder_prefix,\
+     byteorder_prefix
+from pypy.conftest import option
+import sys
 
 class MockDtype(object):
     class itemtype(object):
@@ -195,6 +201,48 @@
         assert _to_coords(13, 'F') == [1, 0, 2]
 
 class AppTestNumArray(BaseNumpyAppTest):
+    def setup_class(cls):
+        # this is a verbatim copy from test_base, because calling the base classes
+        # setup_class method doesn't seem to work at all.
+        if option.runappdirect:
+            if '__pypy__' not in sys.builtin_module_names:
+                import numpy
+                sys.modules['numpypy'] = numpy
+                sys.modules['_numpypy'] = numpy
+        cls.space = gettestobjspace(usemodules=['micronumpy'])
+        cls.w_non_native_prefix = cls.space.wrap(nonnative_byteorder_prefix)
+        cls.w_native_prefix = cls.space.wrap(byteorder_prefix)
+
+        w_tup = cls.space.appexec([], """():
+
+        class CustomIndexObject(object):
+            def __init__(self, index):
+                self.index = index
+            def __index__(self):
+                return self.index
+
+        class CustomIndexIntObject(object):
+            def __init__(self, index, value):
+                self.index = index
+                self.value = value
+            def __index__(self):
+                return self.index
+            def __int__(self):
+                return self.value
+
+        class CustomIntObject(object):
+            def __init__(self, value):
+                self.value = value
+            def __index__(self):
+                return self.value
+
+        return CustomIndexObject, CustomIndexIntObject, CustomIntObject""")
+
+        tup = cls.space.unpackiterable(w_tup)
+        cls.w_CustomIndexObject = tup[0]
+        cls.w_CustomIndexIntObject = tup[1]
+        cls.w_CustomIntObject = tup[2]
+
     def test_ndarray(self):
         from _numpypy import ndarray, array, dtype
 
@@ -332,45 +380,24 @@
     def test_getitem_obj_index(self):
         from _numpypy import arange
 
-        class CustomIndexObject(object):
-            def __init__(self, index):
-                self.index = index
-            def __index__(self):
-                return self.index
-
         a = arange(10)
 
-        assert a[CustomIndexObject(1)] == 1
+        assert a[self.CustomIndexObject(1)] == 1
 
     def test_getitem_obj_prefer_index_to_int(self):
         from _numpypy import arange
 
-        class CustomIndexIntObject(object):
-            def __init__(self, index, value):
-                self.index = index
-                self.value = value
-            def __index__(self):
-                return self.index
-            def __int__(self):
-                return self.value
-
         a = arange(10)
 
 
-        assert a[CustomIndexIntObject(0, 1)] == 0
+        assert a[self.CustomIndexIntObject(0, 1)] == 0
 
     def test_getitem_obj_int(self):
         from _numpypy import arange
 
-        class CustomIntObject(object):
-            def __init__(self, value):
-                self.value = value
-            def __index__(self):
-                return self.value
-
         a = arange(10)
 
-        assert a[CustomIntObject(1)] == 1
+        assert a[self.CustomIntObject(1)] == 1
 
     def test_setitem(self):
         from _numpypy import array
@@ -394,46 +421,25 @@
     def test_setitem_obj_index(self):
         from _numpypy import arange
 
-        class CustomIndexObject(object):
-            def __init__(self, index):
-                self.index = index
-            def __index__(self):
-                return self.index
-
         a = arange(10)
 
-        a[CustomIndexObject(1)] = 100
+        a[self.CustomIndexObject(1)] = 100
         assert a[1] == 100
 
     def test_setitem_obj_prefer_index_to_int(self):
         from _numpypy import arange
 
-        class CustomIndexIntObject(object):
-            def __init__(self, index, value):
-                self.index = index
-                self.value = value
-            def __index__(self):
-                return self.index
-            def __int__(self):
-                return self.value
-
         a = arange(10)
 
-        a[CustomIndexIntObject(0, 1)] = 100
+        a[self.CustomIndexIntObject(0, 1)] = 100
         assert a[0] == 100
 
     def test_setitem_obj_int(self):
         from _numpypy import arange
 
-        class CustomIntObject(object):
-            def __init__(self, index):
-                self.index = index
-            def __index__(self):
-                return self.index
-
         a = arange(10)
 
-        a[CustomIntObject(1)] = 100
+        a[self.CustomIntObject(1)] = 100
 
         assert a[1] == 100
 

From noreply at buildbot.pypy.org  Sun Apr 29 14:35:52 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 14:35:52 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Keep the extra test from the previous
	backout.
Message-ID: <20120429123552.BE2D082F51@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54804:dbcb6999ce17
Date: 2012-04-29 14:24 +0200
http://bitbucket.org/pypy/pypy/changeset/dbcb6999ce17/

Log:	Keep the extra test from the previous backout.

diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -863,4 +863,19 @@
         self.gc.collect(0)                      # keeps LOCAL
         s = nongc.s                             # reload, it moved
         s_adr = llmemory.cast_ptr_to_adr(s)
-        self.checkflags(s_adr, False, False)    # check it survived
+        self.checkflags(s_adr, False, False)    # check it survived; local
+
+    def test_prebuilt_nongc_enterleave(self):
+        from pypy.rpython.memory.gc.test import test_stmtls
+        self.gc.root_walker = test_stmtls.FakeRootWalker()
+        NONGC = lltype.Struct('NONGC', ('s', lltype.Ptr(S)))
+        nongc = lltype.malloc(NONGC, immortal=True, flavor='raw')
+        self.gc.root_walker.prebuilt_nongc = [(nongc, 's')]
+        #
+        s, _ = self.malloc(S, globl=False)      # a local object
+        nongc.s = s
+        self.gc.enter_transactional_mode()      # forces it to become GLOBAL
+        self.gc.leave_transactional_mode()
+        s = nongc.s                             # reload, it moved
+        s_adr = llmemory.cast_ptr_to_adr(s)
+        self.checkflags(s_adr, True, False)     # check it survived; global

From noreply at buildbot.pypy.org  Sun Apr 29 14:35:51 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 14:35:51 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Backed out changeset 4c6a625119f9
Message-ID: <20120429123551.8595F82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54803:5c76173fe058
Date: 2012-04-29 14:23 +0200
http://bitbucket.org/pypy/pypy/changeset/5c76173fe058/

Log:	Backed out changeset 4c6a625119f9

diff --git a/pypy/rpython/memory/gc/stmtls.py b/pypy/rpython/memory/gc/stmtls.py
--- a/pypy/rpython/memory/gc/stmtls.py
+++ b/pypy/rpython/memory/gc/stmtls.py
@@ -238,8 +238,7 @@
         # they are simply added to 'pending'.
         self.collect_roots_from_stack()
         #
-        # Find the roots that are living in raw structures, and in
-        # thread-local raw structures.
+        # Find the roots that are living in raw structures.
         self.collect_from_raw_structures()
         #
         # Also find the roots that are the local copy of GCFLAG_WAS_COPIED
@@ -389,8 +388,6 @@
     def collect_from_raw_structures(self):
         self.gc.root_walker.walk_current_nongc_roots(
             StmGCTLS._trace_drag_out1, self)
-        self.gc.root_walker.walk_current_thrloc_roots(
-            StmGCTLS._trace_drag_out1, self)
 
     def trace_and_drag_out_of_nursery(self, obj):
         # This is called to fix the references inside 'obj', to ensure that
diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py
--- a/pypy/rpython/memory/gc/test/test_stmgc.py
+++ b/pypy/rpython/memory/gc/test/test_stmgc.py
@@ -135,8 +135,6 @@
         pass     # no stack roots in this test file
     def walk_current_nongc_roots(self, *args):
         pass     # no nongc roots in this test file
-    def walk_current_thrloc_roots(self, *args):
-        pass     # no thread-local roots in this test file
 
 
 class StmGCTests:
@@ -853,7 +851,7 @@
         a = self.gc.stm_normalize_global(tr1_adr)
         assert a == sr1_adr
 
-    def test_prebuilt_nongc_main(self):
+    def test_prebuilt_nongc(self):
         from pypy.rpython.memory.gc.test import test_stmtls
         self.gc.root_walker = test_stmtls.FakeRootWalker()
         NONGC = lltype.Struct('NONGC', ('s', lltype.Ptr(S)))
@@ -865,50 +863,4 @@
         self.gc.collect(0)                      # keeps LOCAL
         s = nongc.s                             # reload, it moved
         s_adr = llmemory.cast_ptr_to_adr(s)
-        self.checkflags(s_adr, False, False)    # check it survived; local
-
-    def test_prebuilt_nongc_enterleave(self):
-        from pypy.rpython.memory.gc.test import test_stmtls
-        self.gc.root_walker = test_stmtls.FakeRootWalker()
-        NONGC = lltype.Struct('NONGC', ('s', lltype.Ptr(S)))
-        nongc = lltype.malloc(NONGC, immortal=True, flavor='raw')
-        self.gc.root_walker.prebuilt_nongc = [(nongc, 's')]
-        #
-        s, _ = self.malloc(S, globl=False)      # a local object
-        nongc.s = s
-        self.gc.enter_transactional_mode()      # forces it to become GLOBAL
-        self.gc.leave_transactional_mode()
-        s = nongc.s                             # reload, it moved
-        s_adr = llmemory.cast_ptr_to_adr(s)
-        self.checkflags(s_adr, True, False)     # check it survived; global
-
-    def test_prebuilt_threadlocal_main(self):
-        from pypy.rpython.memory.gc.test import test_stmtls
-        self.gc.root_walker = test_stmtls.FakeRootWalker()
-        THREADLOCAL = lltype.Struct('THREADLOCAL', ('s', lltype.Ptr(S)),
-                                    hints={'stm_thread_local': True})
-        threadlocal = lltype.malloc(THREADLOCAL, immortal=True, flavor='raw')
-        self.gc.root_walker.prebuilt_threadlocal = [(threadlocal, 's')]
-        #
-        s, _ = self.malloc(S, globl=False)      # a local object
-        threadlocal.s = s
-        self.gc.collect(0)                      # keeps LOCAL
-        s = threadlocal.s                       # reload, it moved
-        s_adr = llmemory.cast_ptr_to_adr(s)
-        self.checkflags(s_adr, False, False)    # check it survived; local
-
-    def test_prebuilt_threadlocal_enterleave(self):
-        from pypy.rpython.memory.gc.test import test_stmtls
-        self.gc.root_walker = test_stmtls.FakeRootWalker()
-        THREADLOCAL = lltype.Struct('THREADLOCAL', ('s', lltype.Ptr(S)),
-                                    hints={'stm_thread_local': True})
-        threadlocal = lltype.malloc(THREADLOCAL, immortal=True, flavor='raw')
-        self.gc.root_walker.prebuilt_threadlocal = [(threadlocal, 's')]
-        #
-        s, _ = self.malloc(S, globl=False)     # a local object
-        threadlocal.s = s
-        self.gc.enter_transactional_mode()     # 's' remains LOCAL (difference)
-        self.gc.leave_transactional_mode()
-        s = threadlocal.s                      # reload, it moved
-        s_adr = llmemory.cast_ptr_to_adr(s)
-        self.checkflags(s_adr, False, False)   # check it survived; still local
+        self.checkflags(s_adr, False, False)    # check it survived
diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py
--- a/pypy/rpython/memory/gc/test/test_stmtls.py
+++ b/pypy/rpython/memory/gc/test/test_stmtls.py
@@ -27,7 +27,6 @@
 class FakeRootWalker:
     current_stack = ()
     prebuilt_nongc = ()
-    prebuilt_threadlocal = ()
 
     def collect_list(self, lst):
         A = lltype.Array(llmemory.Address)
@@ -57,10 +56,6 @@
         for root in self.collect_field_list(self.prebuilt_nongc):
             callback(arg, root)
 
-    def walk_current_thrloc_roots(self, callback, arg):
-        for root in self.collect_field_list(self.prebuilt_threadlocal):
-            callback(arg, root)
-
 class FakeGC:
     from pypy.rpython.memory.support import AddressDict, null_address_dict
     AddressStack = get_address_stack()

From noreply at buildbot.pypy.org  Sun Apr 29 14:35:54 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 14:35:54 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Remove the part of the test handling
	thread-locals
Message-ID: <20120429123554.19D2D82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54805:6e370cf3d797
Date: 2012-04-29 14:25 +0200
http://bitbucket.org/pypy/pypy/changeset/6e370cf3d797/

Log:	Remove the part of the test handling thread-locals

diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py
--- a/pypy/translator/stm/test/test_ztranslated.py
+++ b/pypy/translator/stm/test/test_ztranslated.py
@@ -72,22 +72,22 @@
         R = lltype.GcStruct('R', ('x', lltype.Signed))
         S1 = lltype.Struct('S1', ('r', lltype.Ptr(R)))
         s1 = lltype.malloc(S1, immortal=True, flavor='raw')
-        S2 = lltype.Struct('S2', ('r', lltype.Ptr(R)),
-                           hints={'stm_thread_local': True})
-        s2 = lltype.malloc(S2, immortal=True, flavor='raw')
+        #S2 = lltype.Struct('S2', ('r', lltype.Ptr(R)),
+        #                   hints={'stm_thread_local': True})
+        #s2 = lltype.malloc(S2, immortal=True, flavor='raw')
         def do_stuff():
             rstm.run_all_transactions(DoNothing())
             print s1.r.x
-            print s2.r.x
+            #print s2.r.x
         do_stuff._dont_inline_ = True
         def main(argv):
             s1.r = lltype.malloc(R)
             s1.r.x = 42
-            s2.r = lltype.malloc(R)
-            s2.r.x = 43
+            #s2.r = lltype.malloc(R)
+            #s2.r.x = 43
             do_stuff()
             return 0
         #
         t, cbuilder = self.compile(main)
         data = cbuilder.cmdexec('')
-        assert '42\n43\n' in data, "got: %r" % (data,)
+        assert '42\n' in data, "got: %r" % (data,)

From noreply at buildbot.pypy.org  Sun Apr 29 14:35:55 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 14:35:55 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Manually backout most of 133049a5ba84.
Message-ID: <20120429123555.5F25F82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54806:56bc17590024
Date: 2012-04-29 14:29 +0200
http://bitbucket.org/pypy/pypy/changeset/56bc17590024/

Log:	Manually backout most of 133049a5ba84.

	The idea doesn't work as it is. The issue is what occurs in case a
	transaction is aborted. The thread-local data cannot stay around
	for the next transaction, because it now points to half-random data.
	:-(

diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py
--- a/pypy/rpython/memory/gc/stmgc.py
+++ b/pypy/rpython/memory/gc/stmgc.py
@@ -86,8 +86,7 @@
 #     code (see tldict_lookup()).
 #
 # Invariant: between two transactions, all objects visible from the current
-# thread are always GLOBAL, with the exception of the ones only reachable
-# via thread-local data structures, which remain LOCAL.  In particular:
+# thread are always GLOBAL.  In particular:
 #
 #   - The LOCAL objects of a thread are not visible at all from other threads.
 #     This means that in transactional mode there is *no* pointer from a
@@ -95,8 +94,6 @@
 #
 #   - At the end of enter_transactional_mode(), and at the beginning of
 #     leave_transactional_mode(), *all* objects everywhere are GLOBAL.
-#     (With the possible exception of the thread-local ones from the main
-#     thread.)
 #
 # Collection: for now we have only local_collection(), which ignores all
 # GLOBAL objects.
@@ -113,11 +110,9 @@
 #
 #   - A special case is the end-of-transaction collection, done by the same
 #     local_collection() with a twist: all pointers to a LOCAL COPY object
-#     are replaced with copies to the corresponding GLOBAL original.
-#     We additionally mark all surviving LOCAL objects as GLOBAL,
-#     unless they are only reachable via thread-locals.
-#     Then we are back to the situation where this thread sees only
-#     GLOBAL objects (again, with the exception of thread-locals).
+#     are replaced with copies to the corresponding GLOBAL original.  When
+#     it is done, we mark all surviving LOCAL objects as GLOBAL too, and we
+#     are back to the situation where this thread sees only GLOBAL objects.
 #     What we leave to the C code to do "as a finishing touch" is to copy
 #     transactionally the content of the LOCAL COPY objects back over the
 #     GLOBAL originals; before this is done, the transaction can be aborted
@@ -168,7 +163,6 @@
     #needs_write_barrier = "stm"
     prebuilt_gc_objects_are_static_roots = False
     malloc_zero_filled = True    # xxx?
-    handles_thread_locals = True
 
     HDR = lltype.Struct('header', ('tid', lltype.Signed),
                                   ('version', llmemory.Address))
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -168,7 +168,6 @@
         a_random_address = llmemory.cast_ptr_to_adr(foo)
         gcdata.static_root_start = a_random_address      # patched in finish()
         gcdata.static_root_nongcend = a_random_address   # patched in finish()
-        gcdata.static_root_thrlocend = a_random_address  # patched in finish()
         gcdata.static_root_end = a_random_address        # patched in finish()
         gcdata.max_type_id = 13                          # patched in finish()
         gcdata.typeids_z = a_random_address              # patched in finish()
@@ -208,9 +207,6 @@
             'static_root_nongcend',
             annmodel.SomeAddress())
         data_classdef.generalize_attr(
-            'static_root_thrlocend',
-            annmodel.SomeAddress())
-        data_classdef.generalize_attr(
             'static_root_end',
             annmodel.SomeAddress())
         data_classdef.generalize_attr(
@@ -587,11 +583,9 @@
         r_gcdata = self.translator.rtyper.getrepr(s_gcdata)
         ll_instance = rmodel.inputconst(r_gcdata, self.gcdata).value
 
-        lb = self.layoutbuilder
         addresses_of_static_ptrs = (
-            lb.addresses_of_static_ptrs_in_nongc +
-            lb.addresses_of_static_getters_thrloc +
-            lb.addresses_of_static_ptrs)
+            self.layoutbuilder.addresses_of_static_ptrs_in_nongc +
+            self.layoutbuilder.addresses_of_static_ptrs)
         log.info("found %s static roots" % (len(addresses_of_static_ptrs), ))
         ll_static_roots_inside = lltype.malloc(lltype.Array(llmemory.Address),
                                                len(addresses_of_static_ptrs),
@@ -599,23 +593,9 @@
 
         for i in range(len(addresses_of_static_ptrs)):
             ll_static_roots_inside[i] = addresses_of_static_ptrs[i]
-
-        ll_instance.inst_static_root_start = (
-            llmemory.cast_ptr_to_adr(ll_static_roots_inside) +
-            llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address)))
-        ll_instance.inst_static_root_nongcend = (
-            ll_instance.inst_static_root_start +
-            sizeofaddr * len(lb.addresses_of_static_ptrs_in_nongc))
-        ll_instance.inst_static_root_thrlocend = (
-            ll_instance.inst_static_root_nongcend +
-            sizeofaddr * len(lb.addresses_of_static_getters_thrloc))
-        ll_instance.inst_static_root_end = (
-            ll_instance.inst_static_root_thrlocend +
-            sizeofaddr * len(lb.addresses_of_static_ptrs))
-        assert ll_instance.inst_static_root_end == (
-            ll_instance.inst_static_root_start +
-            sizeofaddr * len(addresses_of_static_ptrs))
-
+        ll_instance.inst_static_root_start = llmemory.cast_ptr_to_adr(ll_static_roots_inside) + llmemory.ArrayItemsOffset(lltype.Array(llmemory.Address))
+        ll_instance.inst_static_root_nongcend = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(self.layoutbuilder.addresses_of_static_ptrs_in_nongc)
+        ll_instance.inst_static_root_end = ll_instance.inst_static_root_start + llmemory.sizeof(llmemory.Address) * len(addresses_of_static_ptrs)
         newgcdependencies = []
         newgcdependencies.append(ll_static_roots_inside)
         ll_instance.inst_max_type_id = len(group.members)
@@ -627,15 +607,6 @@
             ll_typeids_z[i] = typeids_z[i]
         ll_instance.inst_typeids_z = llmemory.cast_ptr_to_adr(ll_typeids_z)
         newgcdependencies.append(ll_typeids_z)
-        #
-        if lb.addresses_of_static_getters_thrloc:
-            annhelper = annlowlevel.MixLevelHelperAnnotator(
-                self.translator.rtyper)
-            for getter in lb.addresses_of_static_getters_thrloc:
-                annhelper.getgraph(getter.ptr._obj._callable, [],
-                                   annmodel.SomeAddress())
-            annhelper.finish()
-        #
         return newgcdependencies
 
     def get_finish_tables(self):
@@ -1403,8 +1374,7 @@
 
     def walk_roots(self, collect_stack_root,
                    collect_static_in_prebuilt_nongc,
-                   collect_static_in_prebuilt_gc,
-                   collect_static_in_prebuilt_threadlocal=None):
+                   collect_static_in_prebuilt_gc):
         gcdata = self.gcdata
         gc = self.gc
         if collect_static_in_prebuilt_nongc:
@@ -1415,18 +1385,8 @@
                 if gc.points_to_valid_gc_object(result):
                     collect_static_in_prebuilt_nongc(gc, result)
                 addr += sizeofaddr
-        if collect_static_in_prebuilt_threadlocal:
+        if collect_static_in_prebuilt_gc:
             addr = gcdata.static_root_nongcend
-            end = gcdata.static_root_thrlocend
-            while addr != end:
-                fadr = addr.address[0]
-                fptr = llmemory.cast_adr_to_ptr(fadr, gctypelayout.GETTERFN)
-                result = fptr()
-                if gc.points_to_valid_gc_object(result):
-                    collect_static_in_prebuilt_threadlocal(gc, result)
-                addr += sizeofaddr
-        if collect_static_in_prebuilt_gc:
-            addr = gcdata.static_root_thrlocend
             end = gcdata.static_root_end
             while addr != end:
                 result = addr.address[0]
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -266,15 +266,11 @@
         # the following are lists of addresses of gc pointers living inside the
         # prebuilt structures.  It should list all the locations that could
         # possibly point to a GC heap object.
-        # this lists contains pointers in GcStructs and GcArrays
+        # this list contains pointers in GcStructs and GcArrays
         # (note that on some GCs, we don't need this, so it is empty)
         self.addresses_of_static_ptrs = []
-        # this lists contains pointers in raw Structs and Arrays
+        # this list contains pointers in raw Structs and Arrays
         self.addresses_of_static_ptrs_in_nongc = []
-        # this lists contains addresses of functions that just return
-        # the thread-local address of a GC pointer from a raw Struct
-        # with the 'stm_thread_local' hint.
-        self.addresses_of_static_getters_thrloc = []
         # for debugging, the following list collects all the prebuilt
         # GcStructs and GcArrays
         self.all_prebuilt_gc = []
@@ -430,28 +426,24 @@
         adr = llmemory.cast_ptr_to_adr(value._as_ptr())
         if TYPE._gckind == "gc":
             if gc.prebuilt_gc_objects_are_static_roots or gc.DEBUG:
-                append = self.addresses_of_static_ptrs.append
+                appendto = self.addresses_of_static_ptrs
             else:
                 return
         elif hasattr(TYPE, "_hints") and TYPE._hints.get('stm_thread_local'):
-            assert gc.handles_thread_locals
-            def append(a):
-                def getter():
-                    return a
-                from pypy.annotation import model as annmodel
-                from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
-                annhelper = MixLevelHelperAnnotator(self.translator.rtyper)
-                fptr = annhelper.delayedfunction(getter, [],
-                                                 annmodel.SomeAddress())
-                annhelper.finish()
-                adr = llmemory.cast_ptr_to_adr(fptr)
-                self.addresses_of_static_getters_thrloc.append(adr)
+            # The exception data's value object is skipped: it's a thread-
+            # local data structure.  We assume that objects are stored
+            # only temporarily there, so it is always cleared at the point
+            # where we collect the roots.
+            if TYPE._name == 'ExcData':
+                return
+            # Some other thread-local data don't have any GC pointers
+            # themselves.  These are fine.
+            assert list(gc_pointers_inside(value, adr)) == []
+            return
         else:
-            append = self.addresses_of_static_ptrs_in_nongc.append
+            appendto = self.addresses_of_static_ptrs_in_nongc
         for a in gc_pointers_inside(value, adr, mutable_only=True):
-            append(a)
-
-GETTERFN = lltype.Ptr(lltype.FuncType([], llmemory.Address))
+            appendto.append(a)
 
 # ____________________________________________________________
 #

From noreply at buildbot.pypy.org  Sun Apr 29 14:35:56 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 14:35:56 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Disable again the ThreadLocal class.
Message-ID: <20120429123556.A31BA82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54807:aa7889a3bf14
Date: 2012-04-29 14:31 +0200
http://bitbucket.org/pypy/pypy/changeset/aa7889a3bf14/

Log:	Disable again the ThreadLocal class.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -225,7 +225,7 @@
     llop.stm_thread_stopping(lltype.Void)
 
 
-class ThreadLocal(object):
+class DISABLEDThreadLocal(object):
     """
     A thread-local container.  Use only for one or a few static places,
     e.g. the ExecutionContext in the PyPy interpreter; and store any
@@ -241,9 +241,8 @@
                                    hints={'stm_thread_local': True})
 
     def __init__(self, Class):
-        assert not we_are_translated(), (
-            "You can only have a small number of ThreadLocal() instances"
-            " built during translation.")
+        """NOT_RPYTHON: You can only have a small number of ThreadLocal()
+        instances built during translation."""
         self.Class = Class
         self.threadlocal = lltype.malloc(self.STMTHREADLOCAL, immortal=True)
 
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -119,6 +119,7 @@
     py.test.raises(MyException, rstm.run_all_transactions, DoInOrder())
 
 def test_threadlocal():
+    py.test.skip("disabled")
     # not testing the thread-local factor, but only the general interface
     class Point:
         def __init__(self, x, y):

From noreply at buildbot.pypy.org  Sun Apr 29 14:35:57 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 14:35:57 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: A default value for this attribute is
	really needed.
Message-ID: <20120429123557.E1EF482F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54808:6e50686e9ba9
Date: 2012-04-29 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/6e50686e9ba9/

Log:	A default value for this attribute is really needed.

diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -20,7 +20,7 @@
 
 class Transaction(object):
     _next_transaction = None
-    #_scheduled = False         # for debugging
+    _scheduled = False         # for debugging
 
     def run(self):
         # subclasses can access 'self.retry_counter' here

From noreply at buildbot.pypy.org  Sun Apr 29 14:42:33 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 29 Apr 2012 14:42:33 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Re-enable the methodcache optimization.
 Found another trick
Message-ID: <20120429124233.242D382F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54809:2906eb0e0980
Date: 2012-04-29 14:41 +0200
http://bitbucket.org/pypy/pypy/changeset/2906eb0e0980/

Log:	Re-enable the methodcache optimization. Found another trick that
	should be "kind of good enough for now".

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -366,8 +366,7 @@
     if level in ['2', '3', 'jit']:
         config.objspace.opcodes.suggest(CALL_METHOD=True)
         config.objspace.std.suggest(withrangelist=True)
-        if not config.translation.stm:   # XXX temporary
-            config.objspace.std.suggest(withmethodcache=True)
+        config.objspace.std.suggest(withmethodcache=True)
         config.objspace.std.suggest(withprebuiltchar=True)
         config.objspace.std.suggest(builtinshortcut=True)
         config.objspace.std.suggest(optimized_list_getitem=True)
diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -127,8 +127,17 @@
         ec = self.space.getexecutioncontext()    # create it if needed
         assert len(ec._transaction_pending) == 0
         #
+        if self.space.config.translation.stm:
+            from pypy.objspace.std.typeobject import MethodCache
+            ec._methodcache = MethodCache(self.space)
+        #
         self.space.call_args(self.w_callback, self.args)
         #
+        if self.space.config.translation.stm:
+            # remove the method cache again now, to prevent it from being
+            # promoted to a GLOBAL
+            ec._methodcache = None
+        #
         result = ec._transaction_pending
         ec._transaction_pending = []
         return result
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -409,6 +409,9 @@
         if space.config.translation.stm:
             # with stm, it's important to use one method cache per thread;
             # otherwise, we get all the time spurious transaction conflicts.
+            # For now we have one per transaction: the following field is
+            # set when we start a transaction, and cleared just before
+            # committing.
             cache = space.getexecutioncontext()._methodcache
         else:
             cache = space.fromcache(MethodCache)

From noreply at buildbot.pypy.org  Sun Apr 29 15:29:51 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sun, 29 Apr 2012 15:29:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: Verify that the struct pointer is the
 same as during tracing before optimizing out a quasiimmutable getfield.
 This hopefully fixes the inifite_loop bug of issue 1080.
Message-ID: <20120429132951.C787382F50@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54810:1ba8cd8dfd89
Date: 2012-04-29 15:26 +0200
http://bitbucket.org/pypy/pypy/changeset/1ba8cd8dfd89/

Log:	Verify that the struct pointer is the same as during tracing before
	optimizing out a quasiimmutable getfield. This hopefully fixes the
	inifite_loop bug of issue 1080.

diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -481,7 +481,7 @@
         # already between the tracing and now.  In this case, we are
         # simply ignoring the QUASIIMMUT_FIELD hint and compiling it
         # as a regular getfield.
-        if not qmutdescr.is_still_valid():
+        if not qmutdescr.is_still_valid_for(structvalue.get_key_box()):
             self._remove_guard_not_invalidated = True
             return
         # record as an out-of-line guard
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -6533,9 +6533,9 @@
     def test_quasi_immut_2(self):
         ops = """
         []
-        quasiimmut_field(ConstPtr(myptr), descr=quasiimmutdescr)
+        quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i1 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        i1 = getfield_gc(ConstPtr(quasiptr), descr=quasifielddescr)
         escape(i1)
         jump()
         """
@@ -6585,13 +6585,13 @@
     def test_call_may_force_invalidated_guards_reload(self):
         ops = """
         [i0a, i0b]
-        quasiimmut_field(ConstPtr(myptr), descr=quasiimmutdescr)
+        quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i1 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        i1 = getfield_gc(ConstPtr(quasiptr), descr=quasifielddescr)
         call_may_force(i0b, descr=mayforcevirtdescr)
-        quasiimmut_field(ConstPtr(myptr), descr=quasiimmutdescr)
+        quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i2 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        i2 = getfield_gc(ConstPtr(quasiptr), descr=quasifielddescr)
         i3 = escape(i1)
         i4 = escape(i2)
         jump(i3, i4)
@@ -7833,8 +7833,13 @@
         setfield_gc(p106, p108, descr=nextdescr) # inst_storage
         jump(p106)
         """
-        py.test.raises(InvalidLoop, self.optimize_loop,
-                       ops, expected)
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr2), descr=quasifielddescr)
+        guard_value(p72, -4247) []
+        jump()
+        """
+        self.optimize_loop(ops, expected)
         
 
     def test_issue1080_infinitie_loop_simple(self):
@@ -7852,8 +7857,7 @@
         guard_value(p72, -4247) []
         jump()
         """
-        py.test.raises(InvalidLoop, self.optimize_loop,
-                       ops, expected)
+        self.optimize_loop(ops, expected)
 
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -122,6 +122,7 @@
     quasi.inst_field = -4247
     quasifielddescr = cpu.fielddescrof(QUASI, 'inst_field')
     quasibox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, quasi))
+    quasiptr = quasibox.value
     quasiimmutdescr = QuasiImmutDescr(cpu, quasibox,
                                       quasifielddescr,
                                       cpu.fielddescrof(QUASI, 'mutate_field'))
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -120,8 +120,10 @@
                                     self.fielddescr, self.structbox)
         return fieldbox.constbox()
 
-    def is_still_valid(self):
+    def is_still_valid_for(self, structconst):
         assert self.structbox is not None
+        if not self.structbox.constbox().same_constant(structconst):
+            return False
         cpu = self.cpu
         gcref = self.structbox.getref_base()
         qmut = get_current_qmut_instance(cpu, gcref, self.mutatefielddescr)

From noreply at buildbot.pypy.org  Sun Apr 29 15:29:53 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sun, 29 Apr 2012 15:29:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: hg merge
Message-ID: <20120429132953.4440B82F50@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54811:cc436eb0a04b
Date: 2012-04-29 15:29 +0200
http://bitbucket.org/pypy/pypy/changeset/cc436eb0a04b/

Log:	hg merge

diff --git a/pypy/jit/metainterp/jitexc.py b/pypy/jit/metainterp/jitexc.py
--- a/pypy/jit/metainterp/jitexc.py
+++ b/pypy/jit/metainterp/jitexc.py
@@ -12,7 +12,6 @@
     """
     _go_through_llinterp_uncaught_ = True     # ugh
 
-
 def _get_standard_error(rtyper, Class):
     exdata = rtyper.getexceptiondata()
     clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class)
diff --git a/pypy/jit/metainterp/optimize.py b/pypy/jit/metainterp/optimize.py
--- a/pypy/jit/metainterp/optimize.py
+++ b/pypy/jit/metainterp/optimize.py
@@ -5,3 +5,9 @@
     """Raised when the optimize*.py detect that the loop that
     we are trying to build cannot possibly make sense as a
     long-running loop (e.g. it cannot run 2 complete iterations)."""
+
+    def __init__(self, msg='?'):
+        debug_start("jit-abort")
+        debug_print(msg)
+        debug_stop("jit-abort")
+        self.msg = msg

From noreply at buildbot.pypy.org  Sun Apr 29 23:06:27 2012
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Sun, 29 Apr 2012 23:06:27 +0200 (CEST)
Subject: [pypy-commit] pypy numpypy-issue1137: the magic incantation to call
 the base classes setup_class has been found.
Message-ID: <20120429210627.16D4782F50@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpypy-issue1137
Changeset: r54812:54c62c649f89
Date: 2012-04-29 23:05 +0200
http://bitbucket.org/pypy/pypy/changeset/54c62c649f89/

Log:	the magic incantation to call the base classes setup_class has been
	found.

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -8,13 +8,6 @@
 from pypy.module.micronumpy.interp_numarray import W_NDimArray, shape_agreement
 from pypy.module.micronumpy.test.test_base import BaseNumpyAppTest
 
-# needed only for the verbatim copy of setup_class from test_base
-from pypy.conftest import gettestobjspace
-from pypy.module.micronumpy.interp_dtype import nonnative_byteorder_prefix,\
-     byteorder_prefix
-from pypy.conftest import option
-import sys
-
 class MockDtype(object):
     class itemtype(object):
         @staticmethod
@@ -202,16 +195,7 @@
 
 class AppTestNumArray(BaseNumpyAppTest):
     def setup_class(cls):
-        # this is a verbatim copy from test_base, because calling the base classes
-        # setup_class method doesn't seem to work at all.
-        if option.runappdirect:
-            if '__pypy__' not in sys.builtin_module_names:
-                import numpy
-                sys.modules['numpypy'] = numpy
-                sys.modules['_numpypy'] = numpy
-        cls.space = gettestobjspace(usemodules=['micronumpy'])
-        cls.w_non_native_prefix = cls.space.wrap(nonnative_byteorder_prefix)
-        cls.w_native_prefix = cls.space.wrap(byteorder_prefix)
+        BaseNumpyAppTest.setup_class.im_func(cls)
 
         w_tup = cls.space.appexec([], """():
 

From notifications-noreply at bitbucket.org  Mon Apr 30 02:48:07 2012
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Mon, 30 Apr 2012 00:48:07 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20120430004807.27980.74973@bitbucket13.managed.contegix.com>

You have received a notification from Sameer Regmi.

Hi, I forked pypy. My fork is at https://bitbucket.org/regmi/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Mon Apr 30 09:56:57 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 09:56:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix this test for f6fbfecb93fd.
Message-ID: <20120430075657.1659E82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54813:3aeefa9caa40
Date: 2012-04-30 09:55 +0200
http://bitbucket.org/pypy/pypy/changeset/3aeefa9caa40/

Log:	Fix this test for f6fbfecb93fd.

diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -217,7 +217,8 @@
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
-                    [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
+                    [('debug_print', 'SYS_EXECUTABLE', '??'),
+                     ('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
     
     lltype.free(p, flavor='raw')
 

From noreply at buildbot.pypy.org  Mon Apr 30 10:22:38 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Mon, 30 Apr 2012 10:22:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: Allow OptPure to be disabled by
 enabling OptSimplify if it is and moving this assert to the end of the
 optimization chain
Message-ID: <20120430082238.1C6E082F50@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54814:f2da357373ef
Date: 2012-04-30 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/f2da357373ef/

Log:	Allow OptPure to be disabled by enabling OptSimplify if it is and
	moving this assert to the end of the optimization chain

diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -49,7 +49,8 @@
                 optimizations.append(OptFfiCall())
 
     if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts
-        or 'heap' not in enable_opts or 'unroll' not in enable_opts):
+        or 'heap' not in enable_opts or 'unroll' not in enable_opts
+        or 'pure' not in enable_opts):
         optimizations.append(OptSimplify())
 
     return optimizations, unroll
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -257,8 +257,8 @@
             opnum == rop.COPYSTRCONTENT or       # no effect on GC struct/array
             opnum == rop.COPYUNICODECONTENT):    # no effect on GC struct/array
             return
-        assert opnum != rop.CALL_PURE
         if (opnum == rop.CALL or
+            opnum == rop.CALL_PURE or
             opnum == rop.CALL_MAY_FORCE or
             opnum == rop.CALL_RELEASE_GIL or
             opnum == rop.CALL_ASSEMBLER):
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -525,6 +525,7 @@
 
     @specialize.argtype(0)
     def _emit_operation(self, op):
+        assert op.getopnum() != rop.CALL_PURE
         for i in range(op.numargs()):
             arg = op.getarg(i)
             try:

From noreply at buildbot.pypy.org  Mon Apr 30 10:22:39 2012
From: noreply at buildbot.pypy.org (hakanardo)
Date: Mon, 30 Apr 2012 10:22:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20120430082239.5655282F50@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r54815:87a5aca82fa1
Date: 2012-04-30 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/87a5aca82fa1/

Log:	merge

diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -217,7 +217,8 @@
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
-                    [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
+                    [('debug_print', 'SYS_EXECUTABLE', '??'),
+                     ('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
     
     lltype.free(p, flavor='raw')
 

From noreply at buildbot.pypy.org  Mon Apr 30 11:57:20 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 11:57:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Improve rlib.debug;
	transposed from stm-gc.
Message-ID: <20120430095720.B940582F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54816:765ff6ef6f80
Date: 2012-04-30 10:11 +0200
http://bitbucket.org/pypy/pypy/changeset/765ff6ef6f80/

Log:	Improve rlib.debug; transposed from stm-gc.

diff --git a/pypy/rlib/debug.py b/pypy/rlib/debug.py
--- a/pypy/rlib/debug.py
+++ b/pypy/rlib/debug.py
@@ -1,10 +1,12 @@
 import sys, time
 from pypy.rpython.extregistry import ExtRegistryEntry
+from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.rarithmetic import is_valid_int
 
 
 def ll_assert(x, msg):
     """After translation to C, this becomes an RPyAssert."""
+    assert type(x) is bool, "bad type! got %r" % (type(x),)
     assert x, msg
 
 class Entry(ExtRegistryEntry):
@@ -21,8 +23,13 @@
         hop.exception_cannot_occur()
         hop.genop('debug_assert', vlist)
 
+class FatalError(Exception):
+    pass
+
 def fatalerror(msg):
     # print the RPython traceback and abort with a fatal error
+    if not we_are_translated():
+        raise FatalError(msg)
     from pypy.rpython.lltypesystem import lltype
     from pypy.rpython.lltypesystem.lloperation import llop
     llop.debug_print_traceback(lltype.Void)
@@ -33,6 +40,8 @@
 
 def fatalerror_notb(msg):
     # a variant of fatalerror() that doesn't print the RPython traceback
+    if not we_are_translated():
+        raise FatalError(msg)
     from pypy.rpython.lltypesystem import lltype
     from pypy.rpython.lltypesystem.lloperation import llop
     llop.debug_fatalerror(lltype.Void, msg)
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -916,7 +916,7 @@
         ll_assert(not self.is_in_nursery(obj),
                   "object in nursery after collection")
         # similarily, all objects should have this flag:
-        ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS,
+        ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
                   "missing GCFLAG_TRACK_YOUNG_PTRS")
         # the GCFLAG_VISITED should not be set between collections
         ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -640,7 +640,7 @@
         between collections."""
         tid = self.header(obj).tid
         if tid & GCFLAG_EXTERNAL:
-            ll_assert(tid & GCFLAG_FORWARDED, "bug: external+!forwarded")
+            ll_assert(tid & GCFLAG_FORWARDED != 0, "bug: external+!forwarded")
             ll_assert(not (self.tospace <= obj < self.free),
                       "external flag but object inside the semispaces")
         else:
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -8,7 +8,6 @@
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib import rgc
-from pypy.rlib.debug import ll_assert
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc

From noreply at buildbot.pypy.org  Mon Apr 30 11:57:22 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 11:57:22 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: hg merge default
Message-ID: <20120430095722.B15E182F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54817:f701cef21be4
Date: 2012-04-30 10:17 +0200
http://bitbucket.org/pypy/pypy/changeset/f701cef21be4/

Log:	hg merge default

diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -217,7 +217,8 @@
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
-                    [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
+                    [('debug_print', 'SYS_EXECUTABLE', '??'),
+                     ('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
     
     lltype.free(p, flavor='raw')
 
diff --git a/pypy/jit/metainterp/jitexc.py b/pypy/jit/metainterp/jitexc.py
--- a/pypy/jit/metainterp/jitexc.py
+++ b/pypy/jit/metainterp/jitexc.py
@@ -12,7 +12,6 @@
     """
     _go_through_llinterp_uncaught_ = True     # ugh
 
-
 def _get_standard_error(rtyper, Class):
     exdata = rtyper.getexceptiondata()
     clsdef = rtyper.annotator.bookkeeper.getuniqueclassdef(Class)
diff --git a/pypy/jit/metainterp/optimize.py b/pypy/jit/metainterp/optimize.py
--- a/pypy/jit/metainterp/optimize.py
+++ b/pypy/jit/metainterp/optimize.py
@@ -5,3 +5,9 @@
     """Raised when the optimize*.py detect that the loop that
     we are trying to build cannot possibly make sense as a
     long-running loop (e.g. it cannot run 2 complete iterations)."""
+
+    def __init__(self, msg='?'):
+        debug_start("jit-abort")
+        debug_print(msg)
+        debug_stop("jit-abort")
+        self.msg = msg
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -481,7 +481,7 @@
         # already between the tracing and now.  In this case, we are
         # simply ignoring the QUASIIMMUT_FIELD hint and compiling it
         # as a regular getfield.
-        if not qmutdescr.is_still_valid():
+        if not qmutdescr.is_still_valid_for(structvalue.get_key_box()):
             self._remove_guard_not_invalidated = True
             return
         # record as an out-of-line guard
diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -191,10 +191,13 @@
         # GUARD_OVERFLOW, then the loop is invalid.
         lastop = self.last_emitted_operation
         if lastop is None:
-            raise InvalidLoop
+            raise InvalidLoop('An INT_xxx_OVF was proven not to overflow but' +
+                              'guarded with GUARD_OVERFLOW')
         opnum = lastop.getopnum()
         if opnum not in (rop.INT_ADD_OVF, rop.INT_SUB_OVF, rop.INT_MUL_OVF):
-            raise InvalidLoop
+            raise InvalidLoop('An INT_xxx_OVF was proven not to overflow but' +
+                              'guarded with GUARD_OVERFLOW')
+                             
         self.emit_operation(op)
 
     def optimize_INT_ADD_OVF(self, op):
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -208,7 +208,8 @@
             box = value.box
             assert isinstance(box, Const)
             if not box.same_constant(constbox):
-                raise InvalidLoop
+                raise InvalidLoop('A GURAD_{VALUE,TRUE,FALSE} was proven to' +
+                                  'always fail')
             return
         if emit_operation:
             self.emit_operation(op)
@@ -220,7 +221,7 @@
         if value.is_null():
             return
         elif value.is_nonnull():
-            raise InvalidLoop
+            raise InvalidLoop('A GUARD_ISNULL was proven to always fail')
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
@@ -229,7 +230,7 @@
         if value.is_nonnull():
             return
         elif value.is_null():
-            raise InvalidLoop
+            raise InvalidLoop('A GUARD_NONNULL was proven to always fail')
         self.emit_operation(op)
         value.make_nonnull(op)
 
@@ -278,7 +279,7 @@
         if realclassbox is not None:
             if realclassbox.same_constant(expectedclassbox):
                 return
-            raise InvalidLoop
+            raise InvalidLoop('A GUARD_CLASS was proven to always fail')
         if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
@@ -301,7 +302,8 @@
     def optimize_GUARD_NONNULL_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
-            raise InvalidLoop
+            raise InvalidLoop('A GUARD_NONNULL_CLASS was proven to always ' +
+                              'fail')
         self.optimize_GUARD_CLASS(op)
 
     def optimize_CALL_LOOPINVARIANT(self, op):
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -5090,7 +5090,6 @@
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
 
-
 ##class TestOOtype(BaseTestOptimizeBasic, OOtypeMixin):
 
 ##    def test_instanceof(self):
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -6533,9 +6533,9 @@
     def test_quasi_immut_2(self):
         ops = """
         []
-        quasiimmut_field(ConstPtr(myptr), descr=quasiimmutdescr)
+        quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i1 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        i1 = getfield_gc(ConstPtr(quasiptr), descr=quasifielddescr)
         escape(i1)
         jump()
         """
@@ -6585,13 +6585,13 @@
     def test_call_may_force_invalidated_guards_reload(self):
         ops = """
         [i0a, i0b]
-        quasiimmut_field(ConstPtr(myptr), descr=quasiimmutdescr)
+        quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i1 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        i1 = getfield_gc(ConstPtr(quasiptr), descr=quasifielddescr)
         call_may_force(i0b, descr=mayforcevirtdescr)
-        quasiimmut_field(ConstPtr(myptr), descr=quasiimmutdescr)
+        quasiimmut_field(ConstPtr(quasiptr), descr=quasiimmutdescr)
         guard_not_invalidated() []
-        i2 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        i2 = getfield_gc(ConstPtr(quasiptr), descr=quasifielddescr)
         i3 = escape(i1)
         i4 = escape(i2)
         jump(i3, i4)
@@ -7813,6 +7813,52 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_issue1080_infinitie_loop_virtual(self):
+        ops = """
+        [p10]
+        p52 = getfield_gc(p10, descr=nextdescr) # inst_storage
+        p54 = getarrayitem_gc(p52, 0, descr=arraydescr)
+        p69 = getfield_gc_pure(p54, descr=otherdescr) # inst_w_function
+
+        quasiimmut_field(p69, descr=quasiimmutdescr)
+        guard_not_invalidated() []
+        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
+        guard_value(p71, -4247) []
+
+        p106 = new_with_vtable(ConstClass(node_vtable))
+        p108 = new_array(3, descr=arraydescr)
+        p110 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p110, ConstPtr(myptr2), descr=otherdescr) # inst_w_function
+        setarrayitem_gc(p108, 0, p110, descr=arraydescr)
+        setfield_gc(p106, p108, descr=nextdescr) # inst_storage
+        jump(p106)
+        """
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr2), descr=quasifielddescr)
+        guard_value(p72, -4247) []
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+        
+
+    def test_issue1080_infinitie_loop_simple(self):
+        ops = """
+        [p69]
+        quasiimmut_field(p69, descr=quasiimmutdescr)
+        guard_not_invalidated() []
+        p71 = getfield_gc(p69, descr=quasifielddescr) # inst_code
+        guard_value(p71, -4247) []
+        jump(ConstPtr(myptr))
+        """
+        expected = """
+        []
+        p72 = getfield_gc(ConstPtr(myptr), descr=quasifielddescr)
+        guard_value(p72, -4247) []
+        jump()
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -122,6 +122,7 @@
     quasi.inst_field = -4247
     quasifielddescr = cpu.fielddescrof(QUASI, 'inst_field')
     quasibox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, quasi))
+    quasiptr = quasibox.value
     quasiimmutdescr = QuasiImmutDescr(cpu, quasibox,
                                       quasifielddescr,
                                       cpu.fielddescrof(QUASI, 'mutate_field'))
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -315,7 +315,10 @@
         try:
             jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         except BadVirtualState:
-            raise InvalidLoop
+            raise InvalidLoop('The state of the optimizer at the end of ' +
+                              'peeled loop is inconsistent with the ' +
+                              'VirtualState at the begining of the peeled ' +
+                              'loop')
         jumpop.initarglist(jumpargs)
 
         # Inline the short preamble at the end of the loop
@@ -325,7 +328,11 @@
         for i in range(len(short_inputargs)):
             if short_inputargs[i] in args:
                 if args[short_inputargs[i]] != jmp_to_short_args[i]:
-                    raise InvalidLoop
+                    raise InvalidLoop('The short preamble wants the ' +
+                                      'same box passed to multiple of its ' +
+                                      'inputargs, but the jump at the ' +
+                                      'end of this bridge does not do that.')
+                                      
             args[short_inputargs[i]] = jmp_to_short_args[i]
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         for op in self.short[1:]:
@@ -378,7 +385,10 @@
             #final_virtual_state.debug_print("Bad virtual state at end of loop, ",
             #                                bad)
             #debug_stop('jit-log-virtualstate')
-            raise InvalidLoop
+            raise InvalidLoop('The virtual state at the end of the peeled ' +
+                              'loop is not compatible with the virtual ' +
+                              'state at the start of the loop which makes ' +
+                              'it impossible to close the loop')
             
         #debug_stop('jit-log-virtualstate')
 
@@ -526,8 +536,8 @@
         args = jumpop.getarglist()
         modifier = VirtualStateAdder(self.optimizer)
         virtual_state = modifier.get_virtual_state(args)
-        #debug_start('jit-log-virtualstate')
-        #virtual_state.debug_print("Looking for ")
+        debug_start('jit-log-virtualstate')
+        virtual_state.debug_print("Looking for ")
 
         for target in cell_token.target_tokens:
             if not target.virtual_state:
@@ -536,10 +546,10 @@
             extra_guards = []
 
             bad = {}
-            #debugmsg = 'Did not match '
+            debugmsg = 'Did not match '
             if target.virtual_state.generalization_of(virtual_state, bad):
                 ok = True
-                #debugmsg = 'Matched '
+                debugmsg = 'Matched '
             else:
                 try:
                     cpu = self.optimizer.cpu
@@ -548,13 +558,13 @@
                                                          extra_guards)
 
                     ok = True
-                    #debugmsg = 'Guarded to match '
+                    debugmsg = 'Guarded to match '
                 except InvalidLoop:
                     pass
-            #target.virtual_state.debug_print(debugmsg, bad)
+            target.virtual_state.debug_print(debugmsg, bad)
 
             if ok:
-                #debug_stop('jit-log-virtualstate')
+                debug_stop('jit-log-virtualstate')
 
                 values = [self.getvalue(arg)
                           for arg in jumpop.getarglist()]
@@ -581,7 +591,7 @@
                     jumpop.setdescr(cell_token.target_tokens[0])
                     self.optimizer.send_extra_operation(jumpop)
                 return True
-        #debug_stop('jit-log-virtualstate')
+        debug_stop('jit-log-virtualstate')
         return False
 
 class ValueImporter(object):
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -27,11 +27,15 @@
         if self.generalization_of(other, renum, {}):
             return
         if renum[self.position] != other.position:
-            raise InvalidLoop
+            raise InvalidLoop('The numbering of the virtual states does not ' +
+                              'match. This means that two virtual fields ' +
+                              'have been set to the same Box in one of the ' +
+                              'virtual states but not in the other.')
         self._generate_guards(other, box, cpu, extra_guards)
 
     def _generate_guards(self, other, box, cpu, extra_guards):
-        raise InvalidLoop
+        raise InvalidLoop('Generating guards for making the VirtualStates ' +
+                          'at hand match have not been implemented')
 
     def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
@@ -346,10 +350,12 @@
 
     def _generate_guards(self, other, box, cpu, extra_guards):
         if not isinstance(other, NotVirtualStateInfo):
-            raise InvalidLoop
+            raise InvalidLoop('The VirtualStates does not match as a ' +
+                              'virtual appears where a pointer is needed ' +
+                              'and it is too late to force it.')
 
         if self.lenbound or other.lenbound:
-            raise InvalidLoop
+            raise InvalidLoop('The array length bounds does not match.')
 
         if self.level == LEVEL_KNOWNCLASS and \
            box.nonnull() and \
@@ -400,7 +406,8 @@
             return
 
         # Remaining cases are probably not interesting
-        raise InvalidLoop
+        raise InvalidLoop('Generating guards for making the VirtualStates ' +
+                          'at hand match have not been implemented')
         if self.level == LEVEL_CONSTANT:
             import pdb; pdb.set_trace()
             raise NotImplementedError
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -120,8 +120,10 @@
                                     self.fielddescr, self.structbox)
         return fieldbox.constbox()
 
-    def is_still_valid(self):
+    def is_still_valid_for(self, structconst):
         assert self.structbox is not None
+        if not self.structbox.constbox().same_constant(structconst):
+            return False
         cpu = self.cpu
         gcref = self.structbox.getref_base()
         qmut = get_current_qmut_instance(cpu, gcref, self.mutatefielddescr)
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -8,7 +8,7 @@
 from pypy.jit.metainterp.quasiimmut import get_current_qmut_instance
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.jit.codewriter.policy import StopAtXPolicy
-from pypy.rlib.jit import JitDriver, dont_look_inside, unroll_safe
+from pypy.rlib.jit import JitDriver, dont_look_inside, unroll_safe, promote
 
 
 def test_get_current_qmut_instance():
@@ -506,6 +506,27 @@
             "guard_not_invalidated": 2
         })
 
+    def test_issue1080(self):
+        myjitdriver = JitDriver(greens=[], reds=["n", "sa", "a"])
+        class Foo(object):
+            _immutable_fields_ = ["x?"]
+            def __init__(self, x):
+                self.x = x
+        one, two = Foo(1), Foo(2)
+        def main(n):
+            sa = 0
+            a = one
+            while n:
+                myjitdriver.jit_merge_point(n=n, sa=sa, a=a)
+                sa += a.x
+                if a.x == 1:
+                    a = two
+                elif a.x == 2:
+                    a = one                    
+                n -= 1
+            return sa
+        res = self.meta_interp(main, [10])
+        assert res == main(10)
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -157,13 +157,15 @@
         raises(IOError, _multiprocessing.Connection, -15)
 
     def test_byte_order(self):
+        import socket
+        if not 'fromfd' in dir(socket):
+            skip('No fromfd in socket')
         # The exact format of net strings (length in network byte
         # order) is important for interoperation with others
         # implementations.
         rhandle, whandle = self.make_pair()
         whandle.send_bytes("abc")
         whandle.send_bytes("defg")
-        import socket
         sock = socket.fromfd(rhandle.fileno(),
                              socket.AF_INET, socket.SOCK_STREAM)
         data1 = sock.recv(7)
diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py
--- a/pypy/module/_winreg/test/test_winreg.py
+++ b/pypy/module/_winreg/test/test_winreg.py
@@ -198,7 +198,10 @@
         import nt
         r = ExpandEnvironmentStrings(u"%windir%\\test")
         assert isinstance(r, unicode)
-        assert r == nt.environ["WINDIR"] + "\\test"
+        if 'WINDIR' in nt.environ.keys():
+            assert r == nt.environ["WINDIR"] + "\\test"
+        else:
+            assert r == nt.environ["windir"] + "\\test"
 
     def test_long_key(self):
         from _winreg import (
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -185,6 +185,33 @@
 
         assert dtype("float") is dtype(float)
 
+    def test_index_int8(self):
+        from _numpypy import array, int8
+
+        a = array(range(10), dtype=int8)
+        b = array([0] * 10, dtype=int8)
+        for idx in b: a[idx] += 1
+
+    def test_index_int16(self):
+        from _numpypy import array, int16
+
+        a = array(range(10), dtype=int16)
+        b = array([0] * 10, dtype=int16)
+        for idx in b: a[idx] += 1
+
+    def test_index_int32(self):
+        from _numpypy import array, int32
+
+        a = array(range(10), dtype=int32)
+        b = array([0] * 10, dtype=int32)
+        for idx in b: a[idx] += 1
+
+    def test_index_int64(self):
+        from _numpypy import array, int64
+
+        a = array(range(10), dtype=int64)
+        b = array([0] * 10, dtype=int64)
+        for idx in b: a[idx] += 1
 
 class AppTestTypes(BaseNumpyAppTest):    
     def test_abstract_types(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -54,7 +54,8 @@
             cmdline += ['--jit', ','.join(jitcmdline)]
         cmdline.append(str(self.filepath))
         #
-        env={'PYPYLOG': self.log_string + ':' + str(logfile)}
+        env = os.environ.copy()
+        env['PYPYLOG'] = self.log_string + ':' + str(logfile)
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -572,7 +572,7 @@
                 if i < length and format[i] == '#':
                     # not documented by python
                     i += 1
-                if i >= length or format[i] not in "aAbBcdfHIjmMpSUwWxXyYzZ%":
+                if i >= length or format[i] not in "aAbBcdHIjmMpSUwWxXyYzZ%":
                     raise OperationError(space.w_ValueError,
                                          space.wrap("invalid format string"))
             i += 1
diff --git a/pypy/rlib/test/test_rposix.py b/pypy/rlib/test/test_rposix.py
--- a/pypy/rlib/test/test_rposix.py
+++ b/pypy/rlib/test/test_rposix.py
@@ -133,6 +133,8 @@
                 pass
 
     def test_validate_fd(self):
+        if os.name != 'nt':
+            skip('relevant for windows only')
         assert rposix._validate_fd(0) == 1
         fid = open(str(udir.join('validate_test.txt')), 'w')
         fd = fid.fileno()
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -917,7 +917,7 @@
         ll_assert(not self.is_in_nursery(obj),
                   "object in nursery after collection")
         # similarily, all objects should have this flag:
-        ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS,
+        ll_assert(self.header(obj).tid & GCFLAG_TRACK_YOUNG_PTRS != 0,
                   "missing GCFLAG_TRACK_YOUNG_PTRS")
         # the GCFLAG_VISITED should not be set between collections
         ll_assert(self.header(obj).tid & GCFLAG_VISITED == 0,
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -640,7 +640,7 @@
         between collections."""
         tid = self.header(obj).tid
         if tid & GCFLAG_EXTERNAL:
-            ll_assert(tid & GCFLAG_FORWARDED, "bug: external+!forwarded")
+            ll_assert(tid & GCFLAG_FORWARDED != 0, "bug: external+!forwarded")
             ll_assert(not (self.tospace <= obj < self.free),
                       "external flag but object inside the semispaces")
         else:
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -8,7 +8,6 @@
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib import rgc
-from pypy.rlib.debug import ll_assert
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc

From noreply at buildbot.pypy.org  Mon Apr 30 11:57:23 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 11:57:23 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Reverse the order to avoid unexpected
 effects if someone calls
Message-ID: <20120430095723.EB7EC82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54818:948dfd155f01
Date: 2012-04-30 10:52 +0200
http://bitbucket.org/pypy/pypy/changeset/948dfd155f01/

Log:	Reverse the order to avoid unexpected effects if someone calls the
	constructor without keyword args.

diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -22,7 +22,7 @@
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
-                 multithread=False, translated_to_c=True):
+                 translated_to_c=True, multithread=False):
         self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
         self.AddressStack = get_address_stack(chunk_size, multithread)
         self.AddressDeque = get_address_deque(chunk_size, multithread)

From noreply at buildbot.pypy.org  Mon Apr 30 11:57:25 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 11:57:25 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add a direct low-level operation
	stm_is_enabled().
Message-ID: <20120430095725.60DAB82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54819:c0274354141e
Date: 2012-04-30 11:14 +0200
http://bitbucket.org/pypy/pypy/changeset/c0274354141e/

Log:	Add a direct low-level operation stm_is_enabled().

diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -399,6 +399,7 @@
     # direct_calls and maybe several casts, but it looks less heavy-weight
     # to keep them as operations until the genc stage)
 
+    'stm_is_enabled':         LLOp(canrun=True),  # sideeffects: test_lltyped.py
     'stm_getfield':           LLOp(sideeffects=False, canrun=True),
     'stm_getarrayitem':       LLOp(sideeffects=False, canrun=True),
     'stm_getinteriorfield':   LLOp(sideeffects=False, canrun=True),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -623,16 +623,8 @@
     from pypy.rlib.rtimer import read_timestamp
     return read_timestamp()
 
-def op_stm_descriptor_init():
-    # for direct testing only
-    xxx
-    from pypy.translator.stm import stmgcintf
-    stmgcintf.StmOperations.set_tls(llmemory.NULL, 0)
-
-def op_stm_descriptor_done():
-    xxx
-    from pypy.translator.stm import stmgcintf
-    stmgcintf.StmOperations.del_tls()
+def op_stm_is_enabled():
+    return False
 
 def op_stm_start_transaction():
     pass
diff --git a/pypy/translator/c/src/mem.h b/pypy/translator/c/src/mem.h
--- a/pypy/translator/c/src/mem.h
+++ b/pypy/translator/c/src/mem.h
@@ -259,3 +259,5 @@
 #define OP_GC_GET_RPY_TYPE_INDEX(x, r)   r = -1
 #define OP_GC_IS_RPY_INSTANCE(x, r)      r = 0
 #define OP_GC_DUMP_RPY_HEAP(fd, r)       r = 0
+
+#define OP_STM_IS_ENABLED(r)             r = 0
diff --git a/pypy/translator/c/test/test_lltyped.py b/pypy/translator/c/test/test_lltyped.py
--- a/pypy/translator/c/test/test_lltyped.py
+++ b/pypy/translator/c/test/test_lltyped.py
@@ -903,3 +903,16 @@
             return rstring_to_float(s)
         fn = self.getcompiled(llf, [int])
         assert fn(0) == 42.3
+
+    def test_stm_is_enabled(self):
+        from pypy.rpython.lltypesystem import lltype
+        from pypy.rpython.lltypesystem.lloperation import llop
+        # note that right now the situation of lloperation is a bit messy.
+        # stm_is_enabled is not "sideeffects=False" because otherwise
+        # it is constant-folded away by backendopt/constfold.py.
+        assert llop.stm_is_enabled.sideeffects == True
+        #
+        def llf(i):
+            return int(llop.stm_is_enabled(lltype.Bool))
+        fn = self.getcompiled(llf, [int])
+        assert fn(0) == 0
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import rffi
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib import rstm
 from pypy.rlib.debug import debug_print, ll_assert
 
@@ -95,6 +96,7 @@
 
 def entry_point(argv):
     print "hello world"
+    assert llop.stm_is_enabled(lltype.Bool)
     if len(argv) > 1:
         glob.NUM_THREADS = int(argv[1])
         if len(argv) > 2:
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -162,6 +162,11 @@
             self.count_write_barrier += 1
         newoperations.append(op)
 
+    def stt_stm_is_enabled(self, newoperations, op):
+        c_True = Constant(True, lltype.Bool)
+        op = SpaceOperation('same_as', [c_True], op.result)
+        newoperations.append(op)
+
     def stt_malloc(self, newoperations, op):
         flags = op.args[1].value
         if flags['flavor'] == 'gc':

From noreply at buildbot.pypy.org  Mon Apr 30 11:57:26 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 11:57:26 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix rgc to support both STM and non-STM
	modes.
Message-ID: <20120430095726.9A78782F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54820:5d29b8c930b7
Date: 2012-04-30 11:18 +0200
http://bitbucket.org/pypy/pypy/changeset/5d29b8c930b7/

Log:	Fix rgc to support both STM and non-STM modes.

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -154,21 +154,17 @@
             assert (source_start + length <= dest_start or
                     dest_start + length <= source_start)
 
-    # XXX --- custom version for STM ---
-    for i in range(length):
-        dest[i + dest_start] = source[i + source_start]
-    return
-    # XXX --- the rest is never translated so far ---
-
     TP = lltype.typeOf(source).TO
     assert TP == lltype.typeOf(dest).TO
     if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc':
         # perform a write barrier that copies necessary flags from
         # source to dest
-        if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest,
+        if llop.stm_is_enabled(lltype.Bool) or (
+           not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest,
                                                 source_start, dest_start,
-                                                length):
-            # if the write barrier is not supported, copy by hand
+                                                length)):
+            # if the write barrier is not supported, or if STM is
+            # enabled, copy by hand
             i = 0
             while i < length:
                 dest[i + dest_start] = source[i + source_start]
@@ -190,10 +186,8 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
-    # XXX --- custom version for STM ---
-    # the next two lines are disabled:
-##    if llop.shrink_array(lltype.Bool, p, smallerlength):
-##        return p    # done by the GC
+    if llop.shrink_array(lltype.Bool, p, smallerlength):
+        return p    # done by the GC
 
     # XXX we assume for now that the type of p is GcStruct containing a
     # variable array, with no further pointers anywhere, and exactly one
@@ -206,24 +200,24 @@
     field = getattr(p, TP._names[0])
     setattr(newp, TP._names[0], field)
 
-    # XXX --- custom version for STM ---
-    # the old version first:
-##    ARRAY = getattr(TP, TP._arrayfld)
-##    offset = (llmemory.offsetof(TP, TP._arrayfld) +
-##              llmemory.itemoffsetof(ARRAY, 0))
-##    source_addr = llmemory.cast_ptr_to_adr(p)    + offset
-##    dest_addr   = llmemory.cast_ptr_to_adr(newp) + offset
-##    llmemory.raw_memcopy(source_addr, dest_addr,
-##                         llmemory.sizeof(ARRAY.OF) * smallerlength)
+    if llop.stm_is_enabled(lltype.Bool):
+        # do the copying element by element
+        i = 0
+        while i < smallerlength:
+            newp.chars[i] = p.chars[i]
+            i += 1
+        return newp
 
-##    keepalive_until_here(p)
-##    keepalive_until_here(newp)
+    ARRAY = getattr(TP, TP._arrayfld)
+    offset = (llmemory.offsetof(TP, TP._arrayfld) +
+              llmemory.itemoffsetof(ARRAY, 0))
+    source_addr = llmemory.cast_ptr_to_adr(p)    + offset
+    dest_addr   = llmemory.cast_ptr_to_adr(newp) + offset
+    llmemory.raw_memcopy(source_addr, dest_addr,
+                         llmemory.sizeof(ARRAY.OF) * smallerlength)
 
-    i = 0
-    while i < smallerlength:
-        newp.chars[i] = p.chars[i]
-        i += 1
-
+    keepalive_until_here(p)
+    keepalive_until_here(newp)
     return newp
 ll_shrink_array._annspecialcase_ = 'specialize:ll'
 ll_shrink_array._jit_look_inside_ = False

From noreply at buildbot.pypy.org  Mon Apr 30 11:57:27 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 11:57:27 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fixes
Message-ID: <20120430095727.E0FD082F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54821:6ed85ab7e802
Date: 2012-04-30 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/6ed85ab7e802/

Log:	Fixes

diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -946,6 +946,14 @@
     def op_stack_current(self):
         return 0
 
+    def _stm_not_implemented(self, *args):
+        raise NotImplementedError
+    op_stm_writebarrier = _stm_not_implemented
+    op_stm_normalize_global = _stm_not_implemented
+    op_stm_become_inevitable = _stm_not_implemented
+    op_stm_thread_starting = _stm_not_implemented
+    op_stm_thread_stopping = _stm_not_implemented
+
     # operations on pyobjects!
     for opname in lloperation.opimpls.keys():
         exec py.code.Source("""
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -716,10 +716,8 @@
         string is already nonmovable.  Must be followed by a
         free_nonmovingbuffer call.
         """
-        # XXX --- custom version for STM ---
-        # disabled the "else" part
-        ##if rgc.can_move(data):
-        if 1:
+        from pypy.rpython.lltypesystem.lloperation import llop
+        if llop.stm_is_enabled(lltype.Bool) or rgc.can_move(data):
             count = len(data)
             buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
             for i in range(count):
@@ -743,14 +741,15 @@
         # if 'buf' points inside 'data'.  This is only possible if we
         # followed the 2nd case in get_nonmovingbuffer(); in the first case,
         # 'buf' points to its own raw-malloced memory.
-
-        # XXX --- custom version for STM ---
-##        data = llstrtype(data)
-##        data_start = cast_ptr_to_adr(data) + \
-##            offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
-##        followed_2nd_path = (buf == cast(TYPEP, data_start))
-##        keepalive_until_here(data)
-        followed_2nd_path = False
+        from pypy.rpython.lltypesystem.lloperation import llop
+        if llop.stm_is_enabled(lltype.Bool):
+            followed_2nd_path = False
+        else:
+            data = llstrtype(data)
+            data_start = cast_ptr_to_adr(data) + \
+                offsetof(STRTYPE, 'chars') + itemoffsetof(STRTYPE.chars, 0)
+            followed_2nd_path = (buf == cast(TYPEP, data_start))
+            keepalive_until_here(data)
         if not followed_2nd_path:
             lltype.free(buf, flavor='raw')
     free_nonmovingbuffer._annenforceargs_ = [strtype, None]
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -73,17 +73,22 @@
         assert srcstart >= 0
         assert dststart >= 0
         assert length >= 0
-        # XXX --- custom version for STM ---
-        # the old version first:
-##        src = llmemory.cast_ptr_to_adr(src) + _str_ofs(srcstart)
-##        dst = llmemory.cast_ptr_to_adr(dst) + _str_ofs(dststart)
-##        llmemory.raw_memcopy(src, dst, llmemory.sizeof(CHAR_TP) * length)
-##        keepalive_until_here(src)
-##        keepalive_until_here(dst)
-        i = 0
-        while i < length:
-            dst.chars[dststart + i] = src.chars[srcstart + i]
-            i += 1
+        #
+        # STM requires the slow character-by-character version,
+        # which at least works without forcing the transaction
+        # to become inevitable
+        if llop.stm_is_enabled(Bool):
+            i = 0
+            while i < length:
+                dst.chars[dststart + i] = src.chars[srcstart + i]
+                i += 1
+            return
+        #
+        src = llmemory.cast_ptr_to_adr(src) + _str_ofs(srcstart)
+        dst = llmemory.cast_ptr_to_adr(dst) + _str_ofs(dststart)
+        llmemory.raw_memcopy(src, dst, llmemory.sizeof(CHAR_TP) * length)
+        keepalive_until_here(src)
+        keepalive_until_here(dst)
     copy_string_contents._always_inline_ = True
     return func_with_new_name(copy_string_contents, 'copy_%s_contents' % name)
 
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -136,7 +136,6 @@
 
     def __init__(self, translator):
         from pypy.rpython.memory.gc.base import choose_gc_from_config
-        from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP
 
         super(FrameworkGCTransformer, self).__init__(translator, inline=True)
         if hasattr(self, 'GC_PARAMS'):
@@ -343,6 +342,7 @@
                 annmodel.s_None)
 
         if hasattr(GCClass, 'heap_stats'):
+            from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP
             self.heap_stats_ptr = getfn(GCClass.heap_stats.im_func,
                     [s_gc], annmodel.SomePtr(lltype.Ptr(ARRAY_TYPEID_MAP)),
                     minimal_transform=False)
diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py
--- a/pypy/rpython/memory/gctransform/shadowstack.py
+++ b/pypy/rpython/memory/gctransform/shadowstack.py
@@ -28,7 +28,7 @@
         self.decr_stack = decr_stack
 
         root_iterator = get_root_iterator(gctransformer)
-        def walk_stack_root(callback, arg, start, end):
+        def walk_stack_root(callback, start, end):
             root_iterator.setcontext(NonConstant(llmemory.NULL))
             gc = self.gc
             addr = end
@@ -36,7 +36,7 @@
                 addr = root_iterator.nextleft(gc, start, addr)
                 if addr == llmemory.NULL:
                     return
-                callback(arg, addr)
+                callback(gc, addr)
         self.rootstackhook = walk_stack_root
 
         self.shadow_stack_pool = ShadowStackPool(gcdata)
@@ -56,9 +56,9 @@
         self.shadow_stack_pool.initial_setup()
         BaseRootWalker.setup_root_walker(self)
 
-    def walk_stack_roots(self, collect_stack_root, arg):
+    def walk_stack_roots(self, collect_stack_root):
         gcdata = self.gcdata
-        self.rootstackhook(collect_stack_root, arg,
+        self.rootstackhook(collect_stack_root,
                            gcdata.root_stack_base, gcdata.root_stack_top)
 
     def need_thread_support(self, gctransformer, getfn):

From noreply at buildbot.pypy.org  Mon Apr 30 11:57:29 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 11:57:29 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix the test.
Message-ID: <20120430095729.3AFBB82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54822:89332f512a7c
Date: 2012-04-30 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/89332f512a7c/

Log:	Fix the test.

diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -37,8 +37,6 @@
         t.viewcg()
     return t
 
-ARGS = lltype.FixedSizeArray(lltype.Signed, 3)
-
 class GCTest(object):
     gcpolicy = None
     GC_CAN_MOVE = False
@@ -83,21 +81,22 @@
             cleanups.append(cleanup)
 
         def entrypoint(args):
-            num = args[0]
+            num = name_to_func[args[1]]
             func = funcs0[num]
             if func:
                 res = func()
             else:
                 func = funcs2[num]
-                res = func(args[1], args[2])
+                res = func(int(args[2]), int(args[3]))
             cleanup = cleanups[num]
             if cleanup:
                 cleanup()
             return res
 
         from pypy.translator.c.genc import CStandaloneBuilder
+        from pypy.annotation.listdef import s_list_of_strings
 
-        s_args = annmodel.SomePtr(lltype.Ptr(ARGS))
+        s_args = s_list_of_strings
         t = rtype(entrypoint, [s_args], gcname=cls.gcname,
                   taggedpointers=cls.taggedpointers)
 
@@ -120,7 +119,6 @@
 
     def runner(self, name, statistics=False, transformer=False):
         db = self.db
-        name_to_func = self.name_to_func
         entrygraph = self.entrygraph
         from pypy.rpython.llinterp import LLInterpreter
 
@@ -138,11 +136,19 @@
         # setup => resets the gc
         llinterp.eval_graph(setupgraph, [])
         def run(args):
-            ll_args = lltype.malloc(ARGS, immortal=True)
-            ll_args[0] = name_to_func[name]
+            ARGV = entrygraph.startblock.inputargs[1].concretetype
+            argc = 2 + len(args)
+            ll_args = lltype.malloc(ARGV.TO, argc, flavor='raw')
+            ll_args[0] = rffi.str2charp('')
+            ll_args[1] = rffi.str2charp(name)
             for i in range(len(args)):
-                ll_args[1+i] = args[i]
-            res = llinterp.eval_graph(entrygraph, [ll_args])
+                ll_args[2+i] = rffi.str2charp(str(args[i]))
+            res = llinterp.eval_graph(entrygraph, [argc, ll_args])
+            for i in range(len(args)):
+                rffi.free_charp(ll_args[2+i])
+            rffi.free_charp(ll_args[1])
+            rffi.free_charp(ll_args[0])
+            lltype.free(ll_args, flavor='raw')
             return res
 
         if statistics:

From noreply at buildbot.pypy.org  Mon Apr 30 11:57:30 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 11:57:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20120430095730.796CF82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r54823:87bf11738d67
Date: 2012-04-30 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/87bf11738d67/

Log:	merge heads

diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -49,7 +49,8 @@
                 optimizations.append(OptFfiCall())
 
     if ('rewrite' not in enable_opts or 'virtualize' not in enable_opts
-        or 'heap' not in enable_opts or 'unroll' not in enable_opts):
+        or 'heap' not in enable_opts or 'unroll' not in enable_opts
+        or 'pure' not in enable_opts):
         optimizations.append(OptSimplify())
 
     return optimizations, unroll
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -257,8 +257,8 @@
             opnum == rop.COPYSTRCONTENT or       # no effect on GC struct/array
             opnum == rop.COPYUNICODECONTENT):    # no effect on GC struct/array
             return
-        assert opnum != rop.CALL_PURE
         if (opnum == rop.CALL or
+            opnum == rop.CALL_PURE or
             opnum == rop.CALL_MAY_FORCE or
             opnum == rop.CALL_RELEASE_GIL or
             opnum == rop.CALL_ASSEMBLER):
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -525,6 +525,7 @@
 
     @specialize.argtype(0)
     def _emit_operation(self, op):
+        assert op.getopnum() != rop.CALL_PURE
         for i in range(op.numargs()):
             arg = op.getarg(i)
             try:

From noreply at buildbot.pypy.org  Mon Apr 30 12:49:25 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 12:49:25 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Replace hacking around an llop with just
 a function fishing around
Message-ID: <20120430104925.BAF8982F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54824:cacf5488f1a5
Date: 2012-04-30 12:49 +0200
http://bitbucket.org/pypy/pypy/changeset/cacf5488f1a5/

Log:	Replace hacking around an llop with just a function fishing around
	in the config.

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -18,6 +18,12 @@
     """
     pass
 
+def stm_is_enabled():
+    """Check if we are translating with STM enabled or no.  This function
+    is here rather than in rstm.py to avoid that you have to import rstm.py.
+    """
+    return None   # means 'not translated at all'
+
 # ____________________________________________________________
 # Annotation and specialization
 
@@ -50,6 +56,18 @@
         return hop.genop('gc_set_max_heap_size', [v_nbytes],
                          resulttype=lltype.Void)
 
+class StmIsEnabled(ExtRegistryEntry):
+    _about_ = stm_is_enabled
+
+    def compute_result_annotation(self):
+        config = self.bookkeeper.annotator.translator.config
+        result = config.translation.stm
+        return self.bookkeeper.immutablevalue(result)
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
 def can_move(p):
     """Check if the GC object 'p' is at an address that can move.
     Must not be called with None.  With non-moving GCs, it is always False.
@@ -159,7 +177,7 @@
     if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc':
         # perform a write barrier that copies necessary flags from
         # source to dest
-        if llop.stm_is_enabled(lltype.Bool) or (
+        if stm_is_enabled() or (
            not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest,
                                                 source_start, dest_start,
                                                 length)):
@@ -200,7 +218,7 @@
     field = getattr(p, TP._names[0])
     setattr(newp, TP._names[0], field)
 
-    if llop.stm_is_enabled(lltype.Bool):
+    if stm_is_enabled():
         # do the copying element by element
         i = 0
         while i < smallerlength:
diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py
--- a/pypy/rlib/rstm.py
+++ b/pypy/rlib/rstm.py
@@ -8,6 +8,7 @@
 from pypy.rlib.nonconst import NonConstant
 from pypy.translator.stm.stmgcintf import StmOperations
 
+from pypy.rlib.rgc import stm_is_enabled     # re-exported here
 
 
 NUM_THREADS_DEFAULT = 4     # XXX for now
diff --git a/pypy/rlib/test/test_rstm.py b/pypy/rlib/test/test_rstm.py
--- a/pypy/rlib/test/test_rstm.py
+++ b/pypy/rlib/test/test_rstm.py
@@ -135,3 +135,6 @@
     assert tl.getvalue() is p2
     tl.setvalue(None)
     assert tl.getvalue() is None
+
+def test_stm_is_enabled():
+    assert rstm.stm_is_enabled() is None    # not translated
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -399,7 +399,6 @@
     # direct_calls and maybe several casts, but it looks less heavy-weight
     # to keep them as operations until the genc stage)
 
-    'stm_is_enabled':         LLOp(canrun=True),  # sideeffects: test_lltyped.py
     'stm_getfield':           LLOp(sideeffects=False, canrun=True),
     'stm_getarrayitem':       LLOp(sideeffects=False, canrun=True),
     'stm_getinteriorfield':   LLOp(sideeffects=False, canrun=True),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -623,9 +623,6 @@
     from pypy.rlib.rtimer import read_timestamp
     return read_timestamp()
 
-def op_stm_is_enabled():
-    return False
-
 def op_stm_start_transaction():
     pass
 
diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py
--- a/pypy/rpython/lltypesystem/rffi.py
+++ b/pypy/rpython/lltypesystem/rffi.py
@@ -716,8 +716,7 @@
         string is already nonmovable.  Must be followed by a
         free_nonmovingbuffer call.
         """
-        from pypy.rpython.lltypesystem.lloperation import llop
-        if llop.stm_is_enabled(lltype.Bool) or rgc.can_move(data):
+        if rgc.stm_is_enabled() or rgc.can_move(data):
             count = len(data)
             buf = lltype.malloc(TYPEP.TO, count, flavor='raw')
             for i in range(count):
@@ -741,8 +740,7 @@
         # if 'buf' points inside 'data'.  This is only possible if we
         # followed the 2nd case in get_nonmovingbuffer(); in the first case,
         # 'buf' points to its own raw-malloced memory.
-        from pypy.rpython.lltypesystem.lloperation import llop
-        if llop.stm_is_enabled(lltype.Bool):
+        if rgc.stm_is_enabled():
             followed_2nd_path = False
         else:
             data = llstrtype(data)
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -5,7 +5,7 @@
 from pypy.rlib.objectmodel import _hash_string, enforceargs
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib.debug import ll_assert
-from pypy.rlib import jit
+from pypy.rlib import jit, rgc
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.robject import PyObjRepr, pyobj_repr
 from pypy.rpython.rmodel import inputconst, IntegerRepr
@@ -20,7 +20,6 @@
 from pypy.rpython.rmodel import Repr
 from pypy.rpython.lltypesystem import llmemory
 from pypy.tool.sourcetools import func_with_new_name
-from pypy.rpython.lltypesystem.lloperation import llop
 
 # ____________________________________________________________
 #
@@ -77,7 +76,7 @@
         # STM requires the slow character-by-character version,
         # which at least works without forcing the transaction
         # to become inevitable
-        if llop.stm_is_enabled(Bool):
+        if rgc.stm_is_enabled():
             i = 0
             while i < length:
                 dst.chars[dststart + i] = src.chars[srcstart + i]
diff --git a/pypy/translator/c/src/mem.h b/pypy/translator/c/src/mem.h
--- a/pypy/translator/c/src/mem.h
+++ b/pypy/translator/c/src/mem.h
@@ -259,5 +259,3 @@
 #define OP_GC_GET_RPY_TYPE_INDEX(x, r)   r = -1
 #define OP_GC_IS_RPY_INSTANCE(x, r)      r = 0
 #define OP_GC_DUMP_RPY_HEAP(fd, r)       r = 0
-
-#define OP_STM_IS_ENABLED(r)             r = 0
diff --git a/pypy/translator/c/test/test_lltyped.py b/pypy/translator/c/test/test_lltyped.py
--- a/pypy/translator/c/test/test_lltyped.py
+++ b/pypy/translator/c/test/test_lltyped.py
@@ -903,16 +903,3 @@
             return rstring_to_float(s)
         fn = self.getcompiled(llf, [int])
         assert fn(0) == 42.3
-
-    def test_stm_is_enabled(self):
-        from pypy.rpython.lltypesystem import lltype
-        from pypy.rpython.lltypesystem.lloperation import llop
-        # note that right now the situation of lloperation is a bit messy.
-        # stm_is_enabled is not "sideeffects=False" because otherwise
-        # it is constant-folded away by backendopt/constfold.py.
-        assert llop.stm_is_enabled.sideeffects == True
-        #
-        def llf(i):
-            return int(llop.stm_is_enabled(lltype.Bool))
-        fn = self.getcompiled(llf, [int])
-        assert fn(0) == 0
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -96,7 +96,7 @@
 
 def entry_point(argv):
     print "hello world"
-    assert llop.stm_is_enabled(lltype.Bool)
+    assert rstm.stm_is_enabled()
     if len(argv) > 1:
         glob.NUM_THREADS = int(argv[1])
         if len(argv) > 2:
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -162,11 +162,6 @@
             self.count_write_barrier += 1
         newoperations.append(op)
 
-    def stt_stm_is_enabled(self, newoperations, op):
-        c_True = Constant(True, lltype.Bool)
-        op = SpaceOperation('same_as', [c_True], op.result)
-        newoperations.append(op)
-
     def stt_malloc(self, newoperations, op):
         flags = op.args[1].value
         if flags['flavor'] == 'gc':

From noreply at buildbot.pypy.org  Mon Apr 30 12:56:32 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 12:56:32 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Comment.
Message-ID: <20120430105632.46A1382F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54825:45555c3c4cc0
Date: 2012-04-30 12:55 +0200
http://bitbucket.org/pypy/pypy/changeset/45555c3c4cc0/

Log:	Comment.

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -22,7 +22,8 @@
     """Check if we are translating with STM enabled or no.  This function
     is here rather than in rstm.py to avoid that you have to import rstm.py.
     """
-    return None   # means 'not translated at all'
+    return None   # means 'not translated at all';
+                  # in "if stm_is_enabled()" it is equivalent to False
 
 # ____________________________________________________________
 # Annotation and specialization

From noreply at buildbot.pypy.org  Mon Apr 30 15:25:09 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 15:25:09 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix tests.
Message-ID: <20120430132509.EB07C82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54826:e1b2c505d3ee
Date: 2012-04-30 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/e1b2c505d3ee/

Log:	Fix tests.

diff --git a/pypy/translator/c/test/test_standalone.py b/pypy/translator/c/test/test_standalone.py
--- a/pypy/translator/c/test/test_standalone.py
+++ b/pypy/translator/c/test/test_standalone.py
@@ -450,6 +450,8 @@
         out, err = cbuilder.cmdexec("", expect_crash=True)
         assert out.strip() == ''
         lines = err.strip().splitlines()
+        # ignore the lines pypy_g_ll_deallocator: refcount only
+        lines = [line for line in lines if 'll_deallocator' not in line]
         idx = lines.index('Fatal RPython error: ValueError')   # assert found
         lines = lines[:idx+1]
         assert len(lines) >= 4
@@ -461,6 +463,7 @@
         out2, err2 = cbuilder.cmdexec("x", expect_crash=True)
         assert out2.strip() == ''
         lines2 = err2.strip().splitlines()
+        lines2 = [line for line in lines2 if 'll_deallocator' not in line]
         idx = lines2.index('Fatal RPython error: KeyError')    # assert found
         lines2 = lines2[:idx+1]
         l0, l1, l2 = lines2[-4:-1]
@@ -489,6 +492,8 @@
         out, err = cbuilder.cmdexec("", expect_crash=True)
         assert out.strip() == 'done.'
         lines = err.strip().splitlines()
+        # ignore the lines pypy_g_ll_deallocator: refcount only
+        lines = [line for line in lines if 'll_deallocator' not in line]
         idx = lines.index('Fatal RPython error: KeyError')    # assert found
         lines = lines[:idx+1]
         assert len(lines) >= 5
@@ -526,6 +531,8 @@
         out, err = cbuilder.cmdexec("", expect_crash=True)
         assert out.strip() == 'done.'
         lines = err.strip().splitlines()
+        # ignore the lines pypy_g_ll_deallocator: refcount only
+        lines = [line for line in lines if 'll_deallocator' not in line]
         idx = lines.index('Fatal RPython error: KeyError')     # assert found
         lines = lines[:idx+1]
         assert len(lines) >= 5
@@ -562,6 +569,8 @@
         out, err = cbuilder.cmdexec("", expect_crash=True)
         assert out.strip() == ''
         lines = err.strip().splitlines()
+        # ignore the lines pypy_g_ll_deallocator: refcount only
+        lines = [line for line in lines if 'll_deallocator' not in line]
         idx = lines.index('Fatal RPython error: ValueError')    # assert found
         lines = lines[:idx+1]
         assert len(lines) >= 5

From noreply at buildbot.pypy.org  Mon Apr 30 15:38:48 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 15:38:48 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix test.
Message-ID: <20120430133848.9709E82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54827:9f7e3a9f788f
Date: 2012-04-30 15:38 +0200
http://bitbucket.org/pypy/pypy/changeset/9f7e3a9f788f/

Log:	Fix test.

diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -1139,6 +1139,8 @@
             pass
 
         def f():
+            # first, make sure we don't start with a nursery almost full
+            rgc.collect(0)
             # we need at least 1 obj to allocate a nursery
             a = A()
             nf_a = llop.gc_adr_of_nursery_free(llmemory.Address)

From noreply at buildbot.pypy.org  Mon Apr 30 15:52:49 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 15:52:49 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix tests
Message-ID: <20120430135249.D265582F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54828:69f7007806f7
Date: 2012-04-30 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/69f7007806f7/

Log:	Fix tests

diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -61,6 +61,19 @@
         if tracing:
             self.tracer = Tracer()
 
+    def eval_entry_point(self, graph, args=()):
+        from pypy.rpython.lltypesystem import lltype, rffi
+        ARGV = graph.startblock.inputargs[1].concretetype
+        args = [''] + list(args)
+        ll_args = lltype.malloc(ARGV.TO, len(args), flavor='raw')
+        for i in range(len(args)):
+            ll_args[i] = rffi.str2charp(args[i])
+        res = self.eval_graph(graph, [len(args), ll_args])
+        for arg in ll_args:
+            rffi.free_charp(arg)
+        lltype.free(ll_args, flavor='raw')
+        return res
+
     def eval_graph(self, graph, args=(), recursive=False):
         llframe = self.frame_class(graph, args, self)
         if self.tracer and not recursive:
diff --git a/pypy/rpython/memory/gctransform/test/test_transform.py b/pypy/rpython/memory/gctransform/test/test_transform.py
--- a/pypy/rpython/memory/gctransform/test/test_transform.py
+++ b/pypy/rpython/memory/gctransform/test/test_transform.py
@@ -12,21 +12,17 @@
 
 class LLInterpedTranformerTests:
 
-    def llinterpreter_for_transformed_graph(self, f, args_s):
+    def llinterpreter_for_transformed_graph(self, f):
         from pypy.rpython.llinterp import LLInterpreter
         from pypy.translator.c.genc import CStandaloneBuilder
         from pypy.translator.c import gc
+        from pypy.annotation.listdef import s_list_of_strings
 
-        t = rtype(f, args_s)
+        t = rtype(f, [s_list_of_strings])
         # XXX we shouldn't need an actual gcpolicy here.
         cbuild = CStandaloneBuilder(t, f, t.config, gcpolicy=self.gcpolicy)
         db = cbuild.generate_graphs_for_llinterp()
         graph = cbuild.getentrypointptr()._obj.graph
-        # arguments cannot be GC objects because nobody would put a
-        # proper header on them
-        for v in graph.getargs():
-            if isinstance(v.concretetype, lltype.Ptr):
-                assert v.concretetype.TO._gckind != 'gc', "fix the test!"
         llinterp = LLInterpreter(t.rtyper)
         if conftest.option.view:
             t.view()
@@ -47,20 +43,22 @@
                 d = C()
                 d.x = 2
                 return d
-        def f(x):
+        def f(argv):
+            x = int(argv[1])
             return g(x).x
 
-        llinterp, graph = self.llinterpreter_for_transformed_graph(f, [SomeInteger()])
+        llinterp, graph = self.llinterpreter_for_transformed_graph(f)
 
-        res = llinterp.eval_graph(graph, [0])
-        assert res == f(0)
-        res = llinterp.eval_graph(graph, [1])
-        assert res == f(1)
+        res = llinterp.eval_entry_point(graph, ["0"])
+        assert res == f(["", "0"])
+        res = llinterp.eval_entry_point(graph, ["1"])
+        assert res == f(["", "1"])
 
     def test_simple_varsize(self):
         from pypy.annotation.model import SomeInteger
 
-        def f(x):
+        def f(argv):
+            x = int(argv[1])
             r = []
             for i in range(x):
                 if i % 2:
@@ -68,17 +66,18 @@
             return len(r)
 
 
-        llinterp, graph = self.llinterpreter_for_transformed_graph(f, [SomeInteger()])
+        llinterp, graph = self.llinterpreter_for_transformed_graph(f)
 
-        res = llinterp.eval_graph(graph, [0])
-        assert res == f(0)
-        res = llinterp.eval_graph(graph, [10])
-        assert res == f(10)
+        res = llinterp.eval_entry_point(graph, ["0"])
+        assert res == f(["", "0"])
+        res = llinterp.eval_entry_point(graph, ["10"])
+        assert res == f(["", "10"])
 
     def test_str(self):
         from pypy.annotation.model import SomeBool
 
-        def f(flag):
+        def f(argv):
+            flag = int(argv[1])
             if flag:
                 x = 'a'
             else:
@@ -86,12 +85,12 @@
             return len(x + 'a')
 
 
-        llinterp, graph = self.llinterpreter_for_transformed_graph(f, [SomeBool()])
+        llinterp, graph = self.llinterpreter_for_transformed_graph(f)
 
-        res = llinterp.eval_graph(graph, [True])
-        assert res == f(True)
-        res = llinterp.eval_graph(graph, [False])
-        assert res == f(False)
+        res = llinterp.eval_entry_point(graph, ["1"])
+        assert res == f(["", "1"])
+        res = llinterp.eval_entry_point(graph, ["0"])
+        assert res == f(["", "0"])
 
 class _TestGCTransformer(BaseGCTransformer):
 

From noreply at buildbot.pypy.org  Mon Apr 30 15:52:51 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 15:52:51 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix test
Message-ID: <20120430135251.1E94982F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54829:b0058053a047
Date: 2012-04-30 15:50 +0200
http://bitbucket.org/pypy/pypy/changeset/b0058053a047/

Log:	Fix test

diff --git a/pypy/rpython/memory/gctransform/test/test_framework.py b/pypy/rpython/memory/gctransform/test/test_framework.py
--- a/pypy/rpython/memory/gctransform/test/test_framework.py
+++ b/pypy/rpython/memory/gctransform/test/test_framework.py
@@ -30,7 +30,7 @@
     def entrypoint(argv):
         a = A()
         a.b = g(1)
-        return str(a.b)
+        return a.b
 
     from pypy.rpython.llinterp import LLInterpreter
     from pypy.translator.c.genc import CStandaloneBuilder
@@ -43,18 +43,15 @@
     entrypointptr = cbuild.getentrypointptr()
     entrygraph = entrypointptr._obj.graph
 
-    r_list_of_strings = t.rtyper.getrepr(s_list_of_strings)
-    ll_argv = r_list_of_strings.convert_const([])
-
     llinterp = LLInterpreter(t.rtyper)
 
     # FIIIIISH
     setupgraph = db.gctransformer.frameworkgc_setup_ptr.value._obj.graph
     llinterp.eval_graph(setupgraph, [])
 
-    res = llinterp.eval_graph(entrygraph, [ll_argv])
+    res = llinterp.eval_entry_point(entrygraph, [])
 
-    assert ''.join(res.chars) == "2"
+    assert res == 2
 
 def test_cancollect():
     S = lltype.GcStruct('S', ('x', lltype.Signed))

From noreply at buildbot.pypy.org  Mon Apr 30 15:52:52 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 15:52:52 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Fix tests.
Message-ID: <20120430135252.57A7682F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54830:d17ad6dd8c43
Date: 2012-04-30 15:52 +0200
http://bitbucket.org/pypy/pypy/changeset/d17ad6dd8c43/

Log:	Fix tests.

diff --git a/pypy/rpython/memory/gctransform/test/test_refcounting.py b/pypy/rpython/memory/gctransform/test/test_refcounting.py
--- a/pypy/rpython/memory/gctransform/test/test_refcounting.py
+++ b/pypy/rpython/memory/gctransform/test/test_refcounting.py
@@ -33,15 +33,16 @@
                 return d
         def g(x):
             return h(x).x
-        def f(x):
+        def f(argv):
+            x = int(argv[1])
             r = g(x)
             return r + delcounter.dels
 
-        llinterp, graph = self.llinterpreter_for_transformed_graph(f, [SomeInteger()])
+        llinterp, graph = self.llinterpreter_for_transformed_graph(f)
 
-        res = llinterp.eval_graph(graph, [1])
+        res = llinterp.eval_entry_point(graph, ["1"])
         assert res == 1
-        res = llinterp.eval_graph(graph, [0])
+        res = llinterp.eval_entry_point(graph, ["0"])
         assert res == 3
 
     def test_raw_instance_flavor(self):
@@ -61,7 +62,7 @@
             x = X()
             x.y = Y()
             return x
-        def f():
+        def f(argv):
             from pypy.rlib.objectmodel import free_non_gc_object
             state.freed_counter = 0
             x = g()
@@ -71,8 +72,8 @@
             free_non_gc_object(x)
             # for now we have no automatic decref when free_non_gc_object() is
             # called
-        llinterp, graph = self.llinterpreter_for_transformed_graph(f, [])
-        llinterp.eval_graph(graph, [])
+        llinterp, graph = self.llinterpreter_for_transformed_graph(f)
+        llinterp.eval_entry_point(graph, [])
 
 def test_simple_barrier():
     S = lltype.GcStruct("S", ('x', lltype.Signed))

From noreply at buildbot.pypy.org  Mon Apr 30 16:31:10 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 16:31:10 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Bah.
Message-ID: <20120430143110.5A18B82F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54831:9f13592e6dda
Date: 2012-04-30 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/9f13592e6dda/

Log:	Bah.

diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -412,6 +412,6 @@
     'ref': RefcountingGcPolicy,
     'none': NoneGcPolicy,
     'framework': FrameworkGcPolicy,
-    'framework+asmgcroot': AsmGcRootFrameworkGcPolicy,
+    'framework+asmgcc': AsmGcRootFrameworkGcPolicy,
     'framework+stm': StmFrameworkGcPolicy,
 }

From noreply at buildbot.pypy.org  Mon Apr 30 16:31:11 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 16:31:11 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Remove unused declaration again.
Message-ID: <20120430143111.958D082F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54832:f1c27769bc89
Date: 2012-04-30 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/f1c27769bc89/

Log:	Remove unused declaration again.

diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -45,10 +45,6 @@
                             threadsafe=True)  # release the GIL, but most
                                               # importantly, reacquire it
                                               # around the callback
-c_thread_start_NOGIL = llexternal('RPyThreadStart', [CALLBACK], rffi.LONG,
-                                  _callable=_emulated_start_new_thread,
-                                  _nowrapper=True,  # just call directly
-                                  random_effects_on_gcobjs=False)
 c_thread_get_ident = llexternal('RPyThreadGetIdent', [], rffi.LONG,
                                 _nowrapper=True)    # always call directly
 

From noreply at buildbot.pypy.org  Mon Apr 30 16:31:12 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 16:31:12 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: - Fix test.
Message-ID: <20120430143112.D041382F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54833:23f1f1471eef
Date: 2012-04-30 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/23f1f1471eef/

Log:	- Fix test.
	- Obviously, config.translation.stm is true here. It's not that
	that we should test...

diff --git a/pypy/module/transaction/interp_transaction.py b/pypy/module/transaction/interp_transaction.py
--- a/pypy/module/transaction/interp_transaction.py
+++ b/pypy/module/transaction/interp_transaction.py
@@ -127,13 +127,13 @@
         ec = self.space.getexecutioncontext()    # create it if needed
         assert len(ec._transaction_pending) == 0
         #
-        if self.space.config.translation.stm:
+        if self.space.config.objspace.std.withmethodcache:
             from pypy.objspace.std.typeobject import MethodCache
             ec._methodcache = MethodCache(self.space)
         #
         self.space.call_args(self.w_callback, self.args)
         #
-        if self.space.config.translation.stm:
+        if self.space.config.objspace.std.withmethodcache:
             # remove the method cache again now, to prevent it from being
             # promoted to a GLOBAL
             ec._methodcache = None
diff --git a/pypy/module/transaction/test/test_epoll.py b/pypy/module/transaction/test/test_epoll.py
--- a/pypy/module/transaction/test/test_epoll.py
+++ b/pypy/module/transaction/test/test_epoll.py
@@ -1,5 +1,6 @@
 import py
 from pypy.conftest import gettestobjspace
+py.test.skip("epoll support disabled for now")
 
 
 class AppTestEpoll: 
diff --git a/pypy/module/transaction/test/test_interp_transaction.py b/pypy/module/transaction/test/test_interp_transaction.py
--- a/pypy/module/transaction/test/test_interp_transaction.py
+++ b/pypy/module/transaction/test/test_interp_transaction.py
@@ -3,6 +3,10 @@
 
 
 class FakeSpace:
+    class config:
+        class objspace:
+            class std:
+                withmethodcache = False
     def __init__(self):
         self._spacecache = {}
     def getexecutioncontext(self):

From noreply at buildbot.pypy.org  Mon Apr 30 16:31:14 2012
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 30 Apr 2012 16:31:14 +0200 (CEST)
Subject: [pypy-commit] pypy stm-gc: Add documentation for these options.
Message-ID: <20120430143114.1523482F50@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm-gc
Changeset: r54834:5284aa981a13
Date: 2012-04-30 16:29 +0200
http://bitbucket.org/pypy/pypy/changeset/5284aa981a13/

Log:	Add documentation for these options.

diff --git a/pypy/doc/config/objspace.usemodules.transaction.txt b/pypy/doc/config/objspace.usemodules.transaction.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules.transaction.txt
@@ -0,0 +1,1 @@
+Use the STM-based 'transaction' module.
diff --git a/pypy/doc/config/translation.stm.txt b/pypy/doc/config/translation.stm.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/translation.stm.txt
@@ -0,0 +1,3 @@
+Enable global STM transformation of the translated interpreter.
+Trades performance for running on multiple cores in individual
+"transactions".

From noreply at buildbot.pypy.org  Mon Apr 30 18:38:02 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 30 Apr 2012 18:38:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: remove poll from windows,
	add skip to test
Message-ID: <20120430163802.2DBEC82F50@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54835:67d53ef201a6
Date: 2012-04-30 19:36 +0300
http://bitbucket.org/pypy/pypy/changeset/67d53ef201a6/

Log:	remove poll from windows, add skip to test

diff --git a/pypy/module/select/__init__.py b/pypy/module/select/__init__.py
--- a/pypy/module/select/__init__.py
+++ b/pypy/module/select/__init__.py
@@ -2,6 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 
 import sys
+import os
 
 
 class Module(MixedModule):
@@ -9,11 +10,13 @@
     }
 
     interpleveldefs = {
-        'poll'  : 'interp_select.poll',
         'select': 'interp_select.select',
         'error' : 'space.fromcache(interp_select.Cache).w_error'
     }
 
+    if os.name =='posix':
+        interpleveldefs['poll'] = 'interp_select.poll'
+
     if sys.platform.startswith('linux'):
         interpleveldefs['epoll'] = 'interp_epoll.W_Epoll'
         from pypy.module.select.interp_epoll import cconfig, public_symbols
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -214,6 +214,8 @@
 
     def test_poll(self):
         import select
+        if not hasattr(select, 'poll'):
+            skip("no select.poll() on this platform")
         readend, writeend = self.getpair()
         try:
             class A(object):

From noreply at buildbot.pypy.org  Mon Apr 30 19:05:02 2012
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 30 Apr 2012 19:05:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: correct testing of localtime(-1)
Message-ID: <20120430170502.77C4E82F50@wyvern.cs.uni-duesseldorf.de>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r54836:52324a85becd
Date: 2012-04-30 20:04 +0300
http://bitbucket.org/pypy/pypy/changeset/52324a85becd/

Log:	correct testing of localtime(-1)

diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -64,6 +64,7 @@
 
     def test_localtime(self):
         import time as rctime
+        import os
         raises(TypeError, rctime.localtime, "foo")
         rctime.localtime()
         rctime.localtime(None)
@@ -75,6 +76,10 @@
         assert 0 <= (t1 - t0) < 1.2
         t = rctime.time()
         assert rctime.localtime(t) == rctime.localtime(t)
+        if os.name == 'nt':
+            raises(ValueError, rctime.localtime, -1)
+        else:
+            rctime.localtime(-1)
 
     def test_mktime(self):
         import time as rctime
@@ -108,8 +113,8 @@
         assert long(rctime.mktime(rctime.gmtime(t))) - rctime.timezone == long(t)
         ltime = rctime.localtime()
         assert rctime.mktime(tuple(ltime)) == rctime.mktime(ltime)
-
-        assert rctime.mktime(rctime.localtime(-1)) == -1
+        if os.name != 'nt':
+            assert rctime.mktime(rctime.localtime(-1)) == -1
 
     def test_asctime(self):
         import time as rctime